Archive ingestion (#88)

SumoLogic-Labs · Jun 27, 2021 · 8f764d7 · 8f764d7
1 parent c756725
commit 8f764d7
Show file tree

Hide file tree

Showing 46 changed files with 769 additions and 564 deletions.
diff --git a/.github/workflows/sumocli-build.yml b/.github/workflows/sumocli-build.yml
@@ -49,6 +49,8 @@ jobs:
         uses: actions/checkout@v2
       - name: Download Dependencies
         run: go mod download
+      - name: Run Tests
+        run: go test ./...
       - name: Build Sumocli
         shell: pwsh
         run: |
@@ -100,6 +102,8 @@ jobs:
         uses: actions/checkout@v2
       - name: Download Dependencies
         run: go mod download
+      - name: Run Tests
+        run: go test ./...
       - name: Build Sumocli
         shell: pwsh
         run: |
@@ -139,6 +143,8 @@ jobs:
         uses: actions/checkout@v2
       - name: Download Dependencies
         run: go mod download
+      - name: Run Tests
+        run: go test ./...
       - name: Build Sumocli
         shell: pwsh
         run: |
@@ -207,6 +213,8 @@ jobs:
         uses: actions/checkout@v2
       - name: Download Dependencies
         run: go mod download
+      - name: Run Tests
+        run: go test ./...
       - name: Build Sumocli
         shell: bash
         run: |

diff --git a/.github/workflows/sumocli-tests.yml b/.github/workflows/sumocli-tests.yml
@@ -3,17 +3,47 @@ name: Sumocli Tests
 on: [pull_request, push]
 
 jobs:
-  compile_authorizer_app:
-    name: Test Go Applications
+  compile_linux_binary:
+    name: Test and Compile Linux Binary
     runs-on: ubuntu-latest
     steps:
       - name: Setup Go Environment
         uses: actions/setup-go@v2
         with:
-          go-version: '1.16.2'
+          go-version: '1.16.5'
+      - name: Checkout Code
+        uses: actions/checkout@v2
+      - name: Build Sumocli
+        run: go build ./cmd/sumocli
+      - name: Run Tests
+        run: go test ./...
 
+  compile_macos_binary:
+    name: Test and Compile macOS Binary
+    runs-on: macos-latest
+    steps:
+      - name: Setup Go Environment
+        uses: actions/setup-go@v2
+        with:
+          go-version: '1.16.5'
       - name: Checkout Code
         uses: actions/checkout@v2
+      - name: Build Sumocli
+        run: go build ./cmd/sumocli
+      - name: Run Tests
+        run: go test ./...
 
+  compile_windows_binary:
+    name: Test and Compile Windows Binary
+    runs-on: windows-latest
+    steps:
+      - name: Setup Go Environment
+        uses: actions/setup-go@v2
+        with:
+          go-version: '1.16.5'
+      - name: Checkout Code
+        uses: actions/checkout@v2
       - name: Build Sumocli
-        run: go build -v ./cmd/sumocli
+        run: go build ./cmd/sumocli
+      - name: Run Tests
+        run: go test ./...
diff --git a/COMPATIBILITY.md b/COMPATIBILITY.md
@@ -0,0 +1,100 @@
+# Supported Capabilities
+
+The following tables outline the functionality that Sumocli currently provides.
+
+## Sumo Logic API
+
+You can find further information about the Sumo Logic APIs in their [documentation](https://help.sumologic.com/APIs).
+
+| API | Status | Notes |
+| --- | --- | --- |
+| Access Keys | :white_check_mark: | |
+| Account | :white_check_mark: | |
+| Apps (Beta) | :white_check_mark: | |
+| Archive Ingestion | :white_check_mark: | |
+| Content | :white_check_mark: | |
+| Collectors | :white_check_mark: | |
+| Dashboards | :white_check_mark: | |
+| Dynamic Parsing | :white_check_mark: | |
+| Field Extraction Rules | :white_check_mark: | |
+| Field Management | :white_check_mark: | |
+| Folders | :white_check_mark: | |
+| Health Events | :white_check_mark: | |
+| Ingest Budgets | :white_check_mark: | |
+| Ingest Budgets v2 | :white_check_mark: | |
+| Lookup Tables | :white_check_mark: | Upload CSV file functionality is not currently supported however, it will be added in a future release. | 
+| Monitors | | Functionality is still in development. |
+| Partitions | :white_check_mark: | |
+| Password Policy | :white_check_mark: | |
+| Permissions | :white_check_mark: | |
+| Roles | :white_check_mark: | |
+| SAML Configuration | :white_check_mark: | |
+| Scheduled Views | :white_check_mark: | |
+| Service Allowlist | :white_check_mark: | |
+| Sources | :white_check_mark: | Simple commands like get, list, delete are supported. Support for creating different sources are listed under Sumo Logic Sources Support. |
+| Tokens | :white_check_mark: | |
+| Users | :white_check_mark: | |
+
+## Sumo Logic Sources Support
+
+You can find further information about supported Sumo Logic sources in their [documentation](https://help.sumologic.com/03Send-Data/Sources/03Use-JSON-to-Configure-Sources)
+
+### Log Sources for Installed Collectors
+
+| Source | Status | Notes |
+| --- | --- | --- |
+| Local File Source | :white_check_mark: | |
+| Remote File Source | | |
+| Local Windows Event Log Source | | |
+| Remote Windows Event Log Source | | |
+| Local Windows Performance Source | | |
+| Remote Windows Performance Source | | |
+| Windows Active Directory Source | | |
+| Syslog Source | | |
+| Script Source | | |
+| Docker Log Source | | |
+| Docker Stats Source | | |
+
+### Metric Sources for Installed Collectors
+
+| Source | Status | Notes |
+| --- | --- | --- |
+| Host Metrics Source | | 
+| Streaming Metrics Source | |
+
+### Log Sources for Hosted Collectors
+
+| Source | Status | Notes |
+| --- | --- | --- |
+| Akamai SIEM API Source | | |
+| Amazon S3 Source | | |
+| AWS S3 Archive Source | :white_check_mark: | |
+| AWS Elastic Load Balancing Source | | |
+| AWS CloudFront Source | | |
+| AWS CloudTrail Source | :white_check_mark: | |
+| AWS S3 Audit Source | | |
+| AWS Metadata (Tag) Source | | |
+| Azure Event Hubs Source | | |
+| Carbon Black Cloud Source | | |
+| Cloud Syslog Source | | |
+| Cisco AMP Source | | |
+| Crowdstrike FDR Source | | |
+| Crowdstrike Source | | |
+| CSE AWS EC2 Inventory Source | | |
+| Duo Source | | |
+| HTTP Source | :white_check_mark: | |
+| Microsoft Graph Security API Source | | |
+| Mimecast Source | | |
+| Netskope Source | | |
+| Okta Source | | |
+| Proofpoint TAP Source | | |
+| Salesforce Source | | |
+| Sophos Central Source | | |
+| Tenable Source | | |
+
+### Metrics Sources for Hosted Collectors
+
+| Source | Status | Notes |
+| --- | --- | --- |
+| AWS CloudWatch Source | | |
+
diff --git a/README.md b/README.md
@@ -25,7 +25,7 @@ Sumocli uses two authentication methods;
 - Environment variables
 - Credentials file
 
-When you run a command in sumocli it will first check to see if a credentials file exists, if it can't find one then it will fall back to environment variables this is to ensure that sumocli can run in CI/CD pipelines. 
+When you run a command in sumocli it will first check to see if a credentials file exists, if it can't find one then it will fall back to environment variables. This is to ensure that sumocli can run in CI/CD pipelines. 
 The sections below explain the requirements for each authentication type.
 
 ### Environment Variables
@@ -46,6 +46,15 @@ https://help.sumologic.com/APIs/General-API-Information/Sumo-Logic-Endpoints-and
 ### Credentials File
 
 The credentials file stores the same information as the environment variables however, it can be generated interactively using `sumocli login`. 
+The Sumo Logic Access ID and Access Key are encrypted (using AES-256) before being written to the credentials file to reduce the risk of the credentials being 
+used outside of Sumocli.
+
+Encryption of the Sumo Logic Access ID and Access Key was added in v0.9.0 of Sumocli, if you are running
+an earlier version of Sumocli you will need to regenerate your credentials file by running `sumocli login` if you want 
+to leverage encryption at rest.
+If you need to know which Access ID sumocli is configured to use you can run `sumocli login --showAccessId` and
+the plaintext access ID will be displayed.
+
 The credential file is stored in the following locations depending on your operating system.
 
 ```
@@ -60,8 +69,10 @@ The contents of the credential file is as follows:
 
 ```
 {
+  "version": "v1",
   "accessid": "abcefghi",
   "accesskey": "AbCeFG123",
+  "region": "<regioncode>",
   "endpoint": "https://api.<regioncode>.sumologic.com/api/"
 }
 ```
@@ -70,6 +81,8 @@ The contents of the credential file is as follows:
 
 Documentation for each command can be found by using `sumocli <command> --help`
 
+You can find a list of API endpoints and sources that Sumocli supports in the [supported capabilities](COMPATIBILITY.md) documentation.
+
 ## Contributing
 
 Clone or fork the repo, make your changes and create a pull request. 

diff --git a/api/auth.go b/api/auth.go
@@ -7,6 +7,7 @@ type SumoApiEndpoint struct {
 }
 
 type SumoAuth struct {
+	Version   string `json:"version"`
 	AccessId  string `json:"accessid"`
 	AccessKey string `json:"accesskey"`
 	Region    string `json:"region"`

diff --git a/go.mod b/go.mod
@@ -14,15 +14,18 @@ require (
 	github.com/aws/aws-sdk-go-v2/service/kms v1.3.1
 	github.com/aws/aws-sdk-go-v2/service/s3 v1.9.0
 	github.com/aws/aws-sdk-go-v2/service/sns v1.5.0
-	github.com/aws/aws-sdk-go-v2/service/sts v1.4.1 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.4.1
 	github.com/jarcoal/httpmock v1.0.8
 	github.com/manifoldco/promptui v0.8.0
-	github.com/mattn/go-isatty v0.0.12 // indirect
+	github.com/mattn/go-colorable v0.1.8 // indirect
 	github.com/mitchellh/mapstructure v1.3.3 // indirect
 	github.com/rs/zerolog v1.20.0
 	github.com/satori/go.uuid v1.2.0 // indirect
 	github.com/spf13/cobra v1.1.1
 	github.com/spf13/viper v1.7.1
 	github.com/stretchr/testify v1.6.1 // indirect
-	golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd // indirect
+	golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad // indirect
+	golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c
+	golang.org/x/text v0.3.5 // indirect
+	gopkg.in/yaml.v2 v2.3.0 // indirect
 )
diff --git a/go.sum b/go.sum
@@ -189,8 +189,9 @@ github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzR
 github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
 github.com/manifoldco/promptui v0.8.0 h1:R95mMF+McvXZQ7j1g8ucVZE1gLP3Sv6j9vlF9kyRqQo=
 github.com/manifoldco/promptui v0.8.0/go.mod h1:n4zTdgP0vr0S3w7/O/g98U+e0gwLScEXGwov2nIKuGQ=
-github.com/mattn/go-colorable v0.0.9 h1:UVL0vNpWh04HeJXV0KLcaT7r06gOH2l4OW6ddYRUIY4=
 github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
+github.com/mattn/go-colorable v0.1.8 h1:c1ghPdyEDarC70ftn0y+A/Ee++9zz8ljHG1b13eJ0s8=
+github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
 github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
 github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
 github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY=
@@ -282,8 +283,9 @@ golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnf
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0 h1:hb9wdF1z5waM+dSIICn1l0DkLVDT3hqhhQsDNUmHPRE=
 golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad h1:DN0cp81fZ3njFcrLCytUHRSUkqBjfTo4Tx9RJTWs0EY=
+golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -338,13 +340,17 @@ golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884=
-golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c h1:VwygUrnw9jn88c4u8GD3rZQbqrP/tgas88tPUbBxQrk=
+golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
+golang.org/x/text v0.3.5 h1:i6eZZ+zk0SOf0xgBpEpPD18qWcJda6q1sxt3S0kzyUQ=
+golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
@@ -400,8 +406,9 @@ gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=
 gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74=
 gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
 gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU=
+gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=