feat: optimus upgrades (#770)

raystack · Jul 20, 2023 · 256b989 · 256b989
1 parent f146fcf
commit 256b989
Show file tree

Hide file tree

Showing 525 changed files with 27,064 additions and 18,679 deletions.
diff --git a/.github/workflows/publish-latest.yml b/.github/workflows/publish-latest.yml
@@ -28,10 +28,10 @@ jobs:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
       - name: Run GoReleaser
-        uses: goreleaser/goreleaser-action@v2.6.1
+        uses: goreleaser/goreleaser-action@v4
         with:
           distribution: goreleaser
-          version: v1.8.3
+          version: v1.19
           args: -f .goreleaser.latest.yml --rm-dist --skip-validate
         env:
           GITHUB_TOKEN: ${{ secrets.GO_RELEASER_TOKEN }}
@@ -59,4 +59,4 @@ jobs:
         run: |
           git config --global user.email "41898282+github-actions[bot]@users.noreply.github.com"
           git config --global user.name "github-actions[bot]"
-          yarn deploy
+          yarn deploy
diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml
@@ -33,12 +33,12 @@ jobs:
           go-version: '1.19'
       - name: test binaries
         run: make test-ci
-      - name: Install goveralls
-        run: go install github.com/mattn/goveralls@latest
-      - name: Send coverage
-        env:
-          COVERALLS_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        run: goveralls -coverprofile=coverage.txt -service=github
+      # - name: Install goveralls
+      #   run: go install github.com/mattn/goveralls@latest
+      # - name: Send coverage
+      #   env:
+      #     COVERALLS_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      #   run: goveralls -coverprofile=coverage.txt -service=github
   integration-test:
     runs-on: ubuntu-latest
     services:

diff --git a/.golangci.yml b/.golangci.yml
@@ -60,30 +60,30 @@ linters-settings:
     rules:
       - name: atomic
       - name: context-as-argument
-#      - name: context-keys-type #perf issue
+      #      - name: context-keys-type #perf issue
       - name: defer
       - name: dot-imports
       - name: empty-block
       - name: error-naming
       - name: error-return
-#      - name: error-strings
+      #      - name: error-strings
       - name: early-return
-#      - name: errorf #perf issue
-#      - name: exported
+      #      - name: errorf #perf issue
+      #      - name: exported
       - name: if-return
       - name: increment-decrement
       - name: indent-error-flow
-#      - name: flag-parameter
+      #      - name: flag-parameter
       - name: modifies-parameter
-#      - name: modifies-value-receiver #perf issue
+      #      - name: modifies-value-receiver #perf issue
       - name: package-comments
       - name: range
       - name: receiver-naming
       - name: redefines-builtin-id
       - name: superfluous-else
-#      - name: time-naming #perf issue
+      #      - name: time-naming #perf issue
       - name: unexported-naming
-#      - name: var-declaration #perf issue
+      #      - name: var-declaration #perf issue
       - name: var-naming
       - name: unused-receiver
       - name: unused-parameter
@@ -95,18 +95,18 @@ linters-settings:
     numbers: true
   gomnd:
     ignored-numbers: # Why we have a big range of file permissions
-      - '0o600'
-      - '0o644'
-      - '0o655'
-      - '0o666'
-      - '0o770'
-      - '0o755'
-      - '0o765'
-      - '0o777'
+      - "0o600"
+      - "0o644"
+      - "0o655"
+      - "0o666"
+      - "0o770"
+      - "0o755"
+      - "0o765"
+      - "0o777"
     ignored-functions:
-      - 'survey.MinLength'
-      - 'survey.MaxLength'
-      - 'args.Error'
+      - "survey.MinLength"
+      - "survey.MaxLength"
+      - "args.Error"
   gosec:
     excludes:
       - G101
@@ -121,20 +121,20 @@ linters-settings:
       - "all"
       - "-SA1019"
   goimports:
-    local-prefixes: github.com/odpf/optimus
+    local-prefixes: github.com/raystack/optimus
   gci:
     sections:
       - standard # Captures all standard packages if they do not match another section.
       - default # Contains all imports that could not be matched to another section type.
-      - prefix(github.com/odpf/optimus) # Groups all imports with the specified Prefix.
+      - prefix(github.com/raystack/optimus) # Groups all imports with the specified Prefix.
   gocritic:
     disabled-checks:
       - ifElseChain
       - singleCaseSwitch
     enabled-tags:
       - diagnostic
-#      - style
-#      - opinionated
+      #      - style
+      #      - opinionated
       - performance
   unparam:
     # Inspect exported functions.
@@ -154,4 +154,4 @@ issues:
         - unparam
         - testpackage
 severity:
-  default-severity: error
+  default-severity: error
diff --git a/.goreleaser.latest.yml b/.goreleaser.latest.yml
@@ -10,7 +10,7 @@ builds:
     flags:
       - -a
     ldflags:
-      - -s -w -X github.com/odpf/optimus/config.BuildVersion=latest -X github.com/odpf/optimus/config.BuildCommit={{.FullCommit}} -X github.com/odpf/optimus/config.BuildDate={{.Date}}
+      - -s -w -X github.com/raystack/optimus/config.BuildVersion=latest -X github.com/raystack/optimus/config.BuildCommit={{.FullCommit}} -X github.com/raystack/optimus/config.BuildDate={{.Date}}
     goos:
       - linux
       - darwin
@@ -21,39 +21,38 @@ builds:
     env:
       - CGO_ENABLED=0
 archives:
-  - name_template: "{{ .ProjectName }}_latest_{{ .Os }}_{{ .Arch }}"
-    replacements:
-      darwin: macos
-      linux: linux
-      windows: windows
-      amd64: x86_64
+  - name_template: >-
+      {{ .ProjectName }}_latest_
+      {{- if eq .Os "darwin" }}macos
+      {{- else }}{{ .Os }}{{ end }}_
+      {{- if eq .Arch "amd64" }}x86_64
+      {{- else }}{{ .Arch }}{{ end }}
     format_overrides:
       - goos: windows
         format: zip
 release:
   disable: true
   name_template: "{{.ProjectName}}-latest"
 checksum:
-  name_template: 'checksums.txt'
+  name_template: "checksums.txt"
 snapshot:
   name_template: "latest"
 changelog:
   sort: asc
   filters:
     exclude:
-      - '^docs'
-      - '^test'
-      - '^build'
-      - '^chore'
+      - "^docs"
+      - "^test"
+      - "^build"
+      - "^chore"
 dockers:
-  -
-    goos: linux
+  - goos: linux
     goarch: amd64
     ids:
       - optimus
     dockerfile: Dockerfile
     image_templates:
-      - 'docker.io/odpf/{{.ProjectName}}:latest'
-      - 'docker.io/odpf/{{.ProjectName}}:latest-amd64'
+      - "docker.io/raystack/{{.ProjectName}}:latest"
+      - "docker.io/raystack/{{.ProjectName}}:latest-amd64"
     extra_files:
-      - entrypoint_init_container.sh
+      - entrypoint_init_container.sh
diff --git a/.goreleaser.yml b/.goreleaser.yml
@@ -10,7 +10,7 @@ builds:
     flags:
       - -a
     ldflags:
-      - -s -w -X github.com/odpf/optimus/config.BuildVersion={{ .Version }} -X github.com/odpf/optimus/config.BuildCommit={{.FullCommit}} -X github.com/odpf/optimus/config.BuildDate={{.Date}}
+      - -s -w -X github.com/raystack/optimus/config.BuildVersion={{ .Version }} -X github.com/raystack/optimus/config.BuildCommit={{.FullCommit}} -X github.com/raystack/optimus/config.BuildDate={{.Date}}
     goos:
       - linux
       - darwin
@@ -33,43 +33,42 @@ release:
   draft: true
   prerelease: auto
 checksum:
-  name_template: 'checksums.txt'
+  name_template: "checksums.txt"
 snapshot:
   name_template: "{{ .Tag }}-next"
 changelog:
   sort: asc
   filters:
     exclude:
-      - '^docs'
-      - '^test'
-      - '^build'
-      - '^chore'
+      - "^docs"
+      - "^test"
+      - "^build"
+      - "^chore"
 dockers:
-  -
-    goos: linux
+  - goos: linux
     goarch: amd64
     ids:
       - optimus
     dockerfile: Dockerfile
     image_templates:
-      - 'docker.io/odpf/{{.ProjectName}}:{{ .Version }}'
-      - 'docker.io/odpf/{{.ProjectName}}:{{ .Version }}-amd64'
+      - "docker.io/raystack/{{.ProjectName}}:{{ .Version }}"
+      - "docker.io/raystack/{{.ProjectName}}:{{ .Version }}-amd64"
     extra_files:
       - entrypoint_init_container.sh
 brews:
   - name: optimus
     tap:
-      owner: odpf
+      owner: raystack
       name: homebrew-tap
     license: "Apache 2.0"
     description: "Optimus helps your organization to build & manage data pipelines with ease."
-    homepage: https://odpf.github.io/optimus
+    homepage: https://raystack.github.io/optimus
     folder: Formula
     skip_upload: auto
     dependencies:
       - name: git
     install: |-
       bin.install "optimus"
     commit_author:
-      name: github-actions[bot]
-      email: 41898282+github-actions[bot]@users.noreply.github.com
+      name: Ravi Suhag
+      email: suhag.ravi@gmail.com
diff --git a/Makefile b/Makefile
@@ -1,11 +1,11 @@
 .ONESHELL:
 .DELETE_ON_ERROR:
 MAKEFLAGS += --no-builtin-rules
-NAME = "github.com/odpf/optimus"
+NAME = "github.com/raystack/optimus"
 LAST_COMMIT := $(shell git rev-parse --short HEAD)
 LAST_TAG := "$(shell git rev-list --tags --max-count=1)"
 OPMS_VERSION := "$(shell git describe --tags ${LAST_TAG})-next"
-PROTON_COMMIT := "31ac9046d1a8c95a2f4645b87bf0620a3e6bb8bc"
+PROTON_COMMIT := "1c39e65e529d573a1cd422e44f019c62d65fd10b"
 
 
 .PHONY: build test test-ci generate-proto unit-test-ci integration-test vet coverage clean install lint
@@ -30,9 +30,9 @@ scheduler-resource-test:
 	cd ./ext/scheduler/airflow2/tests && pip3 install -r requirements.txt && python3 -m unittest discover .
 
 generate-proto: ## regenerate protos
-	@echo " > generating protobuf from odpf/proton"
+	@echo " > generating protobuf from raystack/proton"
 	@echo " > [info] make sure correct version of dependencies are installed using 'make install'"
-	@buf generate https://github.com/odpf/proton/archive/${PROTON_COMMIT}.zip#strip_components=1 --template buf.gen.yaml --path odpf/optimus
+	@buf generate https://github.com/raystack/proton/archive/${PROTON_COMMIT}.zip#strip_components=1 --template buf.gen.yaml --path raystack/optimus
 	@echo " > protobuf compilation finished"
 
 unit-test-ci:
@@ -53,9 +53,6 @@ bench:
 coverage: ## print code coverage
 	go test -race -coverprofile coverage.txt -covermode=atomic ./... -tags=unit_test && go tool cover -html=coverage.txt
 
-clean:
-	rm -rf ./optimus ./dist ./api/proto/* ./api/third_party/odpf/*
-
 lint:
 	golangci-lint run --fix
 

diff --git a/README.md b/README.md
@@ -1,10 +1,10 @@
 # Optimus
 
-[![verify workflow](https://github.com/odpf/optimus/actions/workflows/verify.yml/badge.svg)](verification)
-[![publish latest workflow](https://github.com/odpf/optimus/actions/workflows/publish-latest.yml/badge.svg)](build)
-[![Coverage Status](https://coveralls.io/repos/github/odpf/optimus/badge.svg?branch=main)](https://coveralls.io/github/odpf/optimus?branch=main)
+[![verify workflow](https://github.com/raystack/optimus/actions/workflows/verify.yml/badge.svg)](verification)
+[![publish latest workflow](https://github.com/raystack/optimus/actions/workflows/publish-latest.yml/badge.svg)](build)
+[![Coverage Status](https://coveralls.io/repos/github/raystack/optimus/badge.svg?branch=main)](https://coveralls.io/github/raystack/optimus?branch=main)
 [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg?logo=apache)](LICENSE)
-[![Version](https://img.shields.io/github/v/release/odpf/optimus?logo=semantic-release)](Version)
+[![Version](https://img.shields.io/github/v/release/raystack/optimus?logo=semantic-release)](Version)
 
 Optimus is an easy-to-use, reliable, and performant workflow orchestrator for data transformation, data modeling, pipelines, and data quality management. It enables data analysts and engineers to transform their data by writing simple SQL queries and YAML configuration while Optimus handles dependency management, scheduling and all other aspects of running transformation jobs at scale.
 
@@ -29,7 +29,7 @@ Discover why users choose Optimus as their main data transformation tool.
 Optimus has two components, Optimus service that is the core orchestrator installed on server side, and a CLI binary used to interact with this service. You can install Optimus CLI using homebrew on macOS:
 
 ```shell
-$ brew install odpf/tap/optimus
+$ brew install raystack/tap/optimus
 $ optimus --help
 
 Optimus is an easy-to-use, reliable, and performant workflow orchestrator for
@@ -67,10 +67,10 @@ Use "optimus [command] --help" for more information about a command.
 
 Explore the following resources to get started with Optimus:
 
-- [Guides](https://odpf.github.io/optimus/docs/guides/create-job/) provides guidance on using Optimus.
-- [Concepts](https://odpf.github.io/optimus/docs/concepts/overview/) describes all important Optimus concepts.
-- [Reference](https://odpf.github.io/optimus/docs/reference/api/) contains details about configurations, metrics and other aspects of Optimus.
-- [Contribute](https://odpf.github.io/optimus/docs/contribute/contributing/) contains resources for anyone who wants to contribute to Optimus.
+- [Guides](https://raystack.github.io/optimus/docs/guides/create-job/) provides guidance on using Optimus.
+- [Concepts](https://raystack.github.io/optimus/docs/concepts/overview/) describes all important Optimus concepts.
+- [Reference](https://raystack.github.io/optimus/docs/reference/api/) contains details about configurations, metrics and other aspects of Optimus.
+- [Contribute](https://raystack.github.io/optimus/docs/contribute/contributing/) contains resources for anyone who wants to contribute to Optimus.
 
 ## Running locally
 
@@ -82,7 +82,7 @@ Optimus requires the following dependencies:
 Run the following commands to compile `optimus` from source
 
 ```shell
-$ git clone [email protected]:odpf/optimus.git
+$ git clone [email protected]:raystack/optimus.git
 $ cd optimus
 $ make
 ```
@@ -99,7 +99,7 @@ Optimus service can be started with
 $ ./optimus serve
 ```
 
-`serve` command has few required configurations that needs to be set for it to start. Read more about it in [getting started](https://odpf.github.io/optimus/docs/getting-started/configuration).
+`serve` command has few required configurations that needs to be set for it to start. Read more about it in [getting started](https://raystack.github.io/optimus/docs/getting-started/configuration).
 
 ## Compatibility
 
@@ -109,9 +109,9 @@ Optimus is currently undergoing heavy development with frequent, breaking API ch
 
 Development of Optimus happens in the open on GitHub, and we are grateful to the community for contributing bugfixes and improvements. Read below to learn how you can take part in improving Optimus.
 
-Read our [contributing guide](https://odpf.github.io/optimus/docs/contribute/contributing) to learn about our development process, how to propose bugfixes and improvements, and how to build and test your changes to Optimus.
+Read our [contributing guide](https://raystack.github.io/optimus/docs/contribute/contributing) to learn about our development process, how to propose bugfixes and improvements, and how to build and test your changes to Optimus.
 
-To help you get your feet wet and get you familiar with our contribution process, we have a list of [good first issues](https://github.com/odpf/optimus/labels/good%20first%20issue) that contain bugs which have a relatively limited scope. This is a great place to get started.
+To help you get your feet wet and get you familiar with our contribution process, we have a list of [good first issues](https://github.com/raystack/optimus/labels/good%20first%20issue) that contain bugs which have a relatively limited scope. This is a great place to get started.
 
 ## License
 

diff --git a/buf.gen.yaml b/buf.gen.yaml
@@ -7,12 +7,12 @@ plugins:
     # proto file should be.
     # This is necessary while importing a proto file foo/a.proto from another
     # directory, e.g. bar/b.proto
-    opt: paths=source_relative,Modpf/optimus/core/v1beta1/job_run.proto=github.com/odpf/optimus/protos/odpf/optimus/core/v1beta1,Modpf/optimus/core/v1beta1/project.proto=github.com/odpf/optimus/protos/odpf/optimus/core/v1beta1,Modpf/optimus/core/v1beta1/namespace.proto=github.com/odpf/optimus/protos/odpf/optimus/core/v1beta1,Modpf/optimus/core/v1beta1/job_spec.proto=github.com/odpf/optimus/protos/odpf/optimus/core/v1beta1,Modpf/optimus/core/v1beta1/scheduler.proto=github.com/odpf/optimus/protos/odpf/optimus/core/v1beta1,Modpf/optimus/core/v1beta1/status.proto=github.com/odpf/optimus/protos/odpf/optimus/core/v1beta1,Modpf/optimus/core/v1beta1/resource.proto=github.com/odpf/optimus/protos/odpf/optimus/core/v1beta1
+    opt: paths=source_relative,Mraystack/optimus/core/v1beta1/job_run.proto=github.com/raystack/optimus/protos/raystack/optimus/core/v1beta1,Mraystack/optimus/core/v1beta1/project.proto=github.com/raystack/optimus/protos/raystack/optimus/core/v1beta1,Mraystack/optimus/core/v1beta1/namespace.proto=github.com/raystack/optimus/protos/raystack/optimus/core/v1beta1,Mraystack/optimus/core/v1beta1/job_spec.proto=github.com/raystack/optimus/protos/raystack/optimus/core/v1beta1,Mraystack/optimus/core/v1beta1/scheduler.proto=github.com/raystack/optimus/protos/raystack/optimus/core/v1beta1,Mraystack/optimus/core/v1beta1/status.proto=github.com/raystack/optimus/protos/raystack/optimus/core/v1beta1,Mraystack/optimus/core/v1beta1/resource.proto=github.com/raystack/optimus/protos/raystack/optimus/core/v1beta1
   - name: go-grpc
     out: protos
     opt: paths=source_relative,require_unimplemented_servers=true
   - name: grpc-gateway
     out: protos
     opt: paths=source_relative
   - name: openapiv2
-    out: protos
+    out: protos