merge up master

brave-intl · Jul 12, 2023 · 0e0ef0d · 0e0ef0d
2 parents 6fc3482 + cd02710
commit 0e0ef0d
Show file tree

Hide file tree

Showing 87 changed files with 3,430 additions and 935 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -44,28 +44,48 @@ jobs:
       matrix:
         goversion:
         - 1.18
+
     steps:
-    - name: Check out code into the Go module directory
-      uses: actions/checkout@5a4ac9002d0be2fb38bd78e4b4dbde5606d7042f
+    - name: Checkout repository
+      uses: actions/checkout@v3
 
-    - name: Set up Go 1.x
-      uses: actions/setup-go@37335c7bb261b353407cff977110895fa0b4f7d8
+    - name: Set up Go
+      id: setup-go
+      uses: actions/setup-go@v4
       with:
-        go-version: ${{matrix.goversion}}
+        go-version: ${{ matrix.goversion }}
+        cache-dependency-path: "**/go.sum"
+
+    - name: Ensure Module Path
+      run: mkdir -p /opt/go/pkg/mod
+
+    - name: Copy From Module Cache
+      if: steps.setup-go.outputs.cache-hit == 'true'
+      run: |
+        rsync -au "/home/runner/go/pkg/" "/opt/go/pkg"
 
-    - name: Docker Compose Install
+    - name: Install Docker Compose
       uses: KengoTODA/actions-setup-docker-compose@92cbaf8ac8c113c35e1cedd1182f217043fbdd00
       with:
         version: '1.25.4'
 
     - run: docker-compose pull
 
-    - name: Vault
+    - name: Start Vault
       run: |
         docker-compose -f docker-compose.yml -f docker-compose.dev.yml up -d vault;
-        sleep 3;
+        sleep 3
 
-    - name: Test
+    - name: Run Tests
       run: |
         export VAULT_TOKEN=$(docker logs grant-vault 2>&1 | grep "Root Token" | tail -1 | cut -d ' ' -f 3 );
-        docker-compose -f docker-compose.yml -f docker-compose.dev.yml run --rm dev make;
+        docker-compose -f docker-compose.yml -f docker-compose.dev.yml run --rm -v /opt/go/pkg:/go/pkg dev make
+
+    - name: Ensure Module Directory
+      if: steps.setup-go.outputs.cache-hit != 'true'
+      run: mkdir -p /home/runner/go/pkg
+
+    - name: Copy To Module Cache
+      run: |
+        sudo rsync -au "/opt/go/pkg/" "/home/runner/go/pkg"
+        sudo chown -R runner:runner /home/runner/go/pkg
diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml
@@ -35,11 +35,11 @@ jobs:
 
     steps:
     - name: Checkout repository
-      uses: actions/checkout@v2
+      uses: actions/checkout@v3
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
-      uses: github/codeql-action/init@v1
+      uses: github/codeql-action/init@v2
       with:
         languages: ${{ matrix.language }}
         # If you wish to specify custom queries, you can do so here or in a config file.
@@ -49,8 +49,12 @@ jobs:
 
     # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
     # If this step fails, then you should remove it and run the build manually (see below)
-    - name: Autobuild
-      uses: github/codeql-action/autobuild@v1
+    # - name: Autobuild
+    #   uses: github/codeql-action/autobuild@v2
+
+    - name: Download Modules and Build
+      run: |
+        make codeql
 
     # ℹ️ Command-line programs to run using the OS shell.
     # 📚 https://git.io/JvXDl
@@ -59,9 +63,5 @@ jobs:
     #    and modify them (or add more) to build your code if your project
     #    uses a compiled language
 
-    #- run: |
-    #   make bootstrap
-    #   make release
-
     - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v1
+      uses: github/codeql-action/analyze@v2
diff --git a/.github/workflows/generalized-deployments.yaml b/.github/workflows/generalized-deployments.yaml
@@ -14,7 +14,7 @@ jobs:
     - name: Checkout
       uses: actions/checkout@v2
     - name: Configure AWS Credentials
-      uses: aws-actions/configure-aws-credentials@v1
+      uses: aws-actions/configure-aws-credentials@v2
       with:
         aws-access-key-id: ${{ secrets.GDBP_AWS_ACCESS_KEY_ID }}
         aws-secret-access-key: ${{ secrets.GDBP_AWS_SECRET_ACCESS_KEY }}

diff --git a/Dockerfile b/Dockerfile
@@ -1,33 +1,28 @@
 FROM golang:1.18-alpine as builder
 
-
-# put certs in builder image
+# Put certs in builder image.
 RUN apk update
 RUN apk add -U --no-cache ca-certificates && update-ca-certificates
-RUN apk add make
-RUN apk add build-base
-RUN apk add git
-RUN apk add bash
+RUN apk add make build-base git bash
 
 ARG VERSION
 ARG BUILD_TIME
 ARG COMMIT
 
 WORKDIR /src
 COPY . ./
-RUN chown -R nobody:nobody /src/
-RUN mkdir /.cache
-RUN chown -R nobody:nobody /.cache
+
+RUN chown -R nobody:nobody /src/ && mkdir /.cache && chown -R nobody:nobody /.cache
 
 USER nobody
-RUN cd main && go mod download
 
-RUN cd main && CGO_ENABLED=0 GOOS=linux go build \
+RUN cd main && go mod download && CGO_ENABLED=0 GOOS=linux go build \
     -ldflags "-w -s -X main.version=${VERSION} -X main.buildTime=${BUILD_TIME} -X main.commit=${COMMIT}" \
     -o bat-go main.go
 
 FROM alpine:3.15 as base
-# put certs in artifact from builder
+
+# Put certs in artifact from builder.
 COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
 COPY --from=builder /src/main/bat-go /bin/
 
@@ -38,4 +33,3 @@ FROM base as artifact
 COPY --from=builder /src/migrations/ /migrations/
 EXPOSE 3333
 CMD ["bat-go", "serve", "grant", "--enable-job-workers", "true"]
-
diff --git a/Makefile b/Makefile
@@ -16,13 +16,15 @@ ifdef TEST_RUN
 	TEST_FLAGS = --tags=$(TEST_TAGS) $(TEST_PKG) --run=$(TEST_RUN)
 endif
 
-.PHONY: all buildcmd docker test create-json-schema lint clean
+.PHONY: all buildcmd docker test create-json-schema lint clean download-mod
 all: test create-json-schema buildcmd
 
 .DEFAULT: buildcmd
 
+codeql: download-mod buildcmd
+
 buildcmd:
-	cd main && CGO_ENABLED=0 GOOS=$(GOOS) GOARCH=$(GOARCH) go build -v -ldflags "-w -s -X main.version=${GIT_VERSION} -X main.buildTime=${BUILD_TIME} -X main.commit=${GIT_COMMIT}" -o ${OUTPUT}/bat-go main.go
+	cd main && CGO_ENABLED=0 GOOS=$(GOOS) GOARCH=$(GOARCH) go build -ldflags "-w -s -X main.version=${GIT_VERSION} -X main.buildTime=${BUILD_TIME} -X main.commit=${GIT_COMMIT}" -o ${OUTPUT}/bat-go main.go
 
 mock:
 	cd services && mockgen -source=./promotion/claim.go -destination=promotion/mockclaim.go -package=promotion
@@ -40,6 +42,7 @@ mock:
 	cd libs && mockgen -source=./clients/radom/client.go -destination=clients/radom/mock/mock.go -package=mock_radom
 	cd libs && mockgen -source=./clients/bitflyer/client.go -destination=clients/bitflyer/mock/mock.go -package=mock_bitflyer
 	cd libs && mockgen -source=./clients/coingecko/client.go -destination=clients/coingecko/mock/mock.go -package=mock_coingecko
+	cd libs && mockgen -source=./clients/stripe/client.go -destination=clients/stripe/mock/mock.go -package=mock_stripe
 	cd libs && mockgen -source=./backoff/retrypolicy/retrypolicy.go -destination=backoff/retrypolicy/mock/retrypolicy.go -package=mockretrypolicy
 	cd libs && mockgen -source=./aws/s3.go -destination=aws/mock/mock.go -package=mockaws
 	cd libs && mockgen -source=./kafka/dialer.go -destination=kafka/mock/dialer.go -package=mockdialer
@@ -75,13 +78,16 @@ instrumented:
 	sed -i'bak' 's/bitflyer.//g' libs/clients/bitflyer/instrumented_client.go
 	cd libs && gowrap gen -p github.com/brave-intl/bat-go/libs/clients/coingecko -i Client -t ../.prom-gowrap.tmpl -o ./clients/coingecko/instrumented_client.go
 	sed -i'bak' 's/coingecko.//g' libs/clients/coingecko/instrumented_client.go
+	cd libs && gowrap gen -p github.com/brave-intl/bat-go/libs/clients/stripe -i Client -t ../.prom-gowrap.tmpl -o ./clients/stripe/instrumented_client.go
+	sed -i'bak' 's/stripe.//g' libs/clients/stripe/instrumented_client.go
 	# fix all instrumented cause the interfaces are all called "client"
 	sed -i'bak' 's/client_duration_seconds/cbr_client_duration_seconds/g' libs/clients/cbr/instrumented_client.go
 	sed -i'bak' 's/client_duration_seconds/ratios_client_duration_seconds/g' libs/clients/ratios/instrumented_client.go
 	sed -i'bak' 's/client_duration_seconds/reputation_client_duration_seconds/g' libs/clients/reputation/instrumented_client.go
 	sed -i'bak' 's/client_duration_seconds/gemini_client_duration_seconds/g' libs/clients/gemini/instrumented_client.go
 	sed -i'bak' 's/client_duration_seconds/bitflyer_client_duration_seconds/g' libs/clients/bitflyer/instrumented_client.go
 	sed -i'bak' 's/client_duration_seconds/coingecko_client_duration_seconds/g' libs/clients/coingecko/instrumented_client.go
+	sed -i'bak' 's/client_duration_seconds/stripe_client_duration_seconds/g' libs/clients/stripe/instrumented_client.go
 
 %-docker: docker
 	docker build --build-arg COMMIT=$(GIT_COMMIT) --build-arg VERSION=$(GIT_VERSION) \
@@ -185,11 +191,32 @@ format:
 
 format-lint:
 	make format && make lint
-lint:
-	docker run --rm -v "$$(pwd):/app" --workdir /app/libs golangci/golangci-lint:v1.49.0 golangci-lint run -v ./...
-	docker run --rm -v "$$(pwd):/app" --workdir /app/services golangci/golangci-lint:v1.49.0 golangci-lint run -v ./...
-	docker run --rm -v "$$(pwd):/app" --workdir /app/tools golangci/golangci-lint:v1.49.0 golangci-lint run -v ./...
-	docker run --rm -v "$$(pwd):/app" --workdir /app/cmd golangci/golangci-lint:v1.49.0 golangci-lint run -v ./...
-	docker run --rm -v "$$(pwd):/app" --workdir /app/main golangci/golangci-lint:v1.49.0 golangci-lint run -v ./...
-	docker run --rm -v "$$(pwd):/app" --workdir /app/serverless/email/webhook golangci/golangci-lint:v1.49.0 golangci-lint run -v ./...
-	docker run --rm -v "$$(pwd):/app" --workdir /app/serverless/email/unsubscribe golangci/golangci-lint:v1.49.0 golangci-lint run -v ./...
+
+lint: ensure-gomod-volume
+	docker run --rm -v "$$(pwd):/app" -v batgo_lint_gomod:/go/pkg --workdir /app/main golangci/golangci-lint:v1.49.0 golangci-lint run -v ./...
+	docker run --rm -v "$$(pwd):/app" -v batgo_lint_gomod:/go/pkg --workdir /app/cmd golangci/golangci-lint:v1.49.0 golangci-lint run -v ./...
+	docker run --rm -v "$$(pwd):/app" -v batgo_lint_gomod:/go/pkg --workdir /app/libs golangci/golangci-lint:v1.49.0 golangci-lint run -v ./...
+	docker run --rm -v "$$(pwd):/app" -v batgo_lint_gomod:/go/pkg --workdir /app/services golangci/golangci-lint:v1.49.0 golangci-lint run -v ./...
+	docker run --rm -v "$$(pwd):/app" -v batgo_lint_gomod:/go/pkg --workdir /app/tools golangci/golangci-lint:v1.49.0 golangci-lint run -v ./...
+	docker run --rm -v "$$(pwd):/app" -v batgo_lint_gomod:/go/pkg --workdir /app/serverless/email/webhook golangci/golangci-lint:v1.49.0 golangci-lint run -v ./...
+	docker run --rm -v "$$(pwd):/app" -v batgo_lint_gomod:/go/pkg --workdir /app/serverless/email/unsubscribe golangci/golangci-lint:v1.49.0 golangci-lint run -v ./...
+
+download-mod:
+	cd ./cmd && go mod download && cd ..
+	cd ./libs && go mod download && cd ..
+	cd ./main && go mod download && cd ..
+	cd ./services && go mod download && cd ..
+	cd ./tools && go mod download && cd ..
+	cd ./serverless/email/status && go mod download && cd ../../..
+	cd ./serverless/email/unsubscribe && go mod download && cd ../../..
+	cd ./serverless/email/webhook && go mod download && cd ../../..
+
+docker-up-ext: ensure-shared-net
+	$(eval VAULT_TOKEN = $(shell docker logs grant-vault 2>&1 | grep "Root Token" | tail -1 | cut -d ' ' -f 3 ))
+	VAULT_TOKEN=$(VAULT_TOKEN) docker-compose -f docker-compose.yml -f docker-compose.dev.yml -f docker-compose.ext.yml run --rm -p 3333:3333 dev /bin/bash
+
+ensure-gomod-volume:
+	docker volume create batgo_lint_gomod
+
+ensure-shared-net:
+	if [ -z $$(docker network ls -q -f "name=brave_shared_net") ]; then docker network create brave_shared_net; fi
diff --git a/create_dbs.sh b/create_dbs.sh
@@ -0,0 +1,37 @@
+#!/usr/bin/env bash
+
+set -eu
+
+function create_database_and_user() {
+    local database=$1
+    local user=$2
+    local password=$3
+
+    echo "Creating database with user: $database $user"
+    psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" <<-EOSQL
+CREATE USER $user WITH PASSWORD '$password';
+CREATE DATABASE $database;
+GRANT ALL PRIVILEGES ON DATABASE $database TO $user;
+EOSQL
+}
+
+if [ -n $POSTGRES_EXTRA_DATABASES ]; then
+    echo "Creating multiple databases and users: $POSTGRES_EXTRA_DATABASES"
+    for dup in $(echo $POSTGRES_EXTRA_DATABASES | tr ',' ' '); do
+        db=$(echo $dup | awk -F":" '{print $1}')
+        user=$(echo $dup | awk -F":" '{print $2}')
+        password=$(echo $dup | awk -F":" '{print $3}')
+
+        if [ -z "$user"]; then
+            user=$db
+        fi
+
+        if [ -z "$password" ]; then
+            password=$user
+        fi
+
+        create_database_and_user $db $user $password
+    done
+
+    echo "Created multiple databases"
+fi
diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
@@ -15,7 +15,7 @@ services:
       - no-new-privileges:true
     environment:
       - OUTPUT_DIR="/out"
-      - "CHALLENGE_BYPASS_DATABASE_URL=postgres://btokens:password@challenge-bypass-postgres/btokens?sslmode=disable"
+      - "CHALLENGE_BYPASS_DATABASE_URL=postgres://btokens:password@grant-postgres/btokens?sslmode=disable"
       - "VAULT_ADDR=http://vault:8200"
       - TEST_TAGS
       - VAULT_TOKEN

diff --git a/docker-compose.ext.yml b/docker-compose.ext.yml
@@ -0,0 +1,17 @@
+version: "3.4"
+
+networks:
+  brave_shared:
+    name: brave_shared_net
+    external: true
+
+services:
+  dev:
+    networks:
+      - grant
+      - brave_shared
+
+  web:
+    networks:
+      - grant
+      - brave_shared