Merge pull request #1 from willejs/chore/initial-setup

Initial ports service

.devcontainer/devcontainer.json

@@ -0,0 +1,36 @@
+{
+    "name": "Go with Docker and Delve",
+    "image": "golang:1.22",
+    "features": {
+        "ghcr.io/devcontainers/features/docker-in-docker:2": {
+            "version": "latest",
+            "moby": true,
+            "dockerDashComposeVersion": "v2"
+        }
+    },
+    "customizations": {
+        "vscode": {
+            "settings": {
+                "go.toolsManagement.checkForUpdates": "local",
+                "go.useLanguageServer": true,
+                "go.gopath": "/go"
+            },
+            "extensions": [
+                "golang.Go",
+                "ms-azuretools.vscode-docker"
+            ]
+        }
+    },
+    "runArgs": [
+        "--init",
+        "--cap-add=SYS_PTRACE",
+        "--security-opt",
+        "seccomp=unconfined"
+    ],
+    "remoteUser": "root",
+    "postCreateCommand": "go install -v github.com/go-delve/delve/cmd/dlv@latest && go mod download && go version && docker --version && docker compose version",
+    "mounts": [
+        "source=${localWorkspaceFolder},target=/workspace,type=bind,consistency=cached"
+    ],
+    "workspaceFolder": "/workspace"
+}

.github/workflows/ci-cd.yaml

@@ -0,0 +1,93 @@
+name: CI/CD
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
+
+jobs:
+  lint:
+    name: Lint
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+
+      - name: Set up Go
+        uses: actions/setup-go@v4
+        with:
+          go-version: '1.22'
+
+      - name: golangci-lint
+        uses: golangci/golangci-lint-action@v6
+        with:
+          version: v1.60
+
+      # lint the dockerfile too
+      - uses: hadolint/[email protected]
+        with:
+          dockerfile: Dockerfile
+      # lint and validate the helm chart
+      - name: Cache Helm & Kubeconform
+        uses: actions/cache@v3
+        with:
+          path: |
+            /usr/local/bin/helm
+            /usr/local/bin/kubeconform
+          key: ${{ runner.os }}-helm-v3.12.0-kubeconform-v0.6.7
+          restore-keys: |
+            ${{ runner.os }}-helm-
+            ${{ runner.os }}-kubeconform-
+      - name: Install Helm & Kubeconform
+        if: steps.cache.outputs.cache-hit != 'true'
+        run: |
+          # i would build a generic docker image with tools like this in, and probably even a shared action in a
+          # .github private repo that i could use in all my projects
+          curl -LO https://get.helm.sh/helm-v3.12.0-linux-amd64.tar.gz
+          tar -xzf helm-v3.12.0-linux-amd64.tar.gz
+          mv linux-amd64/helm /usr/local/bin/
+          rm helm-v3.12.0-linux-amd64.tar.gz
+
+          curl -LO https://github.com/yannh/kubeconform/releases/download/v0.6.7/kubeconform-linux-amd64.tar.gz
+          tar -xzf kubeconform-linux-amd64.tar.gz
+          mv ./kubeconform /usr/local/bin/
+          rm kubeconform-linux-amd64.tar.gz
+
+      - name: Validate Helm Charts with Kubeconform
+        run: |
+          helm template helm/port-service | kubeconform -exit-on-error -strict -summary
+
+  test:
+    name: Test
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+
+      - name: Set up Go
+        uses: actions/setup-go@v4
+        with:
+          go-version: '1.22'
+
+      - name: Run tests
+        run: |
+          go test -v ./...
+
+  build:
+    name: Build
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+
+      - name: Set up Go
+        uses: actions/setup-go@v4
+        with:
+          go-version: '1.22'
+
+      - name: Build the project
+        run: |
+          go build -v ./...

.gitignore

@@ -7,6 +7,7 @@
 *.dll
 *.so
 *.dylib
+**main
 
 # Test binary, built with `go test -c`
 *.test
@@ -23,3 +24,7 @@ go.work.sum
 
 # env file
 .env
+
+**__debug**
+
+**.DS_Store

.vscode/launch.json

@@ -0,0 +1,15 @@
+{
+    // Use IntelliSense to learn about possible attributes.
+    // Hover to view descriptions of existing attributes.
+    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+    "version": "0.2.0",
+    "configurations": [
+        {
+            "name": "Launch Package",
+            "type": "go",
+            "request": "launch",
+            "mode": "auto",
+            "program": "${workspaceFolder}/cmd/api/main.go",
+        }
+    ]
+}

Dockerfile

@@ -0,0 +1,43 @@
+# Build stage
+FROM golang:1.22 AS builder
+
+# do people even care about the LFS hirarchy anymore?
+WORKDIR /app
+
+# Copy go mod and sum files in first so we can cache the dependencies
+COPY go.mod go.sum ./
+RUN go mod download
+
+# copy in the app and build it
+COPY ./cmd ./cmd
+COPY ./internal ./internal
+
+# statically compile the go binary for the presumed target of amd64 linux
+# whilst its larger, its more portable and will run in a scratch container
+RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o api-server ./cmd/api
+
+# hack to create the nobody user for the scratch container.
+# hadolint ignore=DL3059
+RUN echo "nobody:x:65534:65534:Nobody:/:" > /etc_passwd
+
+# Use multi stage builds. This is the final runtime stage. We can use ephemeral containers in kubernetes now :tada:
+# We could make a development target if this displeases people too.
+FROM scratch
+
+# expose the port and hardcode it for now
+EXPOSE 8080
+
+# copy in the nobody user in
+COPY --from=builder /etc_passwd /etc/passwd
+
+WORKDIR /app
+
+COPY data ./data
+COPY --from=builder /app/api-server /app/api-server
+
+# dont run the app as root, it is insecure
+USER nobody
+
+ENV PORT_FILE="/app/data/ports.json"
+
+CMD ["./api-server"]

Makefile

@@ -0,0 +1,25 @@
+.PHONY: test build build-docker
+test:
+	go test -v ./...
+
+build:
+	go build cmd/api/main.go
+
+run:
+	PORT_FILE=data/ports.json go run cmd/api/main.go
+
+run-jaeger:
+	docker run -d --name jaeger \
+		-e COLLECTOR_OTLP_ENABLED=true \
+		-e COLLECTOR_OTLP_HTTP_PORT=4318 \
+		-p 16686:16686 \
+		-p 4318:4318 \
+		-p 14250:14250 \
+		-p 14268:14268 \
+		jaegertracing/all-in-one:latest
+
+build-docker:
+	docker build --platform linux/amd64  -t willejs/port-service:latest .
+
+deploy:
+	helm install port-service helm/port-service

README.md

@@ -1,2 +1,66 @@
-# port-service
-Simple service that loads ports from a json file and serves them over http
+# ports service assignment
+
+This creates a simple go service that reads a json file containing port data and stores it in an in-memory database, and makes that available over HTTP. The service can be run in a docker container in Kubernetes. The service aims to emit telemetry in the form of logs, traces and metrics.
+
+### Scope
+
+I have decided to try to meet most of the criteria from the original assignment, but I have aimed to sway it a bit more towards SRE and DevOps practices, deviating from the original assignment specification and expanding the scope. I think that this might be a bit more representative of what I would see my self doing day to day, below are some things I have moved out of scope, and into scope.
+
+Hopefully this is ok, and if not, I am happy to do the assignment again with the original scope. I have tried to keep the service simple, and not over engineer it, and it definitely cuts some corners or does things in less than optimal or ideal ways, which i will hopefully explain in comments throughout.
+
+#### Out of scope
+- I am not following hexagonal architecture, i probably wont do it justice in the time i have.
+- The file is not streamed into memory, but read into memory in one go. This is because the file is small, and the service is simple. In future there are ways i would do this and i put comments in the code about it.
+- Tests are rather lacking. There isn't really any business logic or behavior in here, and its mostly implementation. I did not want to spend time creating extensive mocks, but if i were to do this properly I would spend alot more time on testing.
+- The directory layout and structure is not ideal.
+
+#### In scope
+- Simple /ready endpoint
+- Exposed a Otel metrics and prometheus metrics endpoint.
+- HTTP server metrics so that we can use them as high level SLIs (total requests and 99th percentile) to track high level SLOs.
+- Emits Otel Tracing to a jaeger instance, this could be tempo, datadog etc if we wish.
+- Otel Traces down the stack to the database layer.
+- Json logging, and a log level that can be set via an environment variable.
+- Multi stage docker build.
+- Helm chart to deploy the service.
+- CI/CD to build, test and publish the service to a docker registry.
+- Dummy deployment pipeline that pretends to deploy the service to a k8s cluster.
+
+#### Future work
+I am not really happy with the main application code, i think if i spent a bit longer and had some conversations i could clean it up, write more tests, adhere to ddd and make it a bit more idiomatic. I would probably move it to a hexagonal architecture. However, i wanted to showcase some more skills around telemetry, deployment, ci/cd etc, so i tried to timebox myself on this, therefore it feels a bit half finished.
+
+- I would look at creating an elegant logging abstraction that could be used throughout the service, but also other services.
+- Add more context to the traces and logs (like a request id), so that we can trace a request through the system easily.
+- Fully fledged gitops pipeline that would deploy the service to k8s clusters.
+- Plug this into a developer platform like backstage, and create a service catalog entry for it.
+- Create a shared package for otel metrics and tracing.
+- Use gorilla mux or similar to handle routing, this would make it easier to introduce handlers and middleware for things like tracing and logs.
+- Use a real database and auto instrument it with otel, using otel contrib libraries.
+- Decide upon some sort of acceptable testing strategy.
+
+
+### Testing, Building and Running the service
+
+Requirements:
+- Docker
+- Kubernetes (kind, k3s, docker desktop etc)
+- Helm
+
+You can use visual studio code dev containers to run the tests and the service, or you can run it locally.
+
+Steps to run locally:
+1. Clone the repository
+2. Run `make test` to run the tests
+3. Run `make run` to run the service
+4. The service will be available on `localhost:8080`
+   1. /ports, /metrics, /ready endpoints are available
+5. Run `make build` to build the docker image
+6. Run `make deploy` to deploy the docker image to your current k8s context
+7. Observe the helm message to test/explore the service
+
+### Bonus
+I have added a run-jaeger target to the makefile, this will run a jaeger instance in a docker container, and you can view the traces that are emitted from the service at http://localhost:16686/.
+
+### Debugging in k8s
+1. find the pods with `kubectl get pods`
+2. kubectl debug -it $pod-name-here --image=ubuntu --target=port-service -- /bin/bash