split (#43)

.github/workflows/e2e-split.yaml

@@ -0,0 +1,15 @@
+name: e2e-split
+on: push
+jobs:
+  test:
+    name: test
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+      - uses: actions/setup-go@v4
+        with:
+          go-version: '^1.21.1'
+      - name: Run test
+        run: ./e2e/common.sh --manifest_dir=./e2e/split/manifests  --run_test=./e2e/split/run_test.sh --kind_config=./e2e/split/kind-config.yaml

e2e/split/kind-config.yaml

@@ -0,0 +1,11 @@
+kind: Cluster
+apiVersion: kind.x-k8s.io/v1alpha4
+nodes:
+- role: control-plane
+  extraMounts:
+  - hostPath: ./tmp/objstore
+    containerPath: /tmp/objstore
+- role: worker
+  extraMounts:
+  - hostPath: ./tmp/objstore
+    containerPath: /tmp/objstore

e2e/split/manifests/agent.yaml

@@ -0,0 +1,68 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: fuyuu-router-agent
+  namespace: fuyuu-router
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: fuyuu-router-agent
+  template:
+    metadata:
+      labels:
+        app: fuyuu-router-agent
+    spec:
+      containers:
+      - name: fuyuu-router-agent
+        image: fuyuu-router:dev
+        volumeMounts:
+        - name: fuyuu-router-agent-config
+          mountPath: /app/config
+        - name: objstore
+          mountPath: /tmp/objstore
+        command: ["/bin/sh", "-c"]
+        args:
+        - |
+          /app/fuyuu-router agent -c /app/config/config.toml --id agent01 -b nanomq.nanomq:1883 --proxy-host 127.0.0.1:8000 --loglevel debug
+      - name: appserver
+        image: golang:1.21.3
+        command: ["/bin/sh", "-c"]
+        args:
+        - |
+          echo 'package main; import ("net/http"; "fmt"); func main() { http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { fmt.Fprintf(w, "ok") }); http.ListenAndServe(":8000", nil) }' > server.go && go run server.go
+        ports:
+        - containerPort: 8000
+        livenessProbe:
+          httpGet:
+            path: /
+            port: 8000
+          initialDelaySeconds: 15
+          periodSeconds: 5
+        readinessProbe:
+          httpGet:
+            path: /
+            port: 8000
+          initialDelaySeconds: 15
+          periodSeconds: 5
+      volumes:
+      - name: fuyuu-router-agent-config
+        configMap:
+          name: fuyuu-router-agent-config
+      - name: objstore
+        hostPath:
+          path: /tmp/objstore
+          type: Directory
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: fuyuu-router-agent-config
+  namespace: fuyuu-router
+data:
+  config.toml: |
+    [networking]
+    format = "json"
+    large_data_policy = "split"
+    [split]
+    chunk_bytes = 128000

e2e/split/manifests/hub.yaml

@@ -0,0 +1,64 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: fuyuu-router-hub
+  namespace: fuyuu-router
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: fuyuu-router-hub
+  template:
+    metadata:
+      labels:
+        app: fuyuu-router-hub
+    spec:
+      containers:
+      - name: fuyuu-router-hub
+        image: fuyuu-router:dev
+        volumeMounts:
+        - name: fuyuu-router-hub-config
+          mountPath: /app/config
+        - name: objstore
+          mountPath: /tmp/objstore
+        command: ["/bin/sh", "-c"]
+        args:
+        - |
+          /app/fuyuu-router hub -c /app/config/config.toml -b nanomq.nanomq:1883 --loglevel debug
+        ports:
+        - containerPort: 8080
+      volumes:
+      - name: fuyuu-router-hub-config
+        configMap:
+          name: fuyuu-router-hub-config
+      - name: objstore
+        hostPath:
+          path: /tmp/objstore
+          type: Directory
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: fuyuu-router-hub
+  namespace: fuyuu-router
+spec:
+  selector:
+    app: fuyuu-router-hub
+  ports:
+    - protocol: TCP
+      port: 8080
+      targetPort: 8080
+  type: NodePort
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: fuyuu-router-hub-config
+  namespace: fuyuu-router
+data:
+  config.toml: |
+    [networking]
+    format = "json"
+    large_data_policy = "split"
+    [split]
+    chunk_bytes = 128000

e2e/split/manifests/nanomq.yaml

@@ -0,0 +1,125 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: nanomq
+  namespace: nanomq
+  labels:
+    app: nanomq
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: nanomq
+  template:
+    metadata:
+      labels:
+        app: nanomq
+    spec:
+      containers:
+      - name: nanomq
+        image: emqx/nanomq:latest
+        ports:
+        - containerPort: 1883
+        - containerPort: 8083
+        - containerPort: 8883
+        volumeMounts:
+        - name: config-volume
+          mountPath: /etc/nanomq.conf
+          subPath: nanomq.conf
+      volumes:
+      - name: config-volume
+        configMap:
+          name: nanomq-config
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: nanomq-config
+  namespace: nanomq
+data:
+  nanomq.conf: |
+    # NanoMQ Configuration 0.18.0
+  
+    # #============================================================
+    # # NanoMQ Broker
+    # #============================================================
+  
+    mqtt {
+        property_size = 32
+        max_packet_size = 1MB
+        max_mqueue_len = 2048
+        retry_interval = 10s
+        keepalive_multiplier = 1.25
+  
+        # Three of below, unsupported now
+        max_inflight_window = 2048
+        max_awaiting_rel = 10s
+        await_rel_timeout = 10s
+    }
+  
+    listeners.tcp {
+        bind = "0.0.0.0:1883"
+    }
+  
+    # listeners.ssl {
+    # 	bind = "0.0.0.0:8883"
+    # 	keyfile = "/etc/certs/key.pem"
+    # 	certfile = "/etc/certs/cert.pem"
+    # 	cacertfile = "/etc/certs/cacert.pem"
+    # 	verify_peer = false
+    # 	fail_if_no_peer_cert = false
+    # }
+  
+    listeners.ws {
+        bind = "0.0.0.0:8083/mqtt"
+    }
+  
+    http_server {
+        port = 8081
+        limit_conn = 2
+        username = admin
+        password = public
+        auth_type = basic
+        jwt {
+            public.keyfile = "/etc/certs/jwt/jwtRS256.key.pub"
+        }
+    }
+  
+    log {
+        to = [file, console]
+        level = debug
+        dir = "/tmp"
+        file = "nanomq.log"
+        rotation {
+            size = 10MB
+            count = 5
+        }
+    }
+  
+    auth {
+        allow_anonymous = true
+        no_match = allow
+        deny_action = ignore
+  
+        cache = {
+            max_size = 32
+            ttl = 1m
+        }
+  
+        # password = {include "/etc/nanomq_pwd.conf"}
+        # acl = {include "/etc/nanomq_acl.conf"}
+    }
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: nanomq
+  namespace: nanomq
+spec:
+  selector:
+    app: nanomq
+  ports:
+    - protocol: TCP
+      port: 1883
+      targetPort: 1883
+  type: NodePort

e2e/split/manifests/tester.yaml

@@ -0,0 +1,22 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: curl-tester
+  namespace: fuyuu-router
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: curl-tester
+  template:
+    metadata:
+      labels:
+        app: curl-tester
+    spec:
+      containers:
+      - name: curl-tester
+        image: curlimages/curl:latest
+        command:
+        - /bin/sh
+        - -c
+        - sleep 3600

e2e/split/run_test.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+set -eux
+
+picture_file_local="e2e/storage_relay/picture.jpg"
+
+curl_tester_pod=$(kubectl get pods -n fuyuu-router -l app=curl-tester -o jsonpath='{.items[0].metadata.name}')
+
+kubectl cp "$picture_file_local" fuyuu-router/"${curl_tester_pod}":/tmp/picture.jpg
+
+response=$(kubectl exec -n fuyuu-router "${curl_tester_pod}" -- sh -c "
+    curl -H 'FuyuuRouter-IDs: agent01' -F 'file=@/tmp/picture.jpg' http://fuyuu-router-hub.fuyuu-router:8080 -s
+")
+
+if [[ "$response" == "ok" ]]; then
+    exit 0
+else
+    exit 1
+fi