Merge pull request #70 from epsilla-cloud/dev

build allinone docker image for both amd64 and arm64
epsilla-cloud · Oct 19, 2023 · e2587a9 · e2587a9
2 parents 66e0ddb + e1ca4fe
commit e2587a9
Show file tree

Hide file tree

Showing 4 changed files with 121 additions and 14 deletions.
diff --git a/.github/workflows/build-dev.yml b/.github/workflows/build-dev.yml
@@ -0,0 +1,64 @@
+name: Build Vectordb
+
+on:
+  push:
+    branches: [ "dev" ]
+    paths: ['engine/Dockerfile']
+
+jobs:
+  build:
+
+    name: Build Vectordb
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Check out code
+      uses: actions/checkout@v3
+
+    - name: Log in to Docker Hub
+      uses: docker/login-action@f4ef78c080cd8ba55a85445d5b36e214a81df20a
+      with:
+        username: ${{ secrets.DOCKER_USERNAME }}
+        password: ${{ secrets.DOCKER_PASSWORD }}
+
+    - name: Get Current Date 
+      id: date
+      run: echo "::set-output name=date::$(date +'%Y%m%d%H%M%S')"
+
+    - name: Build
+      env:
+        DOCKER_HUB_REPOSITORY: epsilla/vectordb
+        SHA_TAG: ${{ github.sha }}
+      run: |
+        export IMAGE_TAG=${{ steps.date.outputs.date }}
+        export IMAGE_TAG_LATEST="latest"
+        echo "IMAGE_TAG: ${IMAGE_TAG}"
+        echo "SHA: ${SHA_TAG}"
+
+        wget https://github.com/docker/buildx/releases/download/v0.11.2/buildx-v0.11.2.linux-amd64
+        chmod +x buildx-v0.11.2.linux-amd64
+        mkdir -p $HOME/.docker/cli-plugins
+        mv buildx-v0.11.2.linux-amd64 $HOME/.docker/cli-plugins/docker-buildx
+        docker buildx create --use
+
+        
+        cd ./engine
+        docker buildx build --build-arg="RELEASE_VERSION=${IMAGE_TAG}" \
+                            --platform linux/amd64,linux/arm64 \
+                            --label "org.opencontainers.image.description"="Official Epsilla Vector Database Image" \
+                            --label "org.opencontainers.image.documentation"="https://epsilla-inc.gitbook.io/epsilladb" \
+                            --label "org.opencontainers.image.source"="https://github.com/epsilla-cloud/vectordb" \
+                            --label "org.opencontainers.image.title"="Epsilla Vector Database" \
+                            --label "org.opencontainers.image.url"="https://www.epsilla.com" \
+                            --label "org.opencontainers.image.vendor"="Epsilla" \
+                            --label "org.opencontainers.image.version"=${IMAGE_TAG} \
+                            --push \
+                            -t epsilla/vectordb:dev .
+
+
+
+
+        
+
+
+        
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -1,4 +1,4 @@
-name: Build Vectordb
+name: Build Vectordb Docker Image for linux/amd64 and linux/arm64
 
 on:
   push:
@@ -7,7 +7,7 @@ on:
 jobs:
   build:
 
-    name: Build Vectordb
+    name: Build All-in-One Vectordb Docker Image
     runs-on: ubuntu-latest
 
     steps:
@@ -30,13 +30,43 @@ jobs:
         SHA_TAG: ${{ github.sha }}
       run: |
         export IMAGE_TAG=${{ steps.date.outputs.date }}
+        export IMAGE_TAG_LATEST="latest"
         echo "IMAGE_TAG: ${IMAGE_TAG}"
         echo "SHA: ${SHA_TAG}"
+        
         cd ./engine
-        docker build --pull -t ${DOCKER_HUB_REPOSITORY}:${IMAGE_TAG} .
-        docker push ${DOCKER_HUB_REPOSITORY}:${IMAGE_TAG}
-        docker tag  ${DOCKER_HUB_REPOSITORY}:${IMAGE_TAG} ${DOCKER_HUB_REPOSITORY}:latest
-        docker push ${DOCKER_HUB_REPOSITORY}:latest
+        echo "\n\nBuild docker image for amd64 firstly\n\n"
+        
+        docker build --pull -t ${DOCKER_HUB_REPOSITORY}:${IMAGE_TAG}-amd64 .
+        docker push ${DOCKER_HUB_REPOSITORY}:${IMAGE_TAG}-amd64
+
+
+        echo "\n\nBuild allinone docker image both for amd64 and arm64\n\n"
+        
+        wget https://github.com/docker/buildx/releases/download/v0.11.2/buildx-v0.11.2.linux-amd64
+        chmod +x buildx-v0.11.2.linux-amd64
+        mkdir -p $HOME/.docker/cli-plugins
+        mv buildx-v0.11.2.linux-amd64 $HOME/.docker/cli-plugins/docker-buildx
+        docker buildx create --use
+        docker buildx ls
+
+        docker buildx build --build-arg="RELEASE_VERSION=${IMAGE_TAG}" \
+                            --platform linux/amd64,linux/arm64 \
+                            --label "org.opencontainers.image.description"="Official Epsilla Vector Database Image" \
+                            --label "org.opencontainers.image.documentation"="https://epsilla-inc.gitbook.io/epsilladb" \
+                            --label "org.opencontainers.image.source"="https://github.com/epsilla-cloud/vectordb" \
+                            --label "org.opencontainers.image.title"="Epsilla Vector Database" \
+                            --label "org.opencontainers.image.url"="https://www.epsilla.com" \
+                            --label "org.opencontainers.image.vendor"="Epsilla" \
+                            --label "org.opencontainers.image.version"=${IMAGE_TAG} \
+                            --push \
+                            -t ${DOCKER_HUB_REPOSITORY}:${IMAGE_TAG} \
+                            -t ${DOCKER_HUB_REPOSITORY}:${IMAGE_TAG_LATEST} .
+                            
+
+        
+
+        
         
 
 

diff --git a/engine/Dockerfile b/engine/Dockerfile
@@ -7,18 +7,27 @@ ADD ./ /vectordb
 WORKDIR /vectordb
 ENV OATPP_INSTALL_PATH=/vectordb/build/dependencies
 RUN scripts/install_oatpp_modules.sh
-RUN scripts/setup-dev.sh
 RUN mkdir -p /vectordb/build && cd /vectordb/build && cmake .. && make -j $(nproc) && chmod +x vectordb
 
+
+
 FROM ubuntu:22.04
+ARG TARGETARCH
+ARG RELEASE_VERSION=latest
+ENV ENV_RELEASE_VERSION=$RELEASE_VERSION
+
 RUN apt-get update && \
     apt-get install -y --no-install-recommends curl wget net-tools iproute2 supervisor libgomp1 libboost-all-dev s3fs fuse ca-certificates && \
-    wget --no-check-certificate https://github.com/yandex-cloud/geesefs/releases/latest/download/geesefs-linux-amd64 && \
-    chmod +x geesefs-linux-amd64 && \
-    mv geesefs-linux-amd64 /usr/local/bin/geesefs && \
-    geesefs --version
+    echo "====== Install geesefs ======" && \
+    wget --no-check-certificate https://github.com/yandex-cloud/geesefs/releases/latest/download/geesefs-linux-${TARGETARCH} && \
+    chmod +x geesefs-linux-${TARGETARCH} && \
+    mv geesefs-linux-${TARGETARCH} /usr/local/bin/geesefs && \
+    geesefs --version && \
+    apt-get clean all && \
+    rm -rf /var/apt/lists/* /var/cache/apt/*
 COPY --from=builder /vectordb/build/vectordb /vectordb
 COPY ./scripts/heartbeat.sh /heartbeat.sh
 HEALTHCHECK --interval=600s --timeout=30s --retries=1000 CMD bash /heartbeat.sh || exit 0
 
 ENTRYPOINT ["/vectordb"]
+
diff --git a/engine/scripts/heartbeat.sh b/engine/scripts/heartbeat.sh
@@ -6,6 +6,9 @@ STARTUP_FILE=".startup_file"
 CONFIG_URL="https://config.epsilla.com/candidate.json"
 QUERY_URL="https://api.ipify.org"
 
+RELEASE_VERSION=${ENV_RELEASE_VERSION:-latest} 
+
+
 SENTRY_DSN=`curl -s $CONFIG_URL | grep heartbeat | awk -F '"' '{print $(NF-1)}'`
 SENTRY_HOST=`echo $SENTRY_DSN | sed -e "s/[^/]*\/\/\([^@]*@\)\?\([^:/]*\).*/\2/"`
 SENTRY_SECRET=`echo $SENTRY_DSN | cut -d '/' -f3 | cut -d '@' -f1`
@@ -38,7 +41,7 @@ if [ ! -f "${STARTUP_FILE}" ]; then
     \"logger\": \"docker\",
     \"server_name\": \"${HOSTNAME}\",
     \"tags\": {
-      \"version\": \"latest\",
+      \"version\": \"${RELEASE_VERSION}\",
       \"internal_ip\": \"${INTERNAL_IP}\",
       \"external_ip\": \"${EXTERNAL_IP}\",
       \"timestamp\": \"${TIMESTAMP}\",
@@ -55,7 +58,7 @@ if [ ! -f "${STARTUP_FILE}" ]; then
     \"api_key\": \"${POSTHOG_API_KEY}\",
     \"distinct_id\": \"${DISTINCT_ID}\",
     \"properties\": {
-      \"version\": \"latest\",
+      \"version\": \"${RELEASE_VERSION}\",
       \"internal_ip\": \"${INTERNAL_IP}\",
       \"external_ip\": \"${EXTERNAL_IP}\"
     }
@@ -76,6 +79,7 @@ curl -X POST \
   \"level\": \"info\",
   \"server_name\": \"${HOSTNAME}\",
   \"tags\": {
+    \"version\": \"${RELEASE_VERSION}\",
     \"internal_ip\": \"${INTERNAL_IP}\",
     \"external_ip\": \"${EXTERNAL_IP}\",
     \"heart_beat\": \"${DATE_TAG}\",
@@ -96,7 +100,7 @@ curl -X POST -L --header "Content-Type: application/json" \
     \"api_key\": \"${POSTHOG_API_KEY}\",
     \"distinct_id\": \"${DISTINCT_ID}\",
     \"properties\": {
-      \"version\": \"latest\",
+      \"version\": \"${RELEASE_VERSION}\",
       \"internal_ip\": \"${INTERNAL_IP}\",
       \"external_ip\": \"${EXTERNAL_IP}\"
     }