integrate embed into vectordb (#101)

* Update build.yml * Update build-embedding.yml * Update build-embedding.yml * Update build-embedding.yml * Update build-embedding.yml * Update build-embedding.yml * Update build-embedding.yml * Update build-embedding.yml * Create Dockerfile.embed * Update build-embedding.yml * Update build-embedding.yml * Update Dockerfile.embed * Update build.yml * Update VERSION * Update build.yml * Update build-embedding.yml * Update build.yml * Update build-embedding.yml * Update Dockerfile.embed * Update build-embedding.yml * Update build.yml * Update build-embedding.yml
epsilla-cloud · Dec 17, 2023 · f3a988f · f3a988f
1 parent 4cf36e3
commit f3a988f
Show file tree

Hide file tree

Showing 4 changed files with 70 additions and 15 deletions.
diff --git a/.github/workflows/build-embedding.yml b/.github/workflows/build-embedding.yml
@@ -3,6 +3,7 @@ name: Build Embedding Docker Image
 on:
   push:
     branches: [ "auto-embedding" ]
+    paths: ['VERSION']
 
 jobs:
   build:
@@ -19,7 +20,7 @@ jobs:
       with:
         username: ${{ secrets.DOCKER_USERNAME }}
         password: ${{ secrets.DOCKER_PASSWORD }}
-
+    
     - name: Get Current Date 
       id: date
       run: echo "::set-output name=date::$(date +'%Y%m%d%H%M%S')"
@@ -38,9 +39,14 @@ jobs:
         mv buildx-v0.11.2.linux-amd64 $HOME/.docker/cli-plugins/docker-buildx
         docker buildx create --use
 
+
+
         cd ./engine
-        
-        docker buildx build --platform linux/amd64 \
+
+        git clone https://eric-epsilla:${{ secrets.ACCESS_TOKEN }}@github.com/epsilla-cloud/epsilla-embed.git
+
+        docker buildx build -f Dockerfile.embed \
+                            --platform linux/amd64 \
                             --pull \
                             --push \
                             --label "org.opencontainers.image.version"=${BUILD_TIMESTAMP} \
@@ -49,8 +55,6 @@ jobs:
 
 
 
-
-
         
 
 

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -3,7 +3,7 @@ name: Build Vectordb Docker Image for linux/amd64 and linux/arm64
 on:
   push:
     branches: [ "main" ]
-    paths: ['VERSION']
+    paths: ['engine/Dockerfile.embed', 'VERSION']
 
 jobs:
   build:
@@ -39,9 +39,10 @@ jobs:
         export RELEASE_TAG=`cat VERSION`;
         echo "RELEASE_TAG: ${RELEASE_TAG}";
 
+
+
         cd engine;
         
-        
         # echo "========Build docker image for amd64 firstly========";
         # docker build --pull --build-arg="RELEASE_VERSION=${IMAGE_TAG}" \
         #              --label "org.opencontainers.image.description"="Official Epsilla Vector Database Image" \
@@ -54,7 +55,6 @@ jobs:
         #              -f Dockerfile -t ${DOCKER_HUB_REPOSITORY}:${IMAGE_TAG}-amd64 .;
         # docker push ${DOCKER_HUB_REPOSITORY}:${IMAGE_TAG}-amd64;
 
-
         echo "========Build allinone docker image both for amd64 and arm64========"
         wget https://github.com/docker/buildx/releases/download/v0.11.2/buildx-v0.11.2.linux-amd64
         chmod +x buildx-v0.11.2.linux-amd64
@@ -64,6 +64,7 @@ jobs:
         docker buildx ls
 
 
+        ## lite
         docker buildx build --build-arg="RELEASE_VERSION=v${RELEASE_TAG}-${IMAGE_TAG}" \
                             --platform linux/amd64,linux/arm64 \
                             --label "org.opencontainers.image.description"="Official Epsilla Vector Database Image" \
@@ -75,14 +76,25 @@ jobs:
                             --label "org.opencontainers.image.version"=${RELEASE_TAG} \
                             --pull \
                             --push \
-                            -t ${DOCKER_HUB_REPOSITORY}:${RELEASE_TAG} \
-                            -t ${DOCKER_HUB_REPOSITORY}:${IMAGE_TAG_LATEST} .
+                            -t ${DOCKER_HUB_REPOSITORY}:${RELEASE_TAG}-lite .
 
                             
-        
-
-        
-        
+        ## embed
+        git clone https://eric-epsilla:${{ secrets.ACCESS_TOKEN }}@github.com/epsilla-cloud/epsilla-embed.git
 
+        docker buildx build --build-arg="RELEASE_VERSION=v${RELEASE_TAG}-${IMAGE_TAG}" \
+                            --platform linux/amd64,linux/arm64 \
+                            --label "org.opencontainers.image.description"="Official Epsilla Vector Database Image" \
+                            --label "org.opencontainers.image.documentation"="https://epsilla-inc.gitbook.io/epsilladb" \
+                            --label "org.opencontainers.image.source"="https://github.com/epsilla-cloud/vectordb" \
+                            --label "org.opencontainers.image.title"="Epsilla Vector Database" \
+                            --label "org.opencontainers.image.url"="https://epsilla.com" \
+                            --label "org.opencontainers.image.vendor"="Epsilla" \
+                            --label "org.opencontainers.image.version"=${RELEASE_TAG} \
+                            --pull \
+                            --push \
+                            -f Dockerfile.embed \
+                            -t ${DOCKER_HUB_REPOSITORY}:${RELEASE_TAG} \
+                            -t ${DOCKER_HUB_REPOSITORY}:${IMAGE_TAG_LATEST} .
 
         
diff --git a/VERSION b/VERSION
@@ -1 +1 @@
-0.3.0-lite
+0.3.1
diff --git a/engine/Dockerfile.embed b/engine/Dockerfile.embed
@@ -0,0 +1,39 @@
+FROM epsilla/embedding:models AS models
+
+
+FROM epsilla/base AS builder
+ADD ./ /vectordb
+WORKDIR /vectordb
+ENV OATPP_INSTALL_PATH=/vectordb/build/dependencies
+RUN scripts/install_oatpp_modules.sh
+RUN mkdir -p /vectordb/build && cd /vectordb/build && cmake .. && make && chmod +x vectordb
+
+
+FROM epsilla/base
+WORKDIR /root
+
+ARG TARGETARCH
+ARG RELEASE_VERSION=latest
+ENV ENV_RELEASE_VERSION=$RELEASE_VERSION
+ENV PYTHONUNBUFFERED=True
+
+COPY --from=models /root/models /root/models
+COPY --from=builder /vectordb/build/vectordb /vectordb
+COPY ./scripts/heartbeat.sh /heartbeat.sh
+
+COPY ./epsilla-embed             /root/
+COPY ./epsilla-embed/launch.conf /etc/supervisor/conf.d/launch.conf
+
+RUN apt-get update && \
+    apt-get install -y procps supervisor && \
+    pip install --no-cache-dir boto3 poetry --break-system-packages && \
+    poetry update && \
+    poetry install --only main && \
+    rm -rf /var/lib/apt/lists/*
+
+
+EXPOSE 8888 8889
+
+HEALTHCHECK --interval=600s --timeout=30s --retries=1000 CMD bash /heartbeat.sh || exit 0
+
+ENTRYPOINT ["/usr/bin/supervisord", "-n", "-c", "/etc/supervisor/supervisord.conf"]