Fix arm build of oceanbase-ce (#39)

oceanbase · Aug 19, 2024 · 52246b7 · 52246b7
1 parent 47e4813
commit 52246b7
Show file tree

Hide file tree

Showing 10 changed files with 180 additions and 54 deletions.
diff --git a/.github/workflows/build-oceanbase-ce.yml b/.github/workflows/build-oceanbase-ce.yml
@@ -17,6 +17,16 @@ jobs:
   build-oceanbase-ce:
     runs-on: ubuntu-latest
     steps:
+      - name: Free disk space on Ubuntu runner
+        uses: kfir4444/free-disk-space@main
+        with:
+          tool-cache: false
+          android: true
+          dotnet: true
+          haskell: true
+          large-packages: true
+          swap-storage: true
+
       - name: Print environment variables
         run: printenv
 
@@ -29,6 +39,23 @@ jobs:
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
 
+      - name: Build pre image
+        uses: docker/build-push-action@v6
+        with:
+          context: ./oceanbase-ce
+          platforms: linux/amd64
+          file: ./oceanbase-ce/Dockerfile.pre
+          push: false
+          load: true
+          tags: oceanbase-ce-pre
+          build-args: |
+            VERSION=${{ inputs.version }}
+
+      - name: Copy storage files
+        run: |
+          mkdir -p ./oceanbase-ce/share
+          docker run -v ./oceanbase-ce/share:/root/share oceanbase-ce-pre
+
       - name: Build observer image
         uses: docker/build-push-action@v6
         with:

diff --git a/.github/workflows/java-test-oceanbase-ce.yml b/.github/workflows/java-test-oceanbase-ce.yml
@@ -40,6 +40,16 @@ jobs:
   test-oceanbase-ce:
     runs-on: ubuntu-latest
     steps:
+      - name: Free disk space on Ubuntu runner
+        uses: kfir4444/free-disk-space@main
+        with:
+          tool-cache: false
+          android: true
+          dotnet: true
+          haskell: true
+          large-packages: true
+          swap-storage: true
+
       - name: Download artifact
         uses: actions/download-artifact@v4
         with:

diff --git a/.github/workflows/release-lts-oceanbase-ce.yml b/.github/workflows/release-lts-oceanbase-ce.yml
@@ -12,6 +12,33 @@ jobs:
   release-lts-oceanbase-ce:
     runs-on: ubuntu-latest
     steps:
+      - name: Free disk space on Ubuntu runner
+        uses: kfir4444/free-disk-space@main
+        with:
+          tool-cache: false
+          android: true
+          dotnet: true
+          haskell: true
+          large-packages: true
+          swap-storage: true
+
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Set Version variables
+        id: set_version_vars
+        run: echo "version=$(echo $tagName | grep -P '(\d*\.\d*\.\d*\.\d*-\d{18})' --only-matching)" >> $GITHUB_OUTPUT
+
+      - name: Set LTS tag variables
+        id: set_lts_tag_vars
+        run: echo "lts_tag=$(echo $tagName | grep -P '(\d*\.\d*\.\d*)' --only-matching)" >> $GITHUB_OUTPUT
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
       - name: Log in to Docker hub
         uses: docker/login-action@v3
         with:
@@ -32,22 +59,22 @@ jobs:
           username: ${{ github.repository_owner }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
-      - name: Checkout repository
-        uses: actions/checkout@v4
-
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v3
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
-
-      - name: Set Version variables
-        id: set_version_vars
-        run: echo "version=$(echo $tagName | grep -P '(\d*\.\d*\.\d*\.\d*-\d{18})' --only-matching)" >> $GITHUB_OUTPUT
+      - name: Build pre image
+        uses: docker/build-push-action@v6
+        with:
+          context: ./oceanbase-ce
+          platforms: linux/amd64
+          file: ./oceanbase-ce/Dockerfile.pre
+          push: false
+          load: true
+          tags: oceanbase-ce-pre
+          build-args: |
+            VERSION=${{ steps.set_version_vars.outputs.version }}
 
-      - name: Set LTS tag variables
-        id: set_lts_tag_vars
-        run: echo "lts_tag=$(echo $tagName | grep -P '(\d*\.\d*\.\d*)' --only-matching)" >> $GITHUB_OUTPUT
+      - name: Copy storage files
+        run: |
+          mkdir -p ./oceanbase-ce/share
+          docker run -v ./oceanbase-ce/share:/root/share oceanbase-ce-pre
 
       - name: Build and push observer w/o cache
         uses: docker/build-push-action@v6

diff --git a/.github/workflows/release-oceanbase-ce.yml b/.github/workflows/release-oceanbase-ce.yml
@@ -12,6 +12,29 @@ jobs:
   release-oceanbase-ce:
     runs-on: ubuntu-latest
     steps:
+      - name: Free disk space on Ubuntu runner
+        uses: kfir4444/free-disk-space@main
+        with:
+          tool-cache: false
+          android: true
+          dotnet: true
+          haskell: true
+          large-packages: true
+          swap-storage: true
+
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Set Version variables
+        id: set_version_vars
+        run: echo "version=$(echo $tagName | grep -P '(\d*\.\d*\.\d*\.\d*-\d{18})' --only-matching)" >> $GITHUB_OUTPUT
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
       - name: Log in to Docker hub
         uses: docker/login-action@v3
         with:
@@ -32,18 +55,22 @@ jobs:
           username: ${{ github.repository_owner }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
-      - name: Checkout repository
-        uses: actions/checkout@v4
-
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v3
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
+      - name: Build pre image
+        uses: docker/build-push-action@v6
+        with:
+          context: ./oceanbase-ce
+          platforms: linux/amd64
+          file: ./oceanbase-ce/Dockerfile.pre
+          push: false
+          load: true
+          tags: oceanbase-ce-pre
+          build-args: |
+            VERSION=${{ steps.set_version_vars.outputs.version }}
 
-      - name: Set Version variables
-        id: set_version_vars
-        run: echo "version=$(echo $tagName | grep -P '(\d*\.\d*\.\d*\.\d*-\d{18})' --only-matching)" >> $GITHUB_OUTPUT
+      - name: Copy storage files
+        run: |
+          mkdir -p ./oceanbase-ce/share
+          docker run -v ./oceanbase-ce/share:/root/share oceanbase-ce-pre
 
       - name: Build and push observer w/o cache
         uses: docker/build-push-action@v6

diff --git a/.github/workflows/test-oceanbase-ce.yml b/.github/workflows/test-oceanbase-ce.yml
@@ -19,7 +19,7 @@ jobs:
     with:
       cache_key: test-oceanbase-ce
       image_file: oceanbase-ce.tar
-      version: 4.2.3.1-101000032024061316
+      version: 4.3.2.0-100000092024072912
 
   test-slim:
     needs: build

diff --git a/oceanbase-ce/Dockerfile b/oceanbase-ce/Dockerfile
@@ -4,35 +4,30 @@ ARG VERSION
 
 WORKDIR /root
 
-COPY build /root/build
 RUN mkdir -p pkgs
 
+ADD build/demo.yaml /root/
+
 RUN yum install -y yum-utils && \
     yum-config-manager --add-repo http://mirrors.oceanbase.com/oceanbase/OceanBase.repo && \
-    sed -i 's/$releasever/7/' /etc/yum.repos.d/OceanBase.repo && \
     yum install -y ob-deploy && \
     yum clean all
 
 # download and clone all the required packages
-RUN yum install -y --downloadonly --downloaddir=/root/pkgs oceanbase-ce-${VERSION}.el7 oceanbase-ce-libs-${VERSION}.el7 ob-configserver
+RUN yum install -y --downloadonly --downloaddir=/root/pkgs oceanbase-ce-${VERSION}.el8 oceanbase-ce-libs-${VERSION}.el8
 RUN obd mirror clone /root/pkgs/*.rpm
 RUN obd env set IO_DEFAULT_CONFIRM 1
 RUN obd mirror disable remote
-RUN obd cluster deploy demo -c build/deploy.yaml
-RUN obd cluster destroy -f demo
+RUN obd cluster deploy demo -c /root/demo.yaml
 
-RUN obd demo -c oceanbase-ce --oceanbase-ce.home_path=/root/demo --oceanbase-ce.appname=obcluster --oceanbase-ce.scenario=express_oltp --oceanbase-ce.datafile_size=256M --oceanbase-ce.log_disk_size=5G && obd cluster tenant create demo -n test -o express_oltp 
-RUN obd cluster stop demo
-RUN cd /root/demo && tar -cvzf store.tar.gz store
-RUN rm -rf /root/demo/store && rm -rf /root/demo/log/* && rm -rf /root/demo/log_obshell/* && rm -rf /root/demo/etc/*.py && rm -rf /root/demo/etc/obshell && rm -rf /root/demo/etc/*.sql && rm -rf /root/demo/etc/*.log && rm -rf /root/demo/run/*
-RUN cd /root/.obd && rm -rf log/obd && tar -cvzf repository.tar.gz repository && rm -rf /root/.obd/repository && rm -rf /root/.obd/mirror/local/*
+RUN rm -rf /root/demo/store && rm -rf /root/demo/etc
+RUN cd /root/.obd && rm -rf log/obd && tar -cvzf repository.tar.gz repository && rm -rf /root/.obd/repository && rm -rf /root/.obd/mirror/local/* && rm -rf /root/.obd/cluster
 
 FROM openanolis/anolisos
 
 WORKDIR /root
 RUN yum install -y yum-utils && \
     yum-config-manager --add-repo http://mirrors.oceanbase.com/oceanbase/OceanBase.repo && \
-    sed -i 's/$releasever/7/' /etc/yum.repos.d/OceanBase.repo && \
     yum install -y ob-deploy obclient ob-sysbench gettext ncurses-compat-libs openssh-server && \
     yum clean all
 
@@ -47,9 +42,14 @@ RUN ssh-keygen -t rsa -f /root/.ssh/id_rsa -N ''
 RUN cat /root/.ssh/id_rsa.pub >> /root/.ssh/authorized_keys
 
 COPY --from=builder /root/demo /root/demo
-COPY --from=builder /root/demo/store.tar.gz /root/demo
 COPY --from=builder /root/.obd /root/.obd
+
+RUN yum install -y --downloadonly --downloaddir=/tmp ob-configserver && obd mirror clone /tmp/*.rpm && rm -rf /tmp/*.rpm
+
 COPY boot /root/boot
 COPY templates /root/templates
+COPY share/cluster /root/.obd/cluster
+COPY share/store.tar.gz /root/demo/
+COPY share/etc.tar.gz /root/demo/
 
 ENTRYPOINT ["bash", "-c", "/usr/sbin/sshd && /root/boot/start.sh"]
diff --git a/oceanbase-ce/Dockerfile.pre b/oceanbase-ce/Dockerfile.pre
@@ -0,0 +1,28 @@
+FROM openanolis/anolisos
+
+ARG VERSION
+
+WORKDIR /root
+
+RUN mkdir -p pkgs
+
+RUN yum install -y yum-utils && \
+    yum-config-manager --add-repo http://mirrors.oceanbase.com/oceanbase/OceanBase.repo && \
+    yum install -y ob-deploy && \
+    yum clean all
+
+ADD build/demo.yaml /root/
+# download and clone all the required packages
+RUN yum install -y --downloadonly --downloaddir=/root/pkgs oceanbase-ce-${VERSION}.el8 oceanbase-ce-libs-${VERSION}.el8
+RUN obd mirror clone /root/pkgs/*.rpm
+RUN obd env set IO_DEFAULT_CONFIRM 1
+RUN obd mirror disable remote
+RUN obd cluster deploy demo -c /root/demo.yaml
+RUN obd cluster start demo -v && obd cluster tenant create demo -n test -o express_oltp -v
+RUN obd cluster stop demo
+RUN sed -i "s/\s\+hash:.*//g" /root/.obd/cluster/demo/.data
+
+RUN cd /root/demo && tar -cvzf store.tar.gz store
+RUN cd /root/demo && tar -cvzf etc.tar.gz etc
+
+ENTRYPOINT ["bash", "-c", "cp /root/demo/*.tar.gz /root/share && cp -r /root/.obd/cluster /root/share"]
diff --git a/oceanbase-ce/boot/start.sh b/oceanbase-ce/boot/start.sh
@@ -88,7 +88,7 @@ function wait_tenant_connectable() {
 }
 
 function check_tenant_connectable() {
-	obclient -h127.1 -uroot@${OB_TENANT_NAME} -P2881 -e "show databases"
+	obclient -h127.1 -uroot@${OB_TENANT_NAME} -P2881 -e "alter user root identified by ''"
 	return $?
 }
 

diff --git a/oceanbase-ce/build/demo.yaml b/oceanbase-ce/build/demo.yaml
@@ -0,0 +1,21 @@
+oceanbase-ce:
+  servers:
+  - 127.0.0.1
+  global:
+    home_path: /root/demo
+    appname: obcluster
+    cluster_id: 1
+    scenario: express_oltp
+    datafile_size: 256M
+    datafile_next: 256M
+    datafile_maxsize: 3G
+    log_disk_size: 5G
+    enable_syslog_recycle: true
+    enable_syslog_wf: false
+    max_syslog_file_count: 4
+    memory_limit: 6144M
+    production_mode: false
+    __min_full_resource_pool_memory: 1073741824
+    system_memory: 1024M
+    cpu_count: 20
+    root_password: ""
diff --git a/oceanbase-ce/build/deploy.yaml b/oceanbase-ce/build/deploy.yaml