[branch-2.0](feature) pick docker suite framework (#34154)

docker/runtime/doris-compose/Dockerfile

@@ -16,18 +16,48 @@
 # specific language governing permissions and limitations
 # under the License.
 
+#### START ARG ####
+
+# docker build cmd example:
+# docker build -f docker/runtime/doris-compose/Dockerfile -t <your-image-name>:<version> .
+
 # choose a base image
-FROM openjdk:8u342-jdk
+# ARG JDK_IMAGE=openjdk:17-jdk-slim
+ARG JDK_IMAGE=openjdk:8u342-jdk
+
+#### END ARG ####
+
+FROM ${JDK_IMAGE}
+
+RUN <<EOF
+    if [ -d "/usr/local/openjdk-17" ]; then
+        ln -s /usr/local/openjdk-17  /usr/local/openjdk
+    else \
+        ln -s /usr/local/openjdk-8  /usr/local/openjdk
+    fi
+EOF
 
 # set environment variables
-ENV JAVA_HOME="/usr/local/openjdk-8/"
+ENV JAVA_HOME="/usr/local/openjdk"
+ENV JACOCO_VERSION 0.8.8
 
-ADD output /opt/apache-doris/
+RUN mkdir -p /opt/apache-doris/coverage
 
 RUN  sed -i s@/deb.debian.org/@/mirrors.aliyun.com/@g /etc/apt/sources.list
 RUN  apt-get clean
 
 RUN apt-get update && \
-    apt-get install -y default-mysql-client python lsof && \
+    apt-get install -y default-mysql-client python lsof tzdata curl unzip patchelf jq procps && \
+    ln -fs /usr/share/zoneinfo/Asia/Shanghai /etc/localtime && \
+    dpkg-reconfigure -f noninteractive tzdata && \
     apt-get clean
 
+RUN curl -f https://repo1.maven.org/maven2/org/jacoco/jacoco/${JACOCO_VERSION}/jacoco-${JACOCO_VERSION}.zip -o jacoco.zip && \
+    mkdir /jacoco && \
+    unzip jacoco.zip -d /jacoco
+
+# fe and be
+COPY output /opt/apache-doris/
+# in docker, run 'chmod 755 doris_be' first time cost 1min, remove it.
+RUN sed -i 's/\<chmod\>/echo/g' /opt/apache-doris/be/bin/start_be.sh
+

docker/runtime/doris-compose/Readme.md

@@ -0,0 +1,133 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+# Doris compose
+
+Use doris compose to create doris docker compose clusters.
+
+## Requirements
+
+1. The doris image should contains:
+
+```
+/opt/apache-doris/{fe, be, cloud}
+```
+
+if don't create cloud cluster, the image no need to contains the cloud pkg.
+
+
+if build doris use `sh build.sh --fe --be --cloud`, then its output satisfy with all above, then run command in doris root
+
+```
+docker build -f docker/runtime/doris-compose/Dockerfile -t <image> .
+```
+
+will generate a image.
+
+2. Install the dependent python library in 'docker/runtime/doris-compose/requirements.txt'
+
+
+```
+python -m pip install --user -r docker/runtime/doris-compose/requirements.txt
+```
+
+## Usage
+
+### Create a cluster or recreate its containers
+
+```
+python docker/runtime/doris-compose/doris-compose.py up  <cluster-name>   <image?> 
+    --add-fe-num  <add-fe-num>  --add-be-num <add-be-num>
+    --fe-id <fd-id> --be-id <be-id>
+    ...
+    [ --cloud ]
+```
+
+if it's a new cluster, must specific the image.
+
+add fe/be nodes with the specific image, or update existing nodes with `--fe-id`, `--be-id`
+
+
+For create a cloud cluster, steps are as below:
+1. Write cloud s3 store config file, its default path is '/tmp/doris/cloud.ini'.
+   It's defined in environment variable DORIS_CLOUD_CFG_FILE, user can change this env var to change its path.
+   A Example file is locate in 'docker/runtime/doris-compose/resource/cloud.ini.example'.
+2. Use doris compose up command with option '--cloud' to create a new cloud cluster.
+
+The simplest way to create a cloud cluster:
+
+```
+python docker/runtime/doris-compose/doris-compose.py up  <cluster-name>  <image>  --cloud
+```
+
+It will create 1 fdb, 1 meta service server, 1 recycler, 3 fe and 3 be.
+
+### Remove node from the cluster
+
+```
+python docker/runtime/doris-compose/doris-compose.py down  <cluster-name> --fe-id <fe-id>  --be-id<be-id> [--clean]  [--drop-force]
+```
+
+Down the containers and remove it from the DB.
+
+For BE, if specific drop force, it will send dropp sql to FE, otherwise it will send decommission sql to FE.
+
+If specific `--clean`, it will delete its data too.
+
+
+### Start, stop, restart specific nodes
+
+
+```
+python docker/runtime/doris-compose/doris-compose.py start  <cluster-name>  --fe-id  <multiple fe ids>  --be-id <multiple be ids>
+python docker/runtime/doris-compose/doris-compose.py restart  <cluster-name>  --fe-id  <multiple fe ids>  --be-id <multiple be ids>
+```
+
+### List doris cluster
+
+```
+python docker/runtime/doris-compose/doris-compose.py ls <multiple cluster names>
+```
+
+if specific cluster names, it will list all the cluster's nodes.
+
+Otherwise it will just list summary of each clusters.
+
+There are more options about doris-compose. Just try 
+
+```
+python docker/runtime/doris-compose/doris-compose.py  <command> -h 
+```
+
+### Generate regression custom conf file
+
+```
+python docker/runtime/doris-compose/doris-compose.py config <cluster-name>  <doris-root-path>  [-q]  [--connect-follow-fe]
+```
+
+Generate regression-conf-custom.groovy to connect to the specific docker cluster.
+
+### Setup cloud multi clusters test env
+
+steps:
+
+1. Create a new cluster:  `python doris-compose.py up my-cluster  my-image  --add-fe-num 2  --add-be-num 4 --cloud`
+2. Generate regression-conf-custom.groovy: `python doris-compose.py config my-cluster  <doris-root-path> --connect-follow-fe`
+3. Run regression test: `bash run-regression-test.sh --run -times 1 -parallel 1 -suiteParallel 1 -d cloud/multi_cluster`
+