diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 2141e42..fb52652 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -22,7 +22,7 @@ jobs: with: python-version: 3.x - uses: pre-commit/action@v3.0.1 - - uses: pre-commit-ci/lite-action@v1.0.2 + - uses: pre-commit-ci/lite-action@v1.0.3 if: always() run-demo: diff --git a/demo/values.tpl.yaml b/demo/values.tpl.yaml index 5e50f45..d3ac4c1 100644 --- a/demo/values.tpl.yaml +++ b/demo/values.tpl.yaml @@ -2,6 +2,8 @@ global: # Override the imagePullPolicy to always so we can using latest image tags # without risking them being outdated. imagePullPolicy: Always + # Keep the job log for 1h for investigations + batchJobTTL: 3600 developer: urls: @@ -34,6 +36,7 @@ diracx: JobLoggingDB: SandboxMetadataDB: TaskQueueDB: + PilotAgentsDB: osDbs: dbs: JobParametersDB: diff --git a/diracx/templates/diracx/deployment-cli.yaml b/diracx/templates/diracx/deployment-cli.yaml index 96b3f26..c3a2b71 100644 --- a/diracx/templates/diracx/deployment-cli.yaml +++ b/diracx/templates/diracx/deployment-cli.yaml @@ -92,8 +92,6 @@ spec: value: {{ .Values.developer.urls.diracx }} - name: DIRACX_CA_PATH value: /ca/demo-ca.pem - - name: CHRIS_TEST - value: /ca/demo-ca.pem resources: {{- toYaml .Values.resources | nindent 12 }} {{- with .Values.nodeSelector }} diff --git a/diracx/templates/diracx/init-sql/job.yaml b/diracx/templates/diracx/init-sql/job.yaml index 37ef21f..77f0915 100644 --- a/diracx/templates/diracx/init-sql/job.yaml +++ b/diracx/templates/diracx/init-sql/job.yaml @@ -66,6 +66,7 @@ spec: name: diracx-sql-root-connection-urls key: DIRACX_DB_URL_{{ $dbName | upper }} {{- end }} + # The secrets are needed for the extension to appear resources: {{- toYaml .Values.initSql.resources | nindent 12 }} volumes: diff --git a/diracx/values.yaml b/diracx/values.yaml index 97fe44d..76c6fdb 100644 --- a/diracx/values.yaml +++ b/diracx/values.yaml @@ -141,6 +141,7 @@ diracx: # JobLoggingDB: # SandboxMetadataDB: # TaskQueueDB: + # PilotAgentsDB # ProxyDB: # user: proxyUser # password: hush diff --git a/k3s/examples/my.values.yaml b/k3s/examples/my.values.yaml index c01dbff..820cb29 100644 --- a/k3s/examples/my.values.yaml +++ b/k3s/examples/my.values.yaml @@ -17,6 +17,7 @@ diracx: JobLoggingDB: SandboxMetadataDB: TaskQueueDB: + PilotAgentsDB: osDbs: dbs: JobParametersDB: diff --git a/run_demo.sh b/run_demo.sh index 231bf3b..35eb019 100755 --- a/run_demo.sh +++ b/run_demo.sh @@ -97,25 +97,27 @@ function element_not_in_array() { return $found } -usage="${0##*/} [-h|--help] [--exit-when-done] [--offline] [--enable-coverage] [--no-mount-containerd] [--postgres] [--set-value key=value] [--ci-values=values.yaml] [--] [source directories]" +usage="${0##*/} [-h|--help] [--exit-when-done] [--offline] [--postgres] [--enable-coverage] [--no-mount-containerd] [--set-value key=value] [--ci-values=values.yaml] [--load-docker-image=] [--] [source directories]" usage+="\n\n" usage+=" -h|--help: Print this help message and exit\n" +usage+=" --ci-values: Path to a values.yaml file which contains diracx dev settings only enabled for CI\n" usage+=" --exit-when-done: Exit after the demo has been started (it will be left running in the background)\n" usage+=" --enable-coverage: Enable coverage reporting (used by diracx CI)\n" +usage+=" --enable-open-telemetry: lauches OpenTelemetry collection.\n" +usage+=" WARNING: experimental and resource hungry.\n" +usage+=" --load-docker-image: Mount a local docker image into kind\n" +usage+=" WARNING: the ImagePullPolicy MUST not be Always for this to work\n" usage+=" --no-editable-python: Do not install Python source directories in editable mode\n" -usage+=" --offline: Run in a mode which is suitable for fully offline use.\n" -usage+=" WARNING: This may result in some weird behaviour, see the demo documentation for details.\n" -usage+=" Implies: --mount-containerd\n" usage+=" --no-mount-containerd: Mount a directory on the host for the kind containerd storage.\n" usage+=" This option avoids needing to pull container images every time the demo is started.\n" usage+=" WARNING: There is no garbage collection so the directory will grow without bound.\n" -usage+=" --enable-open-telemetry: lauches OpenTelemetry collection.\n" -usage+=" WARNING: experimental and resource hungry.\n" +usage+=" --offline: Run in a mode which is suitable for fully offline use.\n" +usage+=" WARNING: This may result in some weird behaviour, see the demo documentation for details.\n" +usage+=" Implies: --mount-containerd\n" usage+=" --postgres: Runs the demo using a the PostgreSQL database.\n" usage+=" If not specified, it will default to MySQL.\n" usage+=" --set-value: Set a value in the Helm values file. This can be used to override the default values.\n" usage+=" For example, to enable coverage reporting pass: --set-value developer.enableCoverage=true\n" -usage+=" --ci-values: Path to a values.yaml file which contains diracx dev settings only enabled for CI\n" usage+=" source directories: A list of directories containing Python packages to mount in the demo cluster.\n" # Parse command-line switches @@ -126,7 +128,8 @@ declare -a helm_arguments=() enable_coverage=0 editable_python=1 open_telemetry=0 -ci_values_file="" +declare -a ci_values_files=() +declare -a docker_images_to_load=() while [ -n "${1:-}" ]; do case $1 in # Print a brief usage summary and exit @@ -199,6 +202,16 @@ while [ -n "${1:-}" ]; do case $1 in shift continue ;; + --load-docker-image) + shift + if [[ -z "${1:-}" ]]; then + printf "%b Error: --load-docker-image requires an argument\n" ${SKULL_EMOJI} + exit 1 + fi + docker_images_to_load+=("${1}") + shift + continue ;; + --ci-values) shift if [[ -z "${1:-}" ]]; then @@ -210,6 +223,7 @@ while [ -n "${1:-}" ]; do case $1 in printf "%b Error: --ci-values does not point to a file\n" ${SKULL_EMOJI} exit 1; fi + ci_values_files+=("${ci_values_file}") shift continue ;; @@ -244,7 +258,7 @@ done declare -a pkg_dirs=() declare -a python_pkg_names=() node_pkg_name="" -diracx_src_dir="" +declare -a diracx_and_extensions_pkgs=() for src_dir in "$@"; do if [ ${#pkg_dirs[@]} -gt 0 ] && ! element_not_in_array "$src_dir" "${pkg_dirs[@]}"; then @@ -253,18 +267,30 @@ for src_dir in "$@"; do fi pkg_dirs+=("${src_dir}") - # DiracX itself + + # Does that look like a namespace package with the structure we expect ? + # i.e. is it diracx itself or an extension ? if [ -f "${src_dir}/pyproject.toml" ]; then - if grep --quiet 'name = "diracx"' "${src_dir}/pyproject.toml"; then - if [ "${diracx_src_dir}" != "" ]; then - printf "%b Error: Source directory for DiracX was given twice!\n" "${SKULL_EMOJI}" - exit 1 + # Name of the package + pkg_name="$(basename "${src_dir}")" + + # Do we find subdirectories called the same way as the package + if [ -n "$(find "${src_dir}" -mindepth 3 -maxdepth 3 -type d -path "*src/${pkg_name}")" ]; then + # And are there mulftiple pyprojects + if [ -n "$(find "${src_dir}" -mindepth 2 -maxdepth 2 -type f -name "pyproject.toml")" ]; then + # Then let's add all these + while IFS= read -r sub_pkg_dir + do + diracx_and_extensions_pkgs+=("$(basename "${src_dir}")/$(basename "${sub_pkg_dir}")"); + done < <(find "${src_dir}" -mindepth 1 -maxdepth 1 -type d -name "${pkg_name}-*" ) + + continue; + fi; fi - diracx_src_dir="${src_dir}" - continue - fi + fi + # Python packages if [ -f "${src_dir}/pyproject.toml" ]; then while IFS='' read -r pkg_dir; do @@ -289,8 +315,9 @@ for src_dir in "$@"; do done < <(find "$src_dir" -mindepth 1 -maxdepth 1 -type f -name 'package.json') done -if [ "${diracx_src_dir}" != "" ]; then - printf "%b Found DiracX directory: %s\n" ${UNICORN_EMOJI} "${diracx_src_dir}" + +if [ ${#diracx_and_extensions_pkgs[@]} -gt 0 ]; then + printf "%b Found Diracx/Extensions package directories for: %s\n" ${UNICORN_EMOJI} "${diracx_and_extensions_pkgs[*]}" fi if [ ${#python_pkg_names[@]} -gt 0 ]; then printf "%b Found Python package directories for: %s\n" ${UNICORN_EMOJI} "${python_pkg_names[*]}" @@ -462,11 +489,12 @@ if [ ${#python_pkg_names[@]} -gt 0 ]; then json+="\"$pkg_name\"," done fi -if [ "${diracx_src_dir}" != "" ]; then - for pkg_name in "$(basename "${diracx_src_dir}")/diracx-"{cli,client,api,core,routers,db}; do - json+="\"$pkg_name\"," - done -fi + +for diracx_compatible_pkg in "${diracx_and_extensions_pkgs[@]}"; do + json+="\"$diracx_compatible_pkg\"," + +done + json="${json%,}]" sed "s#{{ mounted_python_modules }}#${json}#g" "${demo_dir}/values.yaml.bak" > "${demo_dir}/values.yaml" mv "${demo_dir}/values.yaml" "${demo_dir}/values.yaml.bak" @@ -480,13 +508,18 @@ if grep '{{' "${demo_dir}/values.yaml"; then exit 1 fi + # Add an ingress to the cluster printf "%b Creating an ingress...\n" ${UNICORN_EMOJI} # TODO: This should move to the chart itself if [ ${offline_mode} -eq 0 ]; then curl -L https://raw.githubusercontent.com/kubernetes/ingress-nginx/main/deploy/static/provider/kind/deploy.yaml > "${tmp_dir}/kind-ingress-deploy.yaml" - mv "${tmp_dir}/kind-ingress-deploy.yaml" "${demo_dir}/kind-ingress-deploy.yaml" + # Disable the strict validation of the path + # https://github.com/kubernetes/ingress-nginx/issues/11176 + # https://github.com/kubernetes/ingress-nginx/issues/10200 + sed -E 's/^data: null/data:\n strict-validate-path-type: "false"/g' "${tmp_dir}/kind-ingress-deploy.yaml" > "${demo_dir}/kind-ingress-deploy.yaml" fi + "${demo_dir}/kubectl" apply -f "${demo_dir}/kind-ingress-deploy.yaml" printf "%b Waiting for ingress controller to be created...\n" ${UNICORN_EMOJI} "${demo_dir}/kubectl" wait --namespace ingress-nginx \ @@ -497,9 +530,30 @@ printf "%b Waiting for ingress controller to be created...\n" ${UNICORN_EMOJI} # Install the DiracX chart printf "%b Installing DiracX...\n" ${UNICORN_EMOJI} helm_arguments+=("--values" "${demo_dir}/values.yaml") -if [[ -n "${ci_values_file}" ]]; then - helm_arguments+=("--values" "${ci_values_file}") + +if [ ${#ci_values_files[@]} -ne 0 ]; then + printf "%b Adding extra values.yaml: ${ci_values_files[*]} \n" ${UNICORN_EMOJI} + for ci_values_file in "${ci_values_files[@]}" + do + helm_arguments+=("--values" "${ci_values_file}") + done fi + + +# Load images into kind + +if [ ${#docker_images_to_load[@]} -ne 0 ]; then +printf "%b Loading docker images...\n" ${UNICORN_EMOJI} +for img_name in "${docker_images_to_load[@]}" +do + "${demo_dir}/kind" --name diracx-demo load docker-image "${img_name}" +done + + +fi; + + + if ! "${demo_dir}/helm" install --debug diracx-demo "${script_dir}/diracx" "${helm_arguments[@]}"; then printf "%b Error using helm DiracX\n" ${WARN_EMOJI} echo "Failed to run \"helm install\"" >> "${demo_dir}/.failed"