From b1a057cbd9d6f7edbd8ec0bfc322d91866b5f8bd Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 16 Sep 2024 16:00:56 +0000 Subject: [PATCH 1/7] build(deps): bump pre-commit-ci/lite-action from 1.0.2 to 1.0.3 Bumps [pre-commit-ci/lite-action](https://github.com/pre-commit-ci/lite-action) from 1.0.2 to 1.0.3. - [Release notes](https://github.com/pre-commit-ci/lite-action/releases) - [Commits](https://github.com/pre-commit-ci/lite-action/compare/v1.0.2...v1.0.3) --- updated-dependencies: - dependency-name: pre-commit-ci/lite-action dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 2141e42..fb52652 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -22,7 +22,7 @@ jobs: with: python-version: 3.x - uses: pre-commit/action@v3.0.1 - - uses: pre-commit-ci/lite-action@v1.0.2 + - uses: pre-commit-ci/lite-action@v1.0.3 if: always() run-demo: From cb85f9510a6e18d3cba015887676de8b9dd6b9f9 Mon Sep 17 00:00:00 2001 From: Christophe Haen Date: Thu, 26 Sep 2024 09:59:32 +0200 Subject: [PATCH 2/7] run_demo: introduce --load-docker-image --- run_demo.sh | 43 ++++++++++++++++++++++++++++++++++++------- 1 file changed, 36 insertions(+), 7 deletions(-) diff --git a/run_demo.sh b/run_demo.sh index 8d18868..1f6b157 100755 --- a/run_demo.sh +++ b/run_demo.sh @@ -97,23 +97,25 @@ function element_not_in_array() { return $found } -usage="${0##*/} [-h|--help] [--exit-when-done] [--offline] [--enable-coverage] [--no-mount-containerd] [--set-value key=value] [--ci-values=values.yaml] [--] [source directories]" +usage="${0##*/} [-h|--help] [--exit-when-done] [--offline] [--enable-coverage] [--no-mount-containerd] [--set-value key=value] [--ci-values=values.yaml] [--load-docker-image=] [--] [source directories]" usage+="\n\n" usage+=" -h|--help: Print this help message and exit\n" +usage+=" --ci-values: Path to a values.yaml file which contains diracx dev settings only enabled for CI\n" usage+=" --exit-when-done: Exit after the demo has been started (it will be left running in the background)\n" usage+=" --enable-coverage: Enable coverage reporting (used by diracx CI)\n" +usage+=" --enable-open-telemetry: lauches OpenTelemetry collection.\n" +usage+=" WARNING: experimental and resource hungry.\n" +usage+=" --load-docker-image: Mount a local docker image into kind\n" +usage+=" WARNING: the ImagePullPolicy MUST not be Always for this to work\n" usage+=" --no-editable-python: Do not install Python source directories in editable mode\n" -usage+=" --offline: Run in a mode which is suitable for fully offline use.\n" -usage+=" WARNING: This may result in some weird behaviour, see the demo documentation for details.\n" -usage+=" Implies: --mount-containerd\n" usage+=" --no-mount-containerd: Mount a directory on the host for the kind containerd storage.\n" usage+=" This option avoids needing to pull container images every time the demo is started.\n" usage+=" WARNING: There is no garbage collection so the directory will grow without bound.\n" -usage+=" --enable-open-telemetry: lauches OpenTelemetry collection.\n" -usage+=" WARNING: experimental and resource hungry.\n" +usage+=" --offline: Run in a mode which is suitable for fully offline use.\n" +usage+=" WARNING: This may result in some weird behaviour, see the demo documentation for details.\n" +usage+=" Implies: --mount-containerd\n" usage+=" --set-value: Set a value in the Helm values file. This can be used to override the default values.\n" usage+=" For example, to enable coverage reporting pass: --set-value developer.enableCoverage=true\n" -usage+=" --ci-values: Path to a values.yaml file which contains diracx dev settings only enabled for CI\n" usage+=" source directories: A list of directories containing Python packages to mount in the demo cluster.\n" # Parse command-line switches @@ -125,6 +127,7 @@ enable_coverage=0 editable_python=1 open_telemetry=0 ci_values_file="" +declare -a docker_images_to_load=() while [ -n "${1:-}" ]; do case $1 in # Print a brief usage summary and exit @@ -197,6 +200,16 @@ while [ -n "${1:-}" ]; do case $1 in shift continue ;; + --load-docker-image) + shift + if [[ -z "${1:-}" ]]; then + printf "%b Error: --load-docker-image requires an argument\n" ${SKULL_EMOJI} + exit 1 + fi + docker_images_to_load+=("${1}") + shift + continue ;; + --ci-values) shift if [[ -z "${1:-}" ]]; then @@ -490,6 +503,22 @@ helm_arguments+=("--values" "${demo_dir}/values.yaml") if [[ -n "${ci_values_file}" ]]; then helm_arguments+=("--values" "${ci_values_file}") fi + + +# Load images into kind + +if [ ${#docker_images_to_load[@]} -ne 0 ]; then +printf "%b Loading docker images...\n" ${UNICORN_EMOJI} +for img_name in "${docker_images_to_load[@]}" +do + "${demo_dir}/kind" --name diracx-demo load docker-image "${img_name}" +done + + +fi; + + + if ! "${demo_dir}/helm" install --debug diracx-demo "${script_dir}/diracx" "${helm_arguments[@]}"; then printf "%b Error using helm DiracX\n" ${WARN_EMOJI} echo "Failed to run \"helm install\"" >> "${demo_dir}/.failed" From 6ffaa120d87c203e261b65f54b6ba14021e415f4 Mon Sep 17 00:00:00 2001 From: Christophe Haen Date: Wed, 2 Oct 2024 11:26:00 +0200 Subject: [PATCH 3/7] run_demo: detect extension packages --- demo/values.tpl.yaml | 2 + diracx/templates/diracx/deployment-cli.yaml | 2 - diracx/templates/diracx/init-sql/job.yaml | 1 + run_demo.sh | 47 ++++++++++++++------- 4 files changed, 34 insertions(+), 18 deletions(-) diff --git a/demo/values.tpl.yaml b/demo/values.tpl.yaml index 5e50f45..4e5e18f 100644 --- a/demo/values.tpl.yaml +++ b/demo/values.tpl.yaml @@ -2,6 +2,8 @@ global: # Override the imagePullPolicy to always so we can using latest image tags # without risking them being outdated. imagePullPolicy: Always + # Keep the job log for 1h for investigations + batchJobTTL: 3600 developer: urls: diff --git a/diracx/templates/diracx/deployment-cli.yaml b/diracx/templates/diracx/deployment-cli.yaml index 96b3f26..c3a2b71 100644 --- a/diracx/templates/diracx/deployment-cli.yaml +++ b/diracx/templates/diracx/deployment-cli.yaml @@ -92,8 +92,6 @@ spec: value: {{ .Values.developer.urls.diracx }} - name: DIRACX_CA_PATH value: /ca/demo-ca.pem - - name: CHRIS_TEST - value: /ca/demo-ca.pem resources: {{- toYaml .Values.resources | nindent 12 }} {{- with .Values.nodeSelector }} diff --git a/diracx/templates/diracx/init-sql/job.yaml b/diracx/templates/diracx/init-sql/job.yaml index 3dd7eb5..dfcf8f5 100644 --- a/diracx/templates/diracx/init-sql/job.yaml +++ b/diracx/templates/diracx/init-sql/job.yaml @@ -61,6 +61,7 @@ spec: name: diracx-sql-root-connection-urls key: DIRACX_DB_URL_{{ $dbName | upper }} {{- end }} + # The secrets are needed for the extension to appear resources: {{- toYaml .Values.initSql.resources | nindent 12 }} volumes: diff --git a/run_demo.sh b/run_demo.sh index 1f6b157..5b7d868 100755 --- a/run_demo.sh +++ b/run_demo.sh @@ -247,7 +247,7 @@ done declare -a pkg_dirs=() declare -a python_pkg_names=() node_pkg_name="" -diracx_src_dir="" +declare -a diracx_and_extensions_pkgs=() for src_dir in "$@"; do if [ ${#pkg_dirs[@]} -gt 0 ] && ! element_not_in_array "$src_dir" "${pkg_dirs[@]}"; then @@ -256,18 +256,30 @@ for src_dir in "$@"; do fi pkg_dirs+=("${src_dir}") - # DiracX itself + + # Does that look like a namespace package with the structure we expect ? + # i.e. is it diracx itself or an extension ? if [ -f "${src_dir}/pyproject.toml" ]; then - if grep --quiet 'name = "diracx"' "${src_dir}/pyproject.toml"; then - if [ "${diracx_src_dir}" != "" ]; then - printf "%b Error: Source directory for DiracX was given twice!\n" "${SKULL_EMOJI}" - exit 1 + # Name of the package + pkg_name="$(basename "${src_dir}")" + + # Do we find subdirectories called the same way as the package + if [ -n "$(find "${src_dir}" -mindepth 3 -maxdepth 3 -type d -path "*src/${pkg_name}")" ]; then + # And are there mulftiple pyprojects + if [ -n "$(find "${src_dir}" -mindepth 2 -maxdepth 2 -type f -name "pyproject.toml")" ]; then + # Then let's add all these + while IFS= read -r sub_pkg_dir + do + diracx_and_extensions_pkgs+=("$(basename "${src_dir}")/$(basename "${sub_pkg_dir}")"); + done < <(find "${src_dir}" -mindepth 1 -maxdepth 1 -type d -name "${pkg_name}-*" ) + + continue; + fi; fi - diracx_src_dir="${src_dir}" - continue - fi + fi + # Python packages if [ -f "${src_dir}/pyproject.toml" ]; then while IFS='' read -r pkg_dir; do @@ -292,8 +304,9 @@ for src_dir in "$@"; do done < <(find "$src_dir" -mindepth 1 -maxdepth 1 -type f -name 'package.json') done -if [ "${diracx_src_dir}" != "" ]; then - printf "%b Found DiracX directory: %s\n" ${UNICORN_EMOJI} "${diracx_src_dir}" + +if [ ${#diracx_and_extensions_pkgs[@]} -gt 0 ]; then + printf "%b Found Diracx/Extensions package directories for: %s\n" ${UNICORN_EMOJI} "${diracx_and_extensions_pkgs[*]}" fi if [ ${#python_pkg_names[@]} -gt 0 ]; then printf "%b Found Python package directories for: %s\n" ${UNICORN_EMOJI} "${python_pkg_names[*]}" @@ -465,11 +478,12 @@ if [ ${#python_pkg_names[@]} -gt 0 ]; then json+="\"$pkg_name\"," done fi -if [ "${diracx_src_dir}" != "" ]; then - for pkg_name in "$(basename "${diracx_src_dir}")/diracx-"{cli,client,api,core,routers,db}; do - json+="\"$pkg_name\"," - done -fi + +for diracx_compatible_pkg in "${diracx_and_extensions_pkgs[@]}"; do + json+="\"$diracx_compatible_pkg\"," + +done + json="${json%,}]" sed "s#{{ mounted_python_modules }}#${json}#g" "${demo_dir}/values.yaml.bak" > "${demo_dir}/values.yaml" mv "${demo_dir}/values.yaml" "${demo_dir}/values.yaml.bak" @@ -483,6 +497,7 @@ if grep '{{' "${demo_dir}/values.yaml"; then exit 1 fi + # Add an ingress to the cluster printf "%b Creating an ingress...\n" ${UNICORN_EMOJI} # TODO: This should move to the chart itself From c2f85ec3f4efb5d9f4ce6e3630c485179748ebc4 Mon Sep 17 00:00:00 2001 From: Christophe Haen Date: Thu, 10 Oct 2024 17:09:43 +0200 Subject: [PATCH 4/7] run_demo: make ci-values a list option --- run_demo.sh | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/run_demo.sh b/run_demo.sh index 5b7d868..5121561 100755 --- a/run_demo.sh +++ b/run_demo.sh @@ -126,7 +126,7 @@ declare -a helm_arguments=() enable_coverage=0 editable_python=1 open_telemetry=0 -ci_values_file="" +declare -a ci_values_files=() declare -a docker_images_to_load=() while [ -n "${1:-}" ]; do case $1 in @@ -221,6 +221,7 @@ while [ -n "${1:-}" ]; do case $1 in printf "%b Error: --ci-values does not point to a file\n" ${SKULL_EMOJI} exit 1; fi + ci_values_files+=("${ci_values_file}") shift continue ;; @@ -515,8 +516,13 @@ printf "%b Waiting for ingress controller to be created...\n" ${UNICORN_EMOJI} # Install the DiracX chart printf "%b Installing DiracX...\n" ${UNICORN_EMOJI} helm_arguments+=("--values" "${demo_dir}/values.yaml") -if [[ -n "${ci_values_file}" ]]; then - helm_arguments+=("--values" "${ci_values_file}") + +if [ ${#ci_values_files[@]} -ne 0 ]; then + printf "%b Adding extra values.yaml: ${ci_values_files[*]} \n" ${UNICORN_EMOJI} + for ci_values_file in "${ci_values_files[@]}" + do + helm_arguments+=("--values" "${ci_values_file}") + done fi From f50f547deaa973537c5489b1ecea815c5362fe6b Mon Sep 17 00:00:00 2001 From: martynia Date: Wed, 11 Sep 2024 10:10:36 +0200 Subject: [PATCH 5/7] feat: Add PilotAgentsDB to a list of SQL databases --- demo/values.tpl.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/demo/values.tpl.yaml b/demo/values.tpl.yaml index 4e5e18f..d3ac4c1 100644 --- a/demo/values.tpl.yaml +++ b/demo/values.tpl.yaml @@ -36,6 +36,7 @@ diracx: JobLoggingDB: SandboxMetadataDB: TaskQueueDB: + PilotAgentsDB: osDbs: dbs: JobParametersDB: From 497afaa69c7d23d8b8d6530bf2d60f78d4b1468b Mon Sep 17 00:00:00 2001 From: martynia Date: Fri, 11 Oct 2024 17:32:56 +0200 Subject: [PATCH 6/7] feat: add PilotAgentsDB to values.yaml --- diracx/values.yaml | 1 + k3s/examples/my.values.yaml | 1 + 2 files changed, 2 insertions(+) diff --git a/diracx/values.yaml b/diracx/values.yaml index 3f7cdff..ec74931 100644 --- a/diracx/values.yaml +++ b/diracx/values.yaml @@ -140,6 +140,7 @@ diracx: # JobLoggingDB: # SandboxMetadataDB: # TaskQueueDB: + # PilotAgentsDB # ProxyDB: # user: proxyUser # password: hush diff --git a/k3s/examples/my.values.yaml b/k3s/examples/my.values.yaml index c01dbff..820cb29 100644 --- a/k3s/examples/my.values.yaml +++ b/k3s/examples/my.values.yaml @@ -17,6 +17,7 @@ diracx: JobLoggingDB: SandboxMetadataDB: TaskQueueDB: + PilotAgentsDB: osDbs: dbs: JobParametersDB: From 755c3e82181d8077d5b679757793c5306a53ca8e Mon Sep 17 00:00:00 2001 From: Christophe Haen Date: Tue, 15 Oct 2024 14:51:41 +0200 Subject: [PATCH 7/7] run_demo: disable strict-validate-path-type for nginx ingress --- run_demo.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/run_demo.sh b/run_demo.sh index 5121561..fbd9e8a 100755 --- a/run_demo.sh +++ b/run_demo.sh @@ -504,8 +504,12 @@ printf "%b Creating an ingress...\n" ${UNICORN_EMOJI} # TODO: This should move to the chart itself if [ ${offline_mode} -eq 0 ]; then curl -L https://raw.githubusercontent.com/kubernetes/ingress-nginx/main/deploy/static/provider/kind/deploy.yaml > "${tmp_dir}/kind-ingress-deploy.yaml" - mv "${tmp_dir}/kind-ingress-deploy.yaml" "${demo_dir}/kind-ingress-deploy.yaml" + # Disable the strict validation of the path + # https://github.com/kubernetes/ingress-nginx/issues/11176 + # https://github.com/kubernetes/ingress-nginx/issues/10200 + sed -E 's/^data: null/data:\n strict-validate-path-type: "false"/g' "${tmp_dir}/kind-ingress-deploy.yaml" > "${demo_dir}/kind-ingress-deploy.yaml" fi + "${demo_dir}/kubectl" apply -f "${demo_dir}/kind-ingress-deploy.yaml" printf "%b Waiting for ingress controller to be created...\n" ${UNICORN_EMOJI} "${demo_dir}/kubectl" wait --namespace ingress-nginx \