Adding conda image to docker registry and CI fix (#54)

Adding conda image to docker registry and CI fix
atlas-outreach-data-tools · Aug 26, 2024 · aaee502 · aaee502
1 parent 1b961c2
commit aaee502
Show file tree

Hide file tree

Showing 5 changed files with 56 additions and 124 deletions.
diff --git a/.github/workflows/docker_push.yml b/.github/workflows/docker_push.yml
@@ -7,6 +7,7 @@ on:
       - 'master'
     paths:
       - 'Dockerfile'
+      - 'Dockerfile.conda'
       - '.github/workflows/docker_push.yml'
 
 # Defines two custom environment variables for the workflow. These are used for the Container registry domain, and a name for the Docker image that this workflow builds.
@@ -30,15 +31,15 @@ jobs:
         uses: actions/checkout@v4
       # Uses the `docker/login-action` action to log in to the Container registry registry using the account and password that will publish the packages. Once published, the packages are scoped to the account defined here.
       - name: Log in to the Container registry
-        uses: docker/login-action@v2.1.0
+        uses: docker/login-action@v3.3.0
         with:
           registry: ${{ env.REGISTRY }}
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
       # This step uses [docker/metadata-action](https://github.com/docker/metadata-action#about) to extract tags and labels that will be applied to the specified image. The `id` "meta" allows the output of this step to be referenced in a subsequent step. The `images` value provides the base name for the tags and labels.
       - name: Extract metadata (tags, labels) for Docker
         id: meta
-        uses: docker/metadata-action@v4.3.0
+        uses: docker/metadata-action@v5.5.0
         with:
           flavor: |
             latest=true
@@ -48,15 +49,16 @@ jobs:
       # This step uses the `docker/build-push-action` action to build the image, based on your repository's `Dockerfile`. If the build succeeds, it pushes the image to GitHub Packages.
       # It uses the `context` parameter to define the build's context as the set of files located in the specified path. For more information, see "[Usage](https://github.com/docker/build-push-action#usage)" in the README of the `docker/build-push-action` repository.
       # It uses the `tags` and `labels` parameters to tag and label the image with the output from the "meta" step.
-      - name: Build and push Docker image
+
+      - name: Build and push Docker image - base
         id: push
-        uses: docker/build-push-action@v4.0.0
+        uses: docker/build-push-action@v6.0.0
         with:
           context: .
+          file: Dockerfile
           push: true
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
-          visibility: public
 
       # This step generates an artifact attestation for the image, which is an unforgeable statement about where and how it was built. It increases supply chain security for people who consume the image. For more information, see "[AUTOTITLE](/actions/security-guides/using-artifact-attestations-to-establish-provenance-for-builds)." 
       - name: Generate artifact attestation

diff --git a/.github/workflows/notebooks-runner-autocheck.yml b/.github/workflows/notebooks-runner-autocheck.yml
@@ -1,8 +1,7 @@
 name: Run Jupyter Notebooks
 
 on:
-  # push:
-  pull_request:
+  push:
     branches:
         - 'master'
     paths-ignore:
@@ -44,14 +43,14 @@ jobs:
             NOTEBOOKS="[]"
           fi
           
-          NOTEBOOKS8_cpp=$(find ./8-TeV-examples/cpp -name '*.ipynb' -print0 | jq -Rsc 'split("\u0000")[:-1]')
+          # NOTEBOOKS8_cpp=$(find ./8-TeV-examples/cpp -name '*.ipynb' -print0 | jq -Rsc 'split("\u0000")[:-1]')
           if [ -z "$NOTEBOOKS8_cpp" ]; then
             echo "No notebooks found"
             NOTEBOOKS8_cpp="[]"
           fi
           echo "notebooks8-cpp=$NOTEBOOKS8_cpp" >> $GITHUB_OUTPUT
 
-          NOTEBOOKS8_python=$(find ./8-TeV-examples/python -name '*.ipynb' -print0 | jq -Rsc 'split("\u0000")[:-1]')
+          # NOTEBOOKS8_python=$(find ./8-TeV-examples/python -name '*.ipynb' -print0 | jq -Rsc 'split("\u0000")[:-1]')
           if [ -z "$NOTEBOOKS8_python" ]; then
             echo "No notebooks found"
             NOTEBOOKS8_python="[]"
@@ -111,19 +110,10 @@ jobs:
       - name: Checkout repository
         uses: actions/checkout@v4
 
-      # - name: Set up Python
-      #   uses: actions/setup-python@v5
-      #   with:
-      #     python-version: '3.8'
-
-      - name: Set up Jupyter kernel
-        run: |
-          python -m ipykernel install --user --name=root
-
       - name: Run notebook with Papermill
         run: |
           cd $(dirname ${{ matrix.notebook }})
-          papermill $(basename ${{ matrix.notebook }}) /dev/null
+          papermill --log-level ERROR $(basename ${{ matrix.notebook }}) /dev/null
 
       - name: Output result
         if: always()
@@ -135,7 +125,7 @@ jobs:
 
       - name: Upload results
         if: always()
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4.3.5
         with:
           name: result_run-notebooks-8-cpp-${{ env.sanitized_name }}.txt
           path: ./artifacts
@@ -155,19 +145,10 @@ jobs:
       - name: Checkout repository
         uses: actions/checkout@v4
 
-      # - name: Set up Python
-      #   uses: actions/setup-python@v5
-      #   with:
-      #     python-version: '3.11.6'
-
-      - name: Set up Jupyter kernel
-        run: |
-          python -m ipykernel install --user --name=python3
-
       - name: Run notebook with Papermill
         run: |
           cd $(dirname ${{ matrix.notebook }})
-          papermill $(basename ${{ matrix.notebook }}) /dev/null
+          papermill --log-level ERROR $(basename ${{ matrix.notebook }}) /dev/null
 
       - name: Output result
         if: always()
@@ -179,7 +160,7 @@ jobs:
 
       - name: Upload results
         if: always()
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4.3.5
         with:
           name: result_run-notebooks-8-python-${{ env.sanitized_name }}.txt
           path: ./artifacts
@@ -198,21 +179,10 @@ jobs:
       - name: Checkout repository
         uses: actions/checkout@v4
 
-      # - name: Set up Python
-      #   uses: actions/setup-python@v5
-      #   with:
-      #     python-version: '3.11.6'
-
-
-
-      - name: Set up Jupyter kernel
-        run: |
-          python -m ipykernel install --user --name=root
-
       - name: Run notebook with Papermill
         run: |
           cd $(dirname ${{ matrix.notebook }})
-          papermill $(basename ${{ matrix.notebook }}) /dev/null
+          papermill --log-level ERROR $(basename ${{ matrix.notebook }}) /dev/null
 
       - name: Output result
         if: always()
@@ -224,7 +194,7 @@ jobs:
 
       - name: Upload results
         if: always()
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4.3.5
         with:
           name: result_run-notebooks-13-cpp-${{ env.sanitized_name }}.txt
           path: ./artifacts
@@ -243,21 +213,10 @@ jobs:
       - name: Checkout repository
         uses: actions/checkout@v4
 
-      # - name: Set up Python
-      #   uses: actions/setup-python@v5
-      #   with:
-      #     python-version: '3.11.6'
-
-
-
-      - name: Set up Jupyter kernel
-        run: |
-          python -m ipykernel install --user --name=python3
-
       - name: Run notebook with Papermill
         run: |
           cd $(dirname ${{ matrix.notebook }})
-          papermill $(basename ${{ matrix.notebook }}) /dev/null
+          papermill --log-level ERROR $(basename ${{ matrix.notebook }}) /dev/null
 
       - name: Output result
         if: always()
@@ -269,7 +228,7 @@ jobs:
 
       - name: Upload results
         if: always()
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4.3.5
         with:
           name: result_run-notebooks-13-python-${{ env.sanitized_name }}.txt
           path: ./artifacts
@@ -288,15 +247,10 @@ jobs:
       - name: Checkout repository
         uses: actions/checkout@v4
 
-
-      - name: Set up Jupyter kernel
-        run: |
-          python -m ipykernel install --user --name=python3
-
       - name: Run notebook with Papermill
         run: |
           cd $(dirname ${{ matrix.notebook }})
-          papermill $(basename ${{ matrix.notebook }}) /dev/null
+          papermill --log-level ERROR $(basename ${{ matrix.notebook }}) /dev/null
 
       - name: Output result
         if: always()
@@ -308,7 +262,7 @@ jobs:
 
       - name: Upload results
         if: always()
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4.3.5
         with:
           name: result_run-notebooks-13-uproot-${{ env.sanitized_name }}.txt
           path: ./artifacts
@@ -327,15 +281,10 @@ jobs:
       - name: Checkout repository
         uses: actions/checkout@v4
 
-
-      - name: Set up Jupyter kernel
-        run: |
-          python -m ipykernel install --user --name=python3
-
       - name: Run notebook with Papermill
         run: |
           cd $(dirname ${{ matrix.notebook }})
-          papermill $(basename ${{ matrix.notebook }}) /dev/null
+          papermill --log-level ERROR $(basename ${{ matrix.notebook }}) /dev/null
 
       - name: Output result
         if: always()
@@ -347,7 +296,7 @@ jobs:
 
       - name: Upload results
         if: always()
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4.3.5
         with:
           name: result_run-notebooks-13-rdataframe-${{ env.sanitized_name }}.txt
           path: ./artifacts
@@ -366,15 +315,10 @@ jobs:
       - name: Checkout repository
         uses: actions/checkout@v4
 
-
-      - name: Set up Jupyter kernel
-        run: |
-          python -m ipykernel install --user --name=python3
-
       - name: Run notebook with Papermill
         run: |
           cd $(dirname ${{ matrix.notebook }})
-          papermill $(basename ${{ matrix.notebook }}) /dev/null
+          papermill --log-level ERROR $(basename ${{ matrix.notebook }}) /dev/null
 
       - name: Output result
         if: always()
@@ -386,13 +330,11 @@ jobs:
 
       - name: Upload results
         if: always()
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4.3.5
         with:
           name: result_run-notebooks-13-research-${{ env.sanitized_name }}.txt
           path: ./artifacts
 
-
-
   generate-status-report:
     runs-on: ubuntu-latest
     needs: [run-notebooks-8-cpp,run-notebooks-8-python,run-notebooks-13-cpp,run-notebooks-13-python,run-notebooks-13-uproot,run-notebooks-13-rdataframe,run-notebooks-research]

diff --git a/.gitignore b/.gitignore
@@ -6,3 +6,4 @@
 *DS_Store
 *__pycache__/
 *.html
+atlas-demo*/
diff --git a/Dockerfile b/Dockerfile
@@ -1,35 +1,17 @@
-FROM rootproject/root:6.32.02-ubuntu22.04
-LABEL maintainer="[email protected]"
-WORKDIR /
-ENV DEBIAN_FRONTEND=noninteractive
-USER root
+FROM continuumio/miniconda3
+LABEL author="ATLAS Open Data Team @ CERN 2024"
+LABEL maintainer="Giovanni Guerrieri - [email protected]"
+WORKDIR /opt/app
 
-#install the prerequisites (option always yes activated)
-RUN apt-get update \
-    && apt-get upgrade -y \
-    && apt-get install -y python3 python3-dev git curl python3-pip \                                           
-    && apt-get --yes install \
-    nano\
-    vim
+# Copy the environment.yml file into the container
+COPY binder/environment.yml .
 
+RUN conda env update -n base -f environment.yml && \
+    conda clean -afy && \
+    rm -rf /opt/conda/pkgs/*
+SHELL ["conda", "run", "-n", "analysis", "/bin/bash", "-c"]
 
-RUN pip3 install --no-cache-dir --upgrade  \
-    uproot3 \
-    uproot \
-    tables \
-    matplotlib \
-    pandas \
-    pydot \
-    awkward \
-    awkward-pandas \
-    vector \
-    scikit-learn \
-    lmfit \
-    jupyter \ 
-    ipykernel \ 
-    papermill \
-    dask[distributed] \
-    coffea
-
-WORKDIR /home/jovyan/work/
+# # Expose port 8888 for Jupyter Lab
+# EXPOSE 8888
 
+# CMD ["jupyter", "lab", "--ip=0.0.0.0", "--allow-root", "--no-browser"]
diff --git a/binder/environment.yml b/binder/environment.yml
@@ -2,17 +2,22 @@ name: analysis
 channels:
     - conda-forge
 dependencies:
-  - root
-  - pip
-  - uproot3
-  - uproot
-  - awkward
-  - pandas
-  - numpy
-  - matplotlib
-  - lmfit
-  - scikit-learn
-  - aiohttp
+  - aiohttp==3.9.5
+  - awkward==2.6.7
+  - coffea==0.7.0
+  - ipykernel==6.29.5
+  - jupyter==1.0.0
+  - lmfit==1.3.2
+  - matplotlib==3.9.1
+  - metakernel==0.30.2
+  - numpy==1.26.4
+  - pandas==2.2.2
+  - papermill==2.6.0
+  - pip==24.2
+  - root_base==6.32.2
+  - scikit-learn==1.5.1
+  - uproot==5.3.10
+  - uproot3==3.14.4
   - pip:
-    - vector
-    - jupyterlab_latex
+    - jupyterlab_latex==3.1.0
+    - vector==1.4.1