Merge pull request pyne#1498 from pyne/virtual-machine

bquan0 · Sep 1, 2023 · 8c17fa9 · 8c17fa9
2 parents 8a9ef72 + 8831e09
commit 8c17fa9
Show file tree

Hide file tree

Showing 4 changed files with 179 additions and 49 deletions.
diff --git a/.github/workflows/docker_publish.yml b/.github/workflows/docker_publish.yml
@@ -16,17 +16,22 @@ on:
 
 env:
   DOCKER_IMAGE_BASENAME: ghcr.io/${{ github.repository_owner }}/pyne_ubuntu_22.04_py3
-  DOCKER_IMAGE_TAG: :refs_heads_${{ github.ref_name }}-bk0
+  USE_LATEST_TAG: false
 
 jobs:
-
   # builds and pushes docker images of various stages to ghcr.
   # These docker images are also stored in ghcr and can be pulled
   # to be built upon by the subsequent stage.
   multistage_image_build:
     runs-on: ubuntu-latest
-    outputs:
-      image_tag: ${{ steps.output_tag.outputs.image_tag }}
+    strategy:
+      matrix: 
+        hdf5: ['']
+        hdf5_build_arg: ['NO']
+        include:
+          - hdf5: _hdf5
+            hdf5_build_arg: hdf5-1_12_0
+      fail-fast: false
 
     steps:
       - name: Checkout repository
@@ -38,66 +43,74 @@ jobs:
           registry: ghcr.io
           username: ${{ github.repository_owner }}
           password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Tag images with latest if on the main repo's develop branch
+        if: github.repository_owner == 'pyne' && github.ref_name == 'develop'
+        run: echo "USE_LATEST_TAG=true" >> $GITHUB_ENV
 
       # build base python, moab, dagmc, openmc using multistage docker build action
       - uses: firehed/multistage-docker-build-action@v1
         id: build_all_stages
         with:
-          repository: ${{ env.DOCKER_IMAGE_BASENAME }}
+          repository: ${{ env.DOCKER_IMAGE_BASENAME }}${{ matrix.hdf5 }}
           stages: base_python, moab, dagmc
           server-stage: openmc
           quiet: false
-          tag-latest-on-default: false
-          dockerfile: docker/ubuntu_22.04-dev.dockerfile
-
-      # build HDF5 using multistage docker build action 
-      - uses: firehed/multistage-docker-build-action@v1
-        id: build_dagmc_with_hdf5
-        with:
-          repository: ${{ env.DOCKER_IMAGE_BASENAME }}_hdf5
-          stages: base_python, moab
-          server-stage: dagmc
-          quiet: false
-          tag-latest-on-default: false
+          parallel: true
+          tag-latest-on-default: ${{ env.USE_LATEST_TAG }}
           dockerfile: docker/ubuntu_22.04-dev.dockerfile
-          build-args: build_hdf5=hdf5-1_12_0
-
-      - id: output_tag
-        run: |
-          echo "image_tag=$DOCKER_IMAGE_TAG" >> $GITHUB_OUTPUT
+          build-args: build_hdf5=${{ matrix.hdf5_build_arg }}, pyne_test_base=openmc
 
   # Downloads the images uploaded to ghcr in previous stages and runs pyne
   # tests to check that they work.
   BuildTest:
     needs: [multistage_image_build]
     runs-on: ubuntu-latest
-
+    
     strategy:
-      matrix:
-        stage: [base_python, moab, dagmc, openmc]
+      matrix: 
+        pyne_test_base: [base_python, moab, dagmc, openmc]
         hdf5: ['']
+        hdf5_build_arg: ['NO']
         include:
-          - stage: dagmc
+          - pyne_test_base: dagmc
             hdf5: _hdf5
+            hdf5_build_arg: hdf5-1_12_0
       fail-fast: false
 
-    container:
-      image: ghcr.io/${{ github.repository_owner }}/pyne_ubuntu_22.04_py3${{ matrix.hdf5 }}/${{ matrix.stage }}${{ needs.multistage_image_build.outputs.image_tag }}
     steps:
       - name: Checkout repository
         uses: actions/checkout@v3
 
-      - name: use BuildTest composite action
-        uses: ./.github/actions/build-test
+      - name: Log in to the Container registry
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Tag images with latest if on the main repo's develop branch
+        if: github.repository_owner == 'pyne' && github.ref_name == 'develop'
+        run: echo "USE_LATEST_TAG=true" >> $GITHUB_ENV
+
+      # build test stage and pyne-dev stage using multistage docker build action
+      - uses: firehed/multistage-docker-build-action@v1
+        id: multistage_build_and_test
         with:
-          stage: ${{ matrix.stage }}
-          hdf5: ${{ matrix.hdf5 }}
+          repository: ${{ env.DOCKER_IMAGE_BASENAME }}${{ matrix.hdf5 }}
+          stages: ${{ matrix.pyne_test_base }}
+          server-stage: pyne-dev
+          quiet: false
+          parallel: true
+          tag-latest-on-default: ${{ env.USE_LATEST_TAG }}
+          dockerfile: docker/ubuntu_22.04-dev.dockerfile
+          build-args: build_hdf5=${{ matrix.hdf5_build_arg }}, pyne_test_base=${{ matrix.pyne_test_base }}
 
   # if the previous step that tests the docker images passes then the images
   # can be copied from the ghcr where they are saved using :ci_testing tags to
   # :latest and :stable tags.
   pushing_test_stable_img:
-    if: ${{ github.repository_owner == 'pyne' }}
+    if: github.repository_owner == 'pyne' && github.ref_name == 'develop'
     needs: [BuildTest]
     runs-on: ubuntu-latest
     strategy:
@@ -121,11 +134,5 @@ jobs:
       - name: Push Image to stable img
         uses: akhilerm/[email protected]
         with:
-          src: ${{ env.DOCKER_IMAGE_BASENAME }}${{ matrix.hdf5 }}/${{ matrix.stage }}${{ env.DOCKER_IMAGE_TAG }}
+          src: ${{ env.DOCKER_IMAGE_BASENAME }}${{ matrix.hdf5 }}/${{ matrix.stage }}:latest
           dst: ${{ env.DOCKER_IMAGE_BASENAME }}${{ matrix.hdf5 }}/${{ matrix.stage }}:stable
-
-      - name: Push Image to latest img
-        uses: akhilerm/[email protected]
-        with:
-          src: ${{ env.DOCKER_IMAGE_BASENAME }}${{ matrix.hdf5 }}/${{ matrix.stage }}${{ env.DOCKER_IMAGE_TAG }}
-          dst: ${{ env.DOCKER_IMAGE_BASENAME }}${{ matrix.hdf5 }}/${{ matrix.stage }}:latest
diff --git a/.github/workflows/virtualbox_image.yml b/.github/workflows/virtualbox_image.yml
@@ -0,0 +1,101 @@
+name: Create VirtualBox disk image from Dockerfile
+
+on:
+  # allows us to run workflows manually
+  workflow_dispatch:
+  release:
+    types:
+      - created
+      - edited
+
+env:
+  VM_PASSWORD: temppwd
+  DOCKER_IMAGE_BASENAME: ghcr.io/${{ github.repository_owner }}/pyne_ubuntu_22.04_py3
+
+
+jobs:
+  pyne_image_build:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      - name: Log in to the Container registry
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      # before merging, change tag-latest-on-default to true
+      - name: Build PyNE docker image
+        uses: firehed/multistage-docker-build-action@v1
+        id: build_pyne
+        with:
+          repository: ${{ env.DOCKER_IMAGE_BASENAME }}
+          stages: openmc
+          server-stage: pyne
+          quiet: false
+          tag-latest-on-default: true
+          dockerfile: docker/ubuntu_22.04-dev.dockerfile
+
+  pushing_test_stable_img:
+    needs: [pyne_image_build]
+    runs-on: ubuntu-latest
+    steps:
+      - name: Log in to the Container registry
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Push Image to stable img
+        uses: akhilerm/[email protected]
+        with:
+          src: ${{ env.DOCKER_IMAGE_BASENAME }}/pyne:latest
+          dst: ${{ env.DOCKER_IMAGE_BASENAME }}/pyne:stable
+
+      - name: Push Image to release tag img
+        uses: akhilerm/[email protected]
+        with:
+          src: ${{ env.DOCKER_IMAGE_BASENAME }}/pyne:latest
+          dst: ${{ env.DOCKER_IMAGE_BASENAME }}/pyne:${{ github.ref_name }}
+
+  virtualbox_image_build:
+    needs: [pushing_test_stable_img]
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      - name: Log in to the Container registry
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build VirtualBox image using d2vm
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y --fix-missing \
+            util-linux \
+            udev \
+            parted \
+            e2fsprogs \
+            mount \
+            tar \
+            extlinux \
+            qemu-utils            
+          curl -sL "https://github.com/linka-cloud/d2vm/releases/download/v0.2.0/d2vm_v0.2.0_linux_amd64.tar.gz" | tar -xvz d2vm
+          sudo mv d2vm /usr/local/bin/
+          sudo d2vm convert ghcr.io/${{ github.repository_owner  }}/pyne_ubuntu_22.04_py3/pyne:stable_${{ github.ref_name }} -p ${{ env.VM_PASSWORD }} -o pyne_${{ github.ref_name }}.vdi
+
+      - name: Upload VirtualBox image as artifact
+        uses: actions/upload-artifact@v3
+        with:
+          name: virtualbox_image
+          path: pyne_${{ github.ref_name }}.vdi
+
diff --git a/CHANGELOG.rst b/CHANGELOG.rst
@@ -24,8 +24,7 @@ Next Version
    * Add composite action for BuildTest job in workflows (#1489)
    * Update run conditions for workflows (#1494)
    * Fix gamma intensity calculation (#1496)
-
-
+   * Add workflow to build virtual machines based on Dockerfile (#1490)
 
 v0.7.7
 ======

diff --git a/docker/ubuntu_22.04-dev.dockerfile b/docker/ubuntu_22.04-dev.dockerfile
@@ -1,4 +1,5 @@
 ARG build_hdf5="NO"
+ARG pyne_test_base=openmc
 ARG ubuntu_version=22.04
 
 FROM ubuntu:${ubuntu_version} AS base_python
@@ -98,6 +99,7 @@ RUN export MOAB_HDF5_ARGS=""; \
 # put MOAB on the path
 ENV LD_LIBRARY_PATH $HOME/opt/moab/lib:$LD_LIBRARY_PATH
 ENV LIBRARY_PATH $HOME/opt/moab/lib:$LIBRARY_PATH
+ENV PYNE_MOAB_ARGS "--moab $HOME/opt/moab"
 
 FROM moab AS dagmc
 # build/install DAGMC
@@ -119,6 +121,7 @@ RUN cd /root \
     && make install \
     && cd ../.. \
     && rm -rf DAGMC
+ENV PYNE_DAGMC_ARGS "--dagmc $HOME/opt/dagmc"
 
 FROM dagmc AS openmc
 ARG build_hdf5
@@ -131,8 +134,8 @@ RUN if [ "$build_hdf5" != "NO" ]; then \
     && git checkout tags/v0.13.0 \
     && pip install .
 
-# Build/Install PyNE
-FROM openmc AS pyne
+# Build/Install PyNE from develop branch
+FROM ${pyne_test_base} AS pyne-dev
 ARG build_hdf5
 
 RUN export PYNE_HDF5_ARGS="" ;\
@@ -143,11 +146,31 @@ RUN export PYNE_HDF5_ARGS="" ;\
     && git clone -b develop --single-branch https://github.com/pyne/pyne.git \
     && cd pyne \
     && python setup.py install --user \
-                                --moab $HOME/opt/moab --dagmc $HOME/opt/dagmc \
+                                $PYNE_MOAB_ARGS $PYNE_DAGMC_ARGS \
                                 $PYNE_HDF5_ARGS \
                                 --clean -j 3;
 ENV PATH $HOME/.local/bin:$PATH
-RUN if [ "$build_pyne" = "YES" ]; then \
-        cd $HOME \
-        && nuc_data_make ; \
-    fi
+RUN cd $HOME \
+    && nuc_data_make \
+    && cd $HOME/opt/pyne/tests \
+    && ./ci-run-tests.sh python3
+
+# Build/Install PyNE from release branch
+FROM ${pyne_test_base} AS pyne
+ARG build_hdf5
+
+RUN export PYNE_HDF5_ARGS="" ;\
+    if [ "$build_hdf5" != "NO" ]; then \
+            export PYNE_HDF5_ARGS="--hdf5 $HDF5_INSTALL_PATH" ; \
+    fi;
+COPY . $HOME/opt/pyne
+RUN cd $HOME/opt/pyne \
+    && python setup.py install --user \
+                                $PYNE_MOAB_ARGS $PYNE_DAGMC_ARGS \
+                                $PYNE_HDF5_ARGS \
+                                --clean -j 3;
+ENV PATH $HOME/.local/bin:$PATH
+RUN cd $HOME \
+    && nuc_data_make \
+    && cd $HOME/opt/pyne/tests \
+    && ./ci-run-tests.sh python3