[WIP-RFC] Recipes

shipyard-config/jobs.yaml → recipes/devito-gcc-mpich/shipyard-config/jobs.yaml

@@ -19,7 +19,7 @@ job_specifications:
       num_instances: pool_current_dedicated
       mpi:
         runtime: mpich
-        processes_per_node: 2
+        processes_per_node: 1
         options:
           - --bind-to socket
       pre_execution_command: lscpu;

recipes/devito-gcc-mpich/shipyard-config/pool.yaml

@@ -0,0 +1,16 @@
+pool_specification:
+  id: devito-gcc-mpich8
+  vm_configuration:
+    platform_image:
+      offer: UbuntuServer
+      publisher: Canonical
+      sku: 18.04-LTS
+      native: true
+  vm_count:
+    dedicated: 4
+    low_priority: 0
+  resize_timeout: 00:40:00
+  vm_size: Standard_H8
+  inter_node_communication_enabled: true
+  ssh:
+    username: shipyard

recipes/devito-icc-intelmpi/docker/Dockerfile

@@ -0,0 +1,52 @@
+FROM centos:7.1.1503
+RUN rm /bin/sh && ln -s /bin/bash /bin/sh
+
+# set up base and ssh keys
+COPY ssh_config /root/.ssh/config
+
+RUN yum swap -y fakesystemd systemd
+RUN yum -y update \
+    && yum install -y \
+        build-essential ca-certificates wget openssh-client numactl openssh-server curl gcc-c++ \
+    && yum clean all \
+    && mkdir -p /var/run/sshd \
+    && ssh-keygen -A \
+    && sed -i 's/UsePAM yes/UsePAM no/g' /etc/ssh/sshd_config \
+    && sed -i 's/#PermitRootLogin yes/PermitRootLogin yes/g' /etc/ssh/sshd_config \
+    && sed -i 's/#RSAAuthentication yes/RSAAuthentication yes/g' /etc/ssh/sshd_config \
+    && sed -i 's/#PubkeyAuthentication yes/PubkeyAuthentication yes/g' /etc/ssh/sshd_config \
+    && ssh-keygen -f /root/.ssh/id_rsa -t rsa -N '' \
+    && chmod 600 /root/.ssh/config \
+    && chmod 700 /root/.ssh \
+    && cp /root/.ssh/id_rsa.pub /root/.ssh/authorized_keys
+
+ENV DEBIAN_FRONTEND=noninteractive
+
+RUN yum install -y cpio libgtk2.0 xorg openbox
+COPY /parallel_studio_xe_2020_cluster_edition_online/silent.cfg /tmp/icc-config.cfg
+ADD /parallel_studio_xe_2020_cluster_edition_online /tmp/intel
+RUN cd /tmp/intel && \
+   ./install.sh --silent=/tmp/icc-config.cfg --components intel-mpi-psxe__x86_64 && \
+   ./install.sh --silent=/tmp/icc-config.cfg --components intel-icc__x86_64 && \
+   ./install.sh --silent=/tmp/icc-config.cfg --components intel-mpi-sdk__x86_64 && \
+   cd ../../ && \
+   rm -rf /tmp/intel && \
+   rm /tmp/icc-config.cfg
+
+RUN source /opt/intel/compilers_and_libraries/linux/bin/compilervars.sh -arch intel64 -platform linux && icc --help
+RUN source /opt/intel/compilers_and_libraries/linux/mpi/intel64/bin/mpivars.sh && mpirun --version
+
+# RUN ls /opt/intel/compilers_and_libraries/linux/mpi/intel64/bin/
+
+# Utils
+RUN yum install -y htop vim git-core
+
+# Devito
+RUN yum install -y python3 python3-pip python3-devel
+RUN pip3 install --upgrade pip
+RUN git clone https://github.com/devitocodes/devito.git
+RUN source /opt/intel/compilers_and_libraries/linux/mpi/intel64/bin/mpivars.sh && cd devito && pip3 install -e .[extras]
+
+# set up sshd on port 23
+EXPOSE 23
+CMD ["/usr/sbin/sshd", "-D", "-p", "23"]

recipes/devito-icc-intelmpi/docker/ssh_config

@@ -0,0 +1,4 @@
+Host *
+  Port 23
+  StrictHostKeyChecking no
+  UserKnownHostsFile /dev/null

recipes/devito-icc-intelmpi/shipyard-config/config.yaml

@@ -0,0 +1,5 @@
+batch_shipyard:
+  storage_account_settings: mystorageaccount
+global_resources:
+  docker_images:
+  - gbisbas/devitoicc:v0.4

recipes/devito-icc-intelmpi/shipyard-config/jobs.yaml

@@ -0,0 +1,31 @@
+job_specifications:
+- id: bench-icc-intelmp
+  shared_data_volumes:
+  - azureblob_vol
+  tasks:
+  - docker_image: gbisbas/devitoicc:v0.4
+    environment_variables:
+      DEVITO_ARCH: intel
+      OMP_NUM_THREADS: 8
+      DEVITO_MPI: 1
+      DEVITO_OPENMP: 1
+      KMP_AFFINITY: verbose,scatter,granularity=fine
+      DEVITO_LOGGING: 'DEBUG'
+      LC_ALL: 'C.UTF-8'
+      LANG: 'C.UTF-8'
+      I_MPI_DEBUG: '2'
+      I_MPI_FABRICS: 'ofi'
+      I_MPI_PIN: 1
+      I_MPI_PIN_DOMAIN: socket
+      I_MPI_DAPL_PROVIDER: ofa-v2-ib0
+      I_MPI_DYNAMIC_CONNECTION: 0
+      I_MPI_DAPL_TRANSLATION_CACHE: 0
+      FI_PROVIDER: sockets
+    default_working_dir: container
+    multi_instance:
+      num_instances: pool_current_dedicated
+      mpi:
+        runtime: intelmpi
+        processes_per_node: 1
+      pre_execution_command: source /opt/intel/compilers_and_libraries/linux/bin/compilervars.sh -arch intel64 -platform linux; source /opt/intel/compilers_and_libraries/linux/mpi/intel64/bin/mpivars.sh; 
+    command: python3 devito/benchmarks/user/benchmark.py bench -P acoustic --tn 4000 -d 600 600 600 --resultsdir $AZ_BATCH_NODE_SHARED_DIR/results_bench --dse advanced --dle advanced --autotune off -x 1

recipes/devito-icc-intelmpi/shipyard-config/pool.yaml

@@ -0,0 +1,15 @@
+pool_specification:
+  id: devito-icc-intelmp
+  vm_configuration:
+    platform_image:
+      publisher: OpenLogic
+      offer: CentOS-HPC
+      sku: '7.6'
+  vm_count:
+    dedicated: 4
+    low_priority: 0
+  resize_timeout: 00:40:00
+  vm_size: Standard_H16R
+  inter_node_communication_enabled: true
+  ssh:
+    username: shipyard

recipes/devito-icc-mpich/docker/Dockerfile

@@ -0,0 +1,50 @@
+FROM ubuntu:18.04
+
+# set up base and ssh keys
+COPY ssh_config /root/.ssh/config
+
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends \
+        build-essential ca-certificates wget openssh-client openssh-server \
+        mpich libmpich-dev \
+    && mkdir -p /var/run/sshd \
+    && ssh-keygen -A \
+    && sed -i 's/UsePAM yes/UsePAM no/g' /etc/ssh/sshd_config \
+    && sed -i 's/#PermitRootLogin yes/PermitRootLogin yes/g' /etc/ssh/sshd_config \
+    && sed -i 's/#RSAAuthentication yes/RSAAuthentication yes/g' /etc/ssh/sshd_config \
+    && sed -i 's/#PubkeyAuthentication yes/PubkeyAuthentication yes/g' /etc/ssh/sshd_config \
+    && ssh-keygen -f /root/.ssh/id_rsa -t rsa -N '' \
+    && chmod 600 /root/.ssh/config \
+    && chmod 700 /root/.ssh \
+    && cp /root/.ssh/id_rsa.pub /root/.ssh/authorized_keys
+
+ENV DEBIAN_FRONTEND=noninteractive
+RUN apt-get install -y cpio libgtk2.0 xorg openbox
+
+COPY /parallel_studio_xe_2020_cluster_edition_online/silent.cfg /tmp/icc-config.cfg
+ADD /parallel_studio_xe_2020_cluster_edition_online /tmp/intel
+RUN cd /tmp/intel && \
+  ./install.sh --silent=/tmp/icc-config.cfg --components intel-mpi-psxe__x86_64 && \
+  ./install.sh --silent=/tmp/icc-config.cfg --components intel-icc__x86_64 && \
+  ./install.sh --silent=/tmp/icc-config.cfg --components intel-mpi-sdk__x86_64 && \
+  ./install.sh --silent=/tmp/icc-config.cfg --components intel-mpi-rt__x86_64 && \
+  cd ../../ && \
+  rm -rf /tmp/intel && \
+  rm /tmp/icc-config.cfg
+
+RUN ls opt/intel/compilers_and_libraries/linux/
+RUN /bin/bash -c "source /opt/intel/compilers_and_libraries/linux/bin/compilervars.sh -arch intel64 -platform linux"
+RUN /bin/bash -c "source /opt/intel/compilers_and_libraries/linux/mpi/intel64/bin/mpivars.sh"
+
+# Utils
+RUN apt-get install -y htop vim git-core
+
+# Devito
+RUN apt-get install -y python3 python3-pip
+RUN pip3 install --upgrade pip
+RUN git clone -b azure-make https://github.com/devitocodes/devito.git
+RUN cd devito && pip3 install -e .[extras]
+
+# set up sshd on port 23
+EXPOSE 23
+CMD ["/usr/sbin/sshd", "-D", "-p", "23"]

recipes/devito-icc-mpich/docker/ssh_config

@@ -0,0 +1,4 @@
+Host *
+  Port 23
+  StrictHostKeyChecking no
+  UserKnownHostsFile /dev/null

recipes/devito-icc-mpich/shipyard-config/config.yaml

@@ -0,0 +1,13 @@
+batch_shipyard:
+  storage_account_settings: mystorageaccount
+global_resources:
+  docker_images:
+  - gbisbas/devitoicc:v0.3
+  volumes:
+    shared_data_volumes:
+      azureblob_vol:
+        volume_driver: azureblob
+        storage_account_settings: mystorageaccount
+        azure_blob_container_name: devito-bench
+        container_path: $AZ_BATCH_NODE_SHARED_DIR/bench-results
+        bind_options: rw

recipes/devito-icc-mpich/shipyard-config/jobs.yaml

@@ -0,0 +1,26 @@
+job_specifications:
+- id: bench-icc-mpich
+  shared_data_volumes:
+  - azureblob_vol
+  tasks:
+  - docker_image: gbisbas/devitoicc:v0.3
+    environment_variables:
+      DEVITO_ARCH: intel
+      OMP_NUM_THREADS: 8
+      OMP_PROC_BIND: 'close'
+      DEVITO_MPI: 1
+      DEVITO_OPENMP: 1
+      OMP_PLACES: 'cores'
+      DEVITO_LOGGING: 'DEBUG'
+      LC_ALL: 'C.UTF-8'
+      LANG: 'C.UTF-8'
+    default_working_dir: container
+    multi_instance:
+      num_instances: pool_current_dedicated
+      mpi:
+        runtime: mpich
+        processes_per_node: 2
+        options:
+          - --bind-to socket
+      pre_execution_command: source /opt/intel/compilers_and_libraries/linux/bin/compilervars.sh -arch intel64 -platform linux; source /opt/intel/compilers_and_libraries/linux/mpi/intel64/bin/mpivars.sh; pip3 install mpi4py; 
+    command: python3 devito/benchmarks/user/benchmark.py bench -P acoustic --tn 4000 -d 600 600 600 --resultsdir $AZ_BATCH_NODE_SHARED_DIR/results_bench --dse advanced --dle advanced --autotune off -x 1

shipyard-config/pool.yaml → recipes/devito-icc-mpich/shipyard-config/pool.yaml

@@ -1,5 +1,5 @@
 pool_specification:
-  id: devito-gcc-mpich
+  id: devito-icc-mpich
   vm_configuration:
     platform_image:
       offer: UbuntuServer