update

fwang2 · Aug 21, 2024 · dc7542d · dc7542d
1 parent db92fe4
commit dc7542d
Show file tree

Hide file tree

Showing 127 changed files with 74 additions and 270 deletions.
diff --git a/.gitignore b/.gitignore
diff --git a/Gemfile b/Gemfile
diff --git a/README.md b/README.md
diff --git a/_config.yml b/_config.yml
diff --git a/_includes/head-custom-google-analytics.html b/_includes/head-custom-google-analytics.html
diff --git a/_includes/head-custom.html b/_includes/head-custom.html
diff --git a/_layouts/default.html b/_layouts/default.html
diff --git a/_sass/cayman.scss b/_sass/cayman.scss
diff --git a/_sass/jekyll-theme-cayman.scss b/_sass/jekyll-theme-cayman.scss
diff --git a/_sass/normalize.scss b/_sass/normalize.scss
diff --git a/_sass/rouge-github.scss b/_sass/rouge-github.scss
diff --git a/_sass/variables.scss b/_sass/variables.scss
diff --git a/_site/assets/css/style.css b/_site/assets/css/style.css
diff --git a/_site/assets/css/style.css.map b/_site/assets/css/style.css.map
diff --git a/_site/images/frontier.jpg b/_site/images/frontier.jpg
diff --git a/_site/images/ornl.jpg b/_site/images/ornl.jpg
diff --git a/_site/index.html b/_site/index.html
diff --git a/_site/jekyll-theme-cayman.gemspec b/_site/jekyll-theme-cayman.gemspec
diff --git a/_site/sc22-FL b/_site/sc22-FL
diff --git a/_site/script/bootstrap b/_site/script/bootstrap
diff --git a/_site/script/cibuild b/_site/script/cibuild
diff --git a/_site/script/release b/_site/script/release
diff --git a/_site/script/server b/_site/script/server
diff --git a/_site/script/validate-html b/_site/script/validate-html
diff --git a/andes/Andes.md b/andes/Andes.md
diff --git a/assets/css/style.scss b/assets/css/style.scss
diff --git a/dl-notebooks/NLP-rnn.ipynb b/dl-notebooks/NLP-rnn.ipynb
diff --git a/dl-notebooks/NLP-tweet.ipynb b/dl-notebooks/NLP-tweet.ipynb
diff --git a/dl-notebooks/NLP-word-embeddings.ipynb b/dl-notebooks/NLP-word-embeddings.ipynb
diff --git a/dl-notebooks/NLP-word2vec.ipynb b/dl-notebooks/NLP-word2vec.ipynb
diff --git a/dl-notebooks/ml-PCA.ipynb b/dl-notebooks/ml-PCA.ipynb
diff --git a/dl-notebooks/ml-SVM.ipynb b/dl-notebooks/ml-SVM.ipynb
diff --git a/dl-notebooks/ml-activation.ipynb b/dl-notebooks/ml-activation.ipynb
diff --git a/dl-notebooks/ml-autograd.ipynb b/dl-notebooks/ml-autograd.ipynb
diff --git a/dl-notebooks/ml-basics.ipynb b/dl-notebooks/ml-basics.ipynb
diff --git a/dl-notebooks/ml-bayesian.ipynb b/dl-notebooks/ml-bayesian.ipynb
diff --git a/dl-notebooks/ml-chain-rule.ipynb b/dl-notebooks/ml-chain-rule.ipynb
diff --git a/dl-notebooks/ml-cnn.ipynb b/dl-notebooks/ml-cnn.ipynb
diff --git a/dl-notebooks/ml-gd_1.ipynb b/dl-notebooks/ml-gd_1.ipynb
diff --git a/dl-notebooks/ml-gd_2.ipynb b/dl-notebooks/ml-gd_2.ipynb
diff --git a/dl-notebooks/ml-gd_3.ipynb b/dl-notebooks/ml-gd_3.ipynb
diff --git a/dl-notebooks/ml-gradient-descent.ipynb b/dl-notebooks/ml-gradient-descent.ipynb
diff --git a/dl-notebooks/ml-kmeans.ipynb b/dl-notebooks/ml-kmeans.ipynb
diff --git a/dl-notebooks/ml-linear-regression.ipynb b/dl-notebooks/ml-linear-regression.ipynb
diff --git a/dl-notebooks/ml-logistic.ipynb b/dl-notebooks/ml-logistic.ipynb
diff --git a/dl-notebooks/ml-neural-network.ipynb b/dl-notebooks/ml-neural-network.ipynb
diff --git a/dl-notebooks/ml-normal-equation.ipynb b/dl-notebooks/ml-normal-equation.ipynb
diff --git a/dl-notebooks/ml-propgation.ipynb b/dl-notebooks/ml-propgation.ipynb
diff --git a/dl-notebooks/ml-regularization.ipynb b/dl-notebooks/ml-regularization.ipynb
diff --git a/dl-notebooks/ml-softmax.ipynb b/dl-notebooks/ml-softmax.ipynb
diff --git a/dl-notebooks/ml-transfer-learning.ipynb b/dl-notebooks/ml-transfer-learning.ipynb
diff --git a/dl-notebooks/pytorch-Autograd.ipynb b/dl-notebooks/pytorch-Autograd.ipynb
diff --git a/dl-notebooks/pytorch-Tensors.ipynb b/dl-notebooks/pytorch-Tensors.ipynb
diff --git a/dl-notebooks/pytorch-basics.ipynb b/dl-notebooks/pytorch-basics.ipynb
diff --git a/dl-notebooks/pytorch-vision.ipynb b/dl-notebooks/pytorch-vision.ipynb
diff --git a/dl-ppts/DL_Pytorch.pptx b/dl-ppts/DL_Pytorch.pptx
diff --git a/dl-ppts/classification/knn.ipynb b/dl-ppts/classification/knn.ipynb
diff --git a/dl-ppts/clustering/DBSCAN.ipynb b/dl-ppts/clustering/DBSCAN.ipynb
diff --git a/dl-ppts/clustering/hierarchical_clustering.ipynb b/dl-ppts/clustering/hierarchical_clustering.ipynb
diff --git a/dl-ppts/clustering/kmeans.ipynb b/dl-ppts/clustering/kmeans.ipynb
diff --git a/dl-ppts/jupyterlab-on-summit/README.md b/dl-ppts/jupyterlab-on-summit/README.md
diff --git a/dl-ppts/jupyterlab-on-summit/batch.lsf b/dl-ppts/jupyterlab-on-summit/batch.lsf
diff --git a/dl-ppts/jupyterlab-on-summit/setup_tfconfig.py b/dl-ppts/jupyterlab-on-summit/setup_tfconfig.py
diff --git a/dl-ppts/jupyterlab-on-summit/start-jupyter.sh b/dl-ppts/jupyterlab-on-summit/start-jupyter.sh
diff --git a/dl-ppts/jupyterlab-on-summit/stop-jupyter.sh b/dl-ppts/jupyterlab-on-summit/stop-jupyter.sh
diff --git a/dl-ppts/jupyterlab-on-summit/summit-multi-worker-example.ipynb b/dl-ppts/jupyterlab-on-summit/summit-multi-worker-example.ipynb
diff --git a/dl-ppts/jupyterlab-on-summit/summit-single-worker-example.ipynb b/dl-ppts/jupyterlab-on-summit/summit-single-worker-example.ipynb
diff --git a/dl-ppts/pca/SVD.ipynb b/dl-ppts/pca/SVD.ipynb
diff --git a/dl-ppts/regression/Linear-Regression.ipynb b/dl-ppts/regression/Linear-Regression.ipynb
diff --git a/dl-ppts/regression/logistic-pytorch.ipynb b/dl-ppts/regression/logistic-pytorch.ipynb
diff --git a/dl-ppts/regression/regression-pytorch.ipynb b/dl-ppts/regression/regression-pytorch.ipynb
diff --git a/figures/3neurons.png b/figures/3neurons.png
diff --git a/figures/3neurons2.png b/figures/3neurons2.png
diff --git a/figures/bp-compute-graph.png b/figures/bp-compute-graph.png
diff --git a/figures/bp-steps.png b/figures/bp-steps.png
diff --git a/figures/chain-rule1.png b/figures/chain-rule1.png
diff --git a/figures/chain-rule2.png b/figures/chain-rule2.png
diff --git a/figures/conjugate-transpose.png b/figures/conjugate-transpose.png
diff --git a/figures/cross-entropy.png b/figures/cross-entropy.png
diff --git a/figures/cross-product.png b/figures/cross-product.png
diff --git a/figures/deep_wide1.png b/figures/deep_wide1.png
diff --git a/figures/deep_wide2.png b/figures/deep_wide2.png
diff --git a/figures/example1.png b/figures/example1.png
diff --git a/figures/gd-linear_ab.png b/figures/gd-linear_ab.png
diff --git a/figures/linear-2x-2features.pdf b/figures/linear-2x-2features.pdf
diff --git a/figures/linear-2x-5-degree1.pdf b/figures/linear-2x-5-degree1.pdf
diff --git a/figures/linear-2x-5-degree10.pdf b/figures/linear-2x-5-degree10.pdf
diff --git a/figures/linear-2x-5.pdf b/figures/linear-2x-5.pdf
diff --git a/figures/linear-2x-cost-func.pdf b/figures/linear-2x-cost-func.pdf
diff --git a/figures/linear-2x-costfunc-mode-fit.pdf b/figures/linear-2x-costfunc-mode-fit.pdf
diff --git a/figures/linear-2x.pdf b/figures/linear-2x.pdf
diff --git a/figures/linear-2xplus5.pdf b/figures/linear-2xplus5.pdf
diff --git a/figures/linear-cost-contour.pdf b/figures/linear-cost-contour.pdf
diff --git a/figures/linear-cost-gd.pdf b/figures/linear-cost-gd.pdf
diff --git a/figures/linear-regression-gd.pdf b/figures/linear-regression-gd.pdf
diff --git a/figures/linear.pdf b/figures/linear.pdf
diff --git a/figures/linear1.png b/figures/linear1.png
diff --git a/figures/linear2.png b/figures/linear2.png
diff --git a/figures/logistic_vs_linear.png b/figures/logistic_vs_linear.png
diff --git a/figures/manual_derive.png b/figures/manual_derive.png
diff --git a/figures/neuron.png b/figures/neuron.png
diff --git a/figures/nn_3.drawio b/figures/nn_3.drawio
diff --git a/figures/sigmoid.pdf b/figures/sigmoid.pdf
diff --git a/frontier/Frontier-cmake.md b/frontier/Frontier-cmake.md
diff --git a/frontier/crusher-deepspeed.md b/frontier/crusher-deepspeed.md
diff --git a/frontier/crusher-pytorch.md b/frontier/crusher-pytorch.md
diff --git a/frontier/frontier_pytorch.md b/frontier/frontier_pytorch.md
@@ -1,112 +1,112 @@
 
 
-- [Build from source](#build-from-source)
-  - [prep Frontier modules](#prep-frontier-modules)
-  - [module output on Frontier](#module-output-on-frontier)
-  - [setup miniconda3](#setup-miniconda3)
-  - [build pytorch](#build-pytorch)
-    - [Build options: see `setup.py`](#build-options-see-setuppy)
-    - [regenerate CMAKE build files](#regenerate-cmake-build-files)
-    - [Kineto and roctracer.h problem](#kineto-and-roctracerh-problem)
-  - [Build DeepSpeed](#build-deepspeed)
-  - [Install GPTNeoX](#install-gptneox)
+## On rocm/pytorch compatibility
 
-# Build from source
+![alt text](frontier_rocmTorch_table.png)
 
-## prep Frontier modules
+
+## Basic environment setup
 
 ```
-module load PrgEnv-gnu
-module load gcc/10.3.0
-module load rocm/5.1.0
-module load craype-x86-trento
+ROCM_VER=6.0.0
+module load PrgEnv-gnu/8.5.0
+module load rocm/6.0.0
+module load gcc-native/12.3
+module load craype-accel-amd-gfx90a
+module load cmake
+module load miniforge3/23.11.0-0
+module unload darshan-runtime
 export HCC_AMDGPU_TARGET=gfx90a
 export PYTORCH_ROCM_ARCH=gfx90a
-export ROCM_SOURCE_DIR=/opt/rocm-5.1.0
-export CRAY_CPU_TARGET=x86_64 # just to remove warning noise
+export ROCM_HOME=/opt/rocm-${ROCM_VER}
+export CC=cc
+export CXX=CC
 ```
-## module output on Frontier
 
-```
-Currently Loaded Modules:
-  1) libfabric/1.15.2.0   4) cray-dsmml/0.2.2        7) gcc/10.3.0             10) DefApps/default    13) craype-accel-amd-gfx90a
-  2) craype-network-ofi   5) cray-libsci/22.12.1.1   8) darshan-runtime/3.4.0  11) cray-mpich/8.1.23  14) craype-x86-trento
-  3) craype/2.7.19        6) PrgEnv-gnu/8.3.3        9) hsi/default            12) rocm/5.1.0
-```
 
-Note: One of the module between `craype-x86-trento` and `craype-accel-amd-gfx90a` fixed a linking problem. My guess is the former. 
+## conda and mpi4py setup
 
-## setup miniconda3
-
-```
-wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh
-bash ./Miniconda3-latest-Linux-x86_64.sh -b -p miniconda
-conda create -n pytorch python=3.8
-conda active pytorch
-pip install -r requirements.txt
 ```
+conda create -p /sw/aaims/frontier/rocm600-pt230 \
+    python=3.10 numpy=1 -c conda-forge
 
-## build pytorch
+source activate /sw/aaims/frontier/rocm600-pt230
+pip install pyyaml typing_extensions ninja packaging
 
-```
-git clone --recursive -b IFU-master-2022-11-22 https://github.com/ROCmSoftwarePlatform/pytorch
-python tools/amd_build/build_amd.py
-USE_KINETO=0 USE_ROCM=1 MAX_JOBS=4 python setup.py bdist_wheel 2>&1 | tee output
+# Optional: install mpi4py
+MPICC="cc -shared" pip install --no-cache-dir --no-binary=mpi4py mpi4py
 ```
 
-### Build options: see `setup.py`
+## Build pytorch
 
 ```
-USE_KINETO=0 # disable profiler, ask for roctracer.h
-```
-### regenerate CMAKE build files
+TORCH_VER=release/2.3-frontier
+git clone --recursive -b ${TORCH_VER} \
+    https://github.com/michael-sandoval/pytorch
 
-This will trigger a rebuild for the changed configuration.
+cd pytorch
 
-```
-cd pytorch/build
-rm CMakeCache.txt
-```
-To remove previous build as well:
+# redundant
+#git submodule init
+#git submodule update
 
-```
-python setup.py clean
-```
-### Kineto and roctracer.h problem
+ 
+# If using GCC12 to build torch:
+export CFLAGS=" -Wno-error=maybe-uninitialized -Wno-error=uninitialized -Wno-error=restrict -Wno-error=nonnull"
+export CXXFLAGS=" -Wno-error=maybe-uninitialized -Wno-error=uninitialized -Wno-error=restrict -Wno-error=nonnull"
+export BUILD_TEST=OFF
 
-Kineto requires roctracer, which fails in rocm 5.1.0
 
-```
-if (NOT ROCM_SOURCE_DIR)
-    set(ROCM_SOURCE_DIR "$ENV{ROCM_SOURCE_DIR}")
-    message(INFO " ROCM_SOURCE_DIR = ${ROCM_SOURCE_DIR}")
-endif()
-```
+# Generate HIP files
+python3 tools/amd_build/build_amd.py
 
-For reason unknown at this point, the `ROCM_SOURCE_DIR` is still set as `/opt/rocm` instead of `/opt/rocm-5.1.0` even though the environment variable is set.
 
-So the easy workaround is:
+# Set the PyTorch build version
+export PYTORCH_BUILD_VERSION="2.3.0"
+export PYTORCH_BUILD_NUMBER=1
 
-```
-set(ROCM_SOURCE_DIR /opt/rocm-5.1.0)
+ 
+# Point libkineto away from "/opt/rocm" to "/opt/rocm-x.y.z"
+cp ./frontier_fixes/third_party/kineto/libkineto/CMakeLists.txt \
+    ./third_party/kineto/libkineto/CMakeLists.txt
+
+# Fix "rocm-core/rocm_version.h" reference post HIP conversion (ONLY WHEN USING ROCm < 6.0.0)
+<!-- cp ./frontier_fixes/aten/src/ATen/hip/tunable/TunableGemm.h \
+    ./aten/src/ATen/hip/tunable/TunableGemm.h -->
+
+# Build PyTorch
+USE_ROCM=1 USE_CUDA=OFF USE_NVCC=OFF BUILD_CAFFE2_OPS=0 ROCM_SOURCE_DIR="/opt/rocm-${ROCM_VER}" MAX_JOBS=8 python setup.py bdist_wheel
+ 
 ```
 
-## Build DeepSpeed
+After compilation, I have:
 
 ```
-git clone https://github.com/microsoft/DeepSpeed
-DS_BUILD_FUSED_LAMB=1 DS_BUILD_FUSED_ADAM=1 DS_BUILD_TRANSFORMER=1 DS_BUILD_STOCHASTIC_TRANSFORMER=1  DS_BUILD_UTILS=1 python setup.py bdist_wheel
-python setup.py install
+dist/torch-2.3.0-cp310-cp310-linux_x86_64.whl
 ```
 
-## Install GPTNeoX
+## Verify
 
 ```
-pip install shortuuid # missed from
-git clone https://github.com/EleutherAI/gpt-neox.git
-pip install -r requirements/requirements.txt
-pip install -r requirements/requirements-wandb.txt
-pip install -r requirements/requirements-tensorboard.txt
+pip install dist/*.whl
+
 ```
 
+```
+>>> import torch
+Traceback (most recent call last):
+  File "<stdin>", line 1, in <module>
+  File "/lustre/orion/stf218/proj-shared/fwang2/pytorch/torch/__init__.py", line 546, in <module>
+    raise ImportError(textwrap.dedent('''
+ImportError: Failed to load PyTorch C extensions:
+    It appears that PyTorch has loaded the `torch/_C` folder
+    of the PyTorch repository rather than the C extensions which
+    are expected in the `torch._C` namespace. This can occur when
+    using the `install` workflow. e.g.
+        $ python setup.py install && python -c "import torch"
+
+    This error can generally be solved using the `develop` workflow
+        $ python setup.py develop && python -c "import torch"  # This should succeed
+    or by running Python from a different directory.
+```
 
diff --git a/frontier/frontier_rocmTorch_table.png b/frontier/frontier_rocmTorch_table.png
diff --git a/frontier/rocm551-py310/conda_env.sh b/frontier/rocm551-py310/conda_env.sh
diff --git a/frontier/rocm551-py310/conda_env_old_pkgs.sh b/frontier/rocm551-py310/conda_env_old_pkgs.sh
diff --git a/frontier/rocm551-py310/mpi4py-3.1.3-cp310-cp310-linux_x86_64.whl b/frontier/rocm551-py310/mpi4py-3.1.3-cp310-cp310-linux_x86_64.whl
diff --git a/images/frontier.jpg b/images/frontier.jpg
diff --git a/images/ornl.jpg b/images/ornl.jpg
diff --git a/metrics.ipynb b/metrics.ipynb
diff --git a/nlp-model-scaling.ipynb b/nlp-model-scaling.ipynb
diff --git a/preface.py b/preface.py
diff --git a/sfmono.py b/sfmono.py
diff --git a/summit/JupyterOnSummit.md b/summit/JupyterOnSummit.md
diff --git a/summit/Summit-deepspeed.md b/summit/Summit-deepspeed.md
diff --git a/summit/Summit.md b/summit/Summit.md
diff --git a/summit/olcf-jupyterhub.md b/summit/olcf-jupyterhub.md
diff --git a/tools/data_loader.ipynb b/tools/data_loader.ipynb
diff --git a/tools/linear.ipynb b/tools/linear.ipynb
diff --git a/tools/matplotlib.ipynb b/tools/matplotlib.ipynb
diff --git a/tools/numpy.ipynb b/tools/numpy.ipynb
diff --git a/tools/pandas.ipynb b/tools/pandas.ipynb
diff --git a/units.py b/units.py