Merge branch 'dev' into ruff-pydoc

.github/workflows/code-quality.yaml

@@ -34,7 +34,7 @@ jobs:
       uses: actions/checkout@v3
       with:
         repository: mosaicml/ci-testing
-        ref: v0.0.7
+        ref: v0.0.8
         path: ./ci-testing
     - uses: ./ci-testing/.github/actions/code-quality
       with:

.github/workflows/codeql-analysis.yml

@@ -45,7 +45,7 @@ jobs:
       uses: actions/checkout@v3
       with:
         repository: mosaicml/ci-testing
-        ref: v0.0.7
+        ref: v0.0.8
         path: ./ci-testing
     - uses: ./ci-testing/.github/actions/codeql-analysis
       with:

.github/workflows/coverage.yaml

@@ -16,7 +16,7 @@ jobs:
       uses: actions/checkout@v3
       with:
         repository: mosaicml/ci-testing
-        ref: v0.0.7
+        ref: v0.0.8
         path: ./ci-testing
     - uses: ./ci-testing/.github/actions/coverage
       with:

.github/workflows/daily.yaml

@@ -14,7 +14,7 @@ concurrency:
   cancel-in-progress: ${{ github.ref != 'refs/heads/main' && github.ref != 'refs/heads/dev' }}
 jobs:
   daily-pytest-cpu:
-    uses: mosaicml/ci-testing/.github/workflows/[email protected].7
+    uses: mosaicml/ci-testing/.github/workflows/[email protected].8
     strategy:
       matrix:
         include:
@@ -100,7 +100,7 @@ jobs:
       download-path: artifacts
 
   daily-pytest-gpu:
-    uses: mosaicml/ci-testing/.github/workflows/[email protected].7
+    uses: mosaicml/ci-testing/.github/workflows/[email protected].8
     strategy:
       matrix:
         # Unlike CPU tests, we run daily tests together with GPU tests to minimize launch time

.github/workflows/pr-cpu.yaml

@@ -9,7 +9,7 @@ concurrency:
   cancel-in-progress: ${{ github.ref != 'refs/heads/main' && github.ref != 'refs/heads/dev' }}
 jobs:
   pytest-cpu:
-    uses: mosaicml/ci-testing/.github/workflows/[email protected].7
+    uses: mosaicml/ci-testing/.github/workflows/[email protected].8
     strategy:
       matrix:
         include:

.github/workflows/pr-gpu.yaml

@@ -9,7 +9,7 @@ concurrency:
   cancel-in-progress: ${{ github.ref != 'refs/heads/main' && github.ref != 'refs/heads/dev' }}
 jobs:
   pytest-gpu-1:
-    uses: mosaicml/ci-testing/.github/workflows/[email protected].7
+    uses: mosaicml/ci-testing/.github/workflows/[email protected].8
     strategy:
       matrix:
         include:
@@ -35,7 +35,7 @@ jobs:
       mcloud-api-key: ${{ secrets.MCLOUD_API_KEY }}
 
   pytest-gpu-2:
-    uses: mosaicml/ci-testing/.github/workflows/[email protected].7
+    uses: mosaicml/ci-testing/.github/workflows/[email protected].8
     strategy:
       matrix:
         include:
@@ -62,7 +62,7 @@ jobs:
 
 
   pytest-gpu-4:
-    uses: mosaicml/ci-testing/.github/workflows/[email protected].7
+    uses: mosaicml/ci-testing/.github/workflows/[email protected].8
     strategy:
       matrix:
         include:

.github/workflows/release.yaml

@@ -24,7 +24,7 @@ jobs:
       uses: actions/checkout@v3
       with:
         repository: mosaicml/ci-testing
-        ref: v0.0.7
+        ref: v0.0.8
         path: ./ci-testing
     - uses: ./ci-testing/.github/actions/code-quality
       with:

.github/workflows/smoketest.yaml

@@ -33,7 +33,7 @@ jobs:
       uses: actions/checkout@v3
       with:
         repository: mosaicml/ci-testing
-        ref: v0.0.7
+        ref: v0.0.8
         path: ./ci-testing
     - uses: ./ci-testing/.github/actions/smoketest
       with:

composer/metrics/nlp.py

@@ -178,3 +178,27 @@ def compute(self) -> Tensor:
         """Returns torch.exp() of the LanguageCrossEntropy."""
         avg_loss = super().compute()
         return torch.exp(avg_loss)
+
+
+# For backward compatibility
+class InContextLearningMetric:
+    """InContextLearningMetric only exists for backwards compatibility of checkpoints that contain pickled metrics."""
+
+    def __init__(self):
+        raise RuntimeError(
+            f'This class only exists for maintaining backward compatibility for checkpoints that contain pickled metrics. Please instead use https://github.com/mosaicml/llm-foundry/blob/main/scripts/eval/README.md.',
+        )
+
+    def __getstate__(self):
+        return None
+
+    def __setstate__(self, state):
+        pass
+
+
+InContextLearningCodeEvalAccuracy = InContextLearningMetric
+InContextLearningLMAccuracy = InContextLearningMetric
+InContextLearningLMExpectedCalibrationError = InContextLearningMetric
+InContextLearningMCExpectedCalibrationError = InContextLearningMetric
+InContextLearningQAAccuracy = InContextLearningMetric
+InContextLearningMultipleChoiceAccuracy = InContextLearningMetric

docker/README.md

@@ -15,8 +15,8 @@ all dependencies for both NLP and Vision models. They are built on top of the
 <!-- BEGIN_COMPOSER_BUILD_MATRIX -->
 | Composer Version   | CUDA Support   | Docker Tag                                                     |
 |--------------------|----------------|----------------------------------------------------------------|
-| 0.23.1             | Yes            | `mosaicml/composer:latest`, `mosaicml/composer:0.23.1`         |
-| 0.23.1             | No             | `mosaicml/composer:latest_cpu`, `mosaicml/composer:0.23.1_cpu` |
+| 0.23.2             | Yes            | `mosaicml/composer:latest`, `mosaicml/composer:0.23.2`         |
+| 0.23.2             | No             | `mosaicml/composer:latest_cpu`, `mosaicml/composer:0.23.2_cpu` |
 <!-- END_COMPOSER_BUILD_MATRIX -->
 
 **Note**: For a lightweight installation, we recommended using a [MosaicML PyTorch Image](#pytorch-images) and manually

docker/build_matrix.yaml

@@ -208,9 +208,9 @@
   TORCHVISION_VERSION: 0.16.2
 - AWS_OFI_NCCL_VERSION: ''
   BASE_IMAGE: nvidia/cuda:12.1.1-cudnn8-devel-ubuntu20.04
-  COMPOSER_INSTALL_COMMAND: mosaicml[all]==0.23.1
+  COMPOSER_INSTALL_COMMAND: mosaicml[all]==0.23.2
   CUDA_VERSION: 12.1.1
-  IMAGE_NAME: composer-0-23-1
+  IMAGE_NAME: composer-0-23-2
   MOFED_VERSION: latest-23.10
   NVIDIA_REQUIRE_CUDA_OVERRIDE: cuda>=12.1 brand=tesla,driver>=450,driver<451 brand=tesla,driver>=470,driver<471
     brand=unknown,driver>=470,driver<471 brand=nvidia,driver>=470,driver<471 brand=nvidiartx,driver>=470,driver<471
@@ -231,23 +231,23 @@
   PYTORCH_NIGHTLY_VERSION: ''
   PYTORCH_VERSION: 2.3.1
   TAGS:
-  - mosaicml/composer:0.23.1
+  - mosaicml/composer:0.23.2
   - mosaicml/composer:latest
   TARGET: composer_stage
   TORCHVISION_VERSION: 0.18.1
 - AWS_OFI_NCCL_VERSION: ''
   BASE_IMAGE: ubuntu:20.04
-  COMPOSER_INSTALL_COMMAND: mosaicml[all]==0.23.1
+  COMPOSER_INSTALL_COMMAND: mosaicml[all]==0.23.2
   CUDA_VERSION: ''
-  IMAGE_NAME: composer-0-23-1-cpu
+  IMAGE_NAME: composer-0-23-2-cpu
   MOFED_VERSION: latest-23.10
   NVIDIA_REQUIRE_CUDA_OVERRIDE: ''
   PYTHON_VERSION: '3.11'
   PYTORCH_NIGHTLY_URL: ''
   PYTORCH_NIGHTLY_VERSION: ''
   PYTORCH_VERSION: 2.3.1
   TAGS:
-  - mosaicml/composer:0.23.1_cpu
+  - mosaicml/composer:0.23.2_cpu
   - mosaicml/composer:latest_cpu
   TARGET: composer_stage
   TORCHVISION_VERSION: 0.18.1

docker/generate_build_matrix.py

@@ -231,7 +231,7 @@ def _main():
     composer_entries = []
 
     # The `GIT_COMMIT` is a placeholder and Jenkins will substitute it with the actual git commit for the `composer_staging` images
-    composer_versions = ['0.23.1']  # Only build images for the latest composer version
+    composer_versions = ['0.23.2']  # Only build images for the latest composer version
     composer_python_versions = [PRODUCTION_PYTHON_VERSION]  # just build composer against the latest
 
     for product in itertools.product(composer_python_versions, composer_versions, cuda_options):