fix Spark-version-based build CI

The current version writes the Spark version to a file that is not read anymore. It needs to be replaced with the proper SBT command.

.github/workflows/ci.yml

@@ -18,11 +18,11 @@ on:
         required: true
 
 jobs:
-  spark_versions_validation:
+  spark-versions-validation:
     if: ${{ github.event_name != 'workflow_dispatch' }}
     strategy:
       matrix:
-        spark_version: ['3.2.4', '3.3.2', '3.4.1']
+        spark-version: ['3.4.1', '3.5.0']
     runs-on: ubuntu-latest
     steps: 
     - uses: actions/checkout@v4
@@ -32,33 +32,33 @@ jobs:
         distribution: "temurin"
         cache: "sbt"
     - env:
-        SPARK_VERSION_OVERRIDE: ${{ matrix.spark_version }}
-      run: |
-        echo "$SPARK_VERSION_OVERRIDE" > project/spark-version.conf
-    - run: cat project/spark-version.conf
+        SPARK_VERSION: ${{ matrix.spark-version }}
+      run: echo "SPARK_VERSION_OVERRIDE=${SPARK_VERSION}" >> $GITHUB_ENV
+    - run: sbt "show sparkVersion"
     - run: sbt compile 
     - run: sbt test 
 
   build:
     runs-on: ubuntu-latest
-    env:
-      SPARK_VERSION_OVERRIDE: ${{ inputs.spark-version }}
     steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-java@v3
         with:
           java-version: 8
           distribution: "temurin"
           cache: "sbt"
-      - if: ${{ env.SPARK_VERSION_OVERRIDE != '' }}
-        run: echo "$SPARK_VERSION_OVERRIDE" > project/spark-version.conf
+      - if: ${{ github.event.inputs.spark-version != '' }}
+        env:
+          SPARK_VERSION: ${{ inputs.spark-version }}
+        run: echo "SPARK_VERSION_OVERRIDE=${SPARK_VERSION}" >> $GITHUB_ENV 
+      - run: sbt "show sparkVersion"
       - run: sbt compile 
       - run: sbt test 
       - run: sbt package
-      - run: tar cf artefacts.tar target/scala-*/*.jar */target/scala-*/*.jar
+      - run: tar cf artefacts.tar target/ */target/
       - uses: actions/[email protected]
         with:
           name: Artefacts
           path: artefacts.tar
 
-run-name: ${{ github.event_name == 'workflow_dispatch' && format('Building for Spark {0}', inputs.spark-version) || ' '}}
+run-name: ${{ github.event_name == 'workflow_dispatch' && format('Building for Spark {0}', inputs.spark-version) || ' '}}

build.sbt

@@ -4,7 +4,7 @@ ThisBuild / versionScheme := Some("strict")
 ThisBuild / scalaVersion := "2.12.12"
 ThisBuild / organization := "com.amadeus.dataio"
 val sparkVersion = settingKey[String]("The version of Spark used for building.")
-ThisBuild / sparkVersion := "3.4.1"
+ThisBuild / sparkVersion := sys.env.getOrElse("SPARK_VERSION_OVERRIDE", "3.4.1")
 
 // Common dependencies
 ThisBuild / libraryDependencies ++= Seq(