Add node 22.2.0 (#3628)

# New Changes
- Adds node `22.2.0` support 
- Removes experimental `json` imports with standard typescript imports
- This removes the need to use `assert` to grab objects out of `.json`
files
  - #3622
- Adds **nan** `v2.19.0` to resolutions for node 22 compatibility
- Updates docs to have a new `.md` file that gives detailed instructions
on how to introduce new node versions of node
- **Removes** outdated `generator-teraslice` package
- Release: (minor) **teraslice** `v1.8.0`

Ref to issue #3605

.github/workflows/publish-tag.yml

@@ -42,7 +42,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        node-version: [18.19.1, 20.11.1]
+        node-version: [18.19.1, 20.11.1, 22.2.0]
     steps:
       # we login to docker to publish new teraslice image
       - name: Login to Docker Hub

.github/workflows/test.yml

@@ -32,7 +32,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        node-version: [18.19.1, 20.11.1]
+        node-version: [18.19.1, 20.11.1, 22.2.0]
     steps:
       - name: Check out code
         uses: actions/checkout@v3
@@ -55,7 +55,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        node-version: [18.19.1, 20.11.1]
+        node-version: [18.19.1, 20.11.1, 22.2.0]
     steps:
       - name: Check out code
         uses: actions/checkout@v3
@@ -83,7 +83,7 @@ jobs:
       # opensearch is finiky, keep testing others if it fails
       fail-fast: false
       matrix:
-        node-version: [18.19.1, 20.11.1]
+        node-version: [18.19.1, 20.11.1, 22.2.0]
         search-version: [elasticsearch6, elasticsearch7, opensearch1, opensearch2]
     steps:
       - name: Check out code
@@ -117,7 +117,7 @@ jobs:
       # opensearch is finiky, keep testing others if it fails
       fail-fast: false
       matrix:
-        node-version: [18.19.1, 20.11.1]
+        node-version: [18.19.1, 20.11.1, 22.2.0]
         search-version: [elasticsearch6, elasticsearch7, opensearch1, opensearch2]
     steps:
       - name: Check out code
@@ -151,7 +151,7 @@ jobs:
       # opensearch is finiky, keep testing others if it fails
       fail-fast: false
       matrix:
-        node-version: [18.19.1, 20.11.1]
+        node-version: [18.19.1, 20.11.1, 22.2.0]
         search-version: [elasticsearch6, elasticsearch7, opensearch1, opensearch2]
     steps:
       - name: Check out code
@@ -185,7 +185,7 @@ jobs:
       # opensearch is finiky, keep testing others if it fails
       fail-fast: false
       matrix:
-        node-version: [18.19.1, 20.11.1]
+        node-version: [18.19.1, 20.11.1, 22.2.0]
         search-version: [elasticsearch6, elasticsearch7, opensearch1, opensearch2]
     steps:
       - name: Check out code
@@ -223,7 +223,7 @@ jobs:
       # opensearch is finiky, keep testing others if it fails
       fail-fast: false
       matrix:
-        node-version: [18.19.1, 20.11.1]
+        node-version: [18.19.1, 20.11.1, 22.2.0]
     steps:
       - name: Check out code
         uses: actions/checkout@v3

Dockerfile

@@ -14,6 +14,9 @@ COPY packages /app/source/packages
 COPY scripts /app/source/scripts
 COPY types /app/source/types
 
+# Check to see if distutils is installed because python 3.12 removed it
+RUN python3 -c "import distutils" || (apk update && apk add py3-setuptools)
+
 RUN yarn --prod=false --frozen-lockfile \
     && yarn build \
     && yarn \

docs/development/node.md

@@ -0,0 +1,83 @@
+---
+title: Updating Node Versions
+sidebar_label: Node
+---
+
+## Adding a new node verison
+
+Updating node is a multi-step process that requires changes across multiple Terascope repositories.
+
+### 1. Updating Node in the Base Docker Image Repository
+
+Start by modifying the CI YAML files in the base-docker-image repository.
+
+https://github.com/terascope/base-docker-image
+
+
+#### Modify CI YAML Files
+
+There are two files that need to be updated to create new base images with the appropriate Node versions for Teraslice. These files are:
+
+- build.yml
+- release.yml
+
+Add the new Node version to the matrix list in both files.
+
+https://github.com/terascope/base-docker-image/tree/master/.github/workflows
+
+#### Push and Merge Changes
+
+Push your changes to a new branch.
+Once the changes are merged and a new release is made, the CI will create new base Docker images with the updated Node version.
+
+#### Verify New Docker Images
+
+The new Docker images will be available on Docker Hub here:
+
+https://hub.docker.com/r/terascope/node-base/tags
+
+### 2. Updating node in assets
+
+Before we can update teraslice we must update the assets. This is because teraslice e2e tests will break if we don't already have asset releases with the new node version.
+
+To start, update `teraslice-cli` to have the ability to build assets with the specified node version.
+
+Update the `bundleTarget` in teraslice/packages/teraslice-cli/src/helpers/asset-src.ts and the `bundle-target` `choices` array in teraslice/packages/teraslice-cli/src/helpers/yargs-options.ts
+
+Afterwards bump `teraslice-cli` and make a new npm release.
+
+Next we need to update the workflows repo to include the new node version:
+
+https://github.com/terascope/workflows
+
+Update node matrices in workflows/.github/workflows/asset-build-and-publish.yml and workflows/.github/workflows/asset-test.yml
+
+After it's merged, Copy hash from the merge of workflows and append to `uses` in .github/workflows/build-and-publish-asset.yml and .github/workflows/test-asset.yml of each of these assets:
+
+- https://github.com/terascope/file-assets
+- https://github.com/terascope/standard-assets
+- https://github.com/terascope/kafka-assets
+- https://github.com/terascope/elasticsearch-assets
+
+Example `test-asset.yml`:
+
+```yaml
+jobs:
+  call-asset-test-workflow:
+    uses: terascope/workflows/.github/workflows/asset-test.yml@ieqo2423biu22 <- Replace everything after `@` w/ new merge hash
+    secrets: inherit
+```
+
+**_IMPORTANT NOTE:_** elasticsearch-assets has it's own `test-asset.yml` so it's important to manually add the node version to the matrices of that file.
+
+Create all new releases for the assets before moving on to the next step.
+
+### 3. Updating node in teraslice
+
+Add the new node version to all the spots needed inside each file in `.github/workflows` in teraslice.
+
+- publish-master.yml
+- publish-tag.yml
+- test.yml
+
+Once the node version has been added to the arrays of all the workflow .yml files, pushing the branch and making a new PR should trigger CI to run new tests with the added node version. Once all tests are passing, then a new node version has been succesfully introduced to teraslice.

e2e/test/fixtures/jobs/generate-to-es.ts

@@ -0,0 +1,21 @@
+export default {
+    name: 'generator',
+    slicers: 1,
+    lifecycle: 'persistent',
+    workers: 2,
+    analytics: false,
+    assets: ['elasticsearch', 'standard'],
+    max_retries: 0,
+    operations: [
+        {
+            _op: 'data_generator',
+            size: 100
+        },
+        {
+            _op: 'elasticsearch_bulk',
+            index: 'replace-me-1000',
+            type: 'events',
+            size: 1000
+        }
+    ]
+};

e2e/test/fixtures/jobs/generator-asset.ts

@@ -0,0 +1,23 @@
+export default {
+    name: 'generator-asset',
+    slicers: 1,
+    lifecycle: 'persistent',
+    workers: 2,
+    assets: ['ex1', 'standard'],
+    max_retries: 0,
+    analytics: false,
+    operations: [
+        {
+            _op: 'data_generator',
+            size: 1000
+        },
+        {
+            _op: 'drop_property',
+            property: 'userAgent'
+        },
+        {
+            _op: 'delay',
+            ms: 100
+        }
+    ]
+};

e2e/test/fixtures/jobs/generator-large-asset.ts

@@ -0,0 +1,19 @@
+export default {
+    name: 'generator',
+    slicers: 1,
+    lifecycle: 'once',
+    workers: 3,
+    analytics: false,
+    assets: ['standard', 'large-example-asset'],
+    max_retries: 0,
+    operations: [
+        {
+            _op: 'data_generator',
+            size: 1000,
+            stress_test: false
+        },
+        {
+            _op: 'noop'
+        }
+    ]
+};

e2e/test/fixtures/jobs/generator.ts

@@ -0,0 +1,19 @@
+export default {
+    name: 'generator',
+    slicers: 1,
+    lifecycle: 'persistent',
+    workers: 3,
+    analytics: false,
+    assets: ['standard'],
+    max_retries: 0,
+    operations: [
+        {
+            _op: 'data_generator',
+            size: 1000,
+            stress_test: false
+        },
+        {
+            _op: 'noop'
+        }
+    ]
+};

e2e/test/fixtures/jobs/id.ts

@@ -0,0 +1,23 @@
+export default {
+    name: 'ID_Reindex',
+    lifecycle: 'once',
+    analytics: true,
+    slicers: 2,
+    workers: 4,
+    assets: ['elasticsearch'],
+    operations: [
+        {
+            _op: 'id_reader',
+            index: 'replace-me-1000',
+            type: 'events',
+            size: 500,
+            key_type: 'base64url'
+        },
+        {
+            _op: 'elasticsearch_bulk',
+            index: 'test-id_reindex-1000',
+            type: 'events',
+            size: 200
+        }
+    ]
+};