-
Notifications
You must be signed in to change notification settings - Fork 13
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
This PR makes the following changes:
- Creates a `kubernetesV2` directory under
`packages/teraslice/src/lib/cluster/services/cluster/backends` with
copies of all files in `kubernetes` directory
- Modify `K8s` class to use `@kubernetes/client-node` instead of
`kubernetes-client`
- Modify `index.ts`:
- Rename class `KubernetesClusterBackendV2`
- Account for typed responses from new client
- Account for `kubernetesV2` possibility anywhere
`cluster_manager_type`, `CLUSTERING_TYPE` or `TEST_PLATFORM` are checked
- Duplicate unit tests related to k8s to run k8sV2 also
- Add e2e-k8s-v2-tests to github workflow
- Update `ts-scripts test` and `ts-scripts k8s-env` to allow for
kubernetesV2 clustering
- Add `clustering-type` and `keep-open` flags to `ts-scripts k8s-env`
- Add k8sV2 script to root package.json
- Add `@terascope/types:0.17.2` as scripts dependency
Ref: #1799- Loading branch information
Showing
56 changed files
with
3,622 additions
and
88 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -259,6 +259,48 @@ jobs: | |
| run: yarn --silent test:k8s --node-version ${{ matrix.node-version }} | ||
| working-directory: ./e2e | ||
|
|
||
| e2e-k8s-v2-tests: | ||
| runs-on: ubuntu-latest | ||
| strategy: | ||
| # opensearch is finiky, keep testing others if it fails | ||
| fail-fast: false | ||
| matrix: | ||
| node-version: [18.19.1, 20.11.1] | ||
| steps: | ||
| - name: Check out code | ||
| uses: actions/checkout@v3 | ||
|
|
||
| - name: Setup Node ${{ matrix.node-version }} | ||
| uses: actions/setup-node@v3 | ||
| with: | ||
| node-version: ${{ matrix.node-version }} | ||
| cache: 'yarn' | ||
|
|
||
| # we login to docker to avoid docker pull limit rates | ||
| - name: Login to Docker Hub | ||
| uses: docker/login-action@v2 | ||
| with: | ||
| username: ${{ secrets.DOCKER_USERNAME }} | ||
| password: ${{ secrets.DOCKER_PASSWORD }} | ||
|
|
||
| - name: Install and build packages | ||
| run: yarn setup | ||
| env: | ||
| YARN_SETUP_ARGS: "--prod=false --silent" | ||
|
|
||
| - name: Compile e2e code | ||
| run: yarn build | ||
| working-directory: ./e2e | ||
|
|
||
| - name: Install Kind and Kubectl | ||
| uses: helm/[email protected] | ||
| with: | ||
| install_only: "true" | ||
|
|
||
| - name: Test k8s V2 elasticsearch7 | ||
| run: yarn --silent test:k8sV2 --node-version ${{ matrix.node-version }} | ||
| working-directory: ./e2e | ||
|
|
||
| e2e-external-storage-tests: | ||
| runs-on: ubuntu-latest | ||
| strategy: | ||
|
|
||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,55 @@ | ||
| import { Ex } from 'teraslice-client-js'; | ||
| import { WorkerNode } from '@terascope/types/dist/src/teraslice.js'; | ||
| import { pDelay } from '@terascope/utils'; | ||
| import { TerasliceHarness } from '../../teraslice-harness.js'; | ||
| import { DEFAULT_WORKERS, TEST_PLATFORM } from '../../config.js'; | ||
|
|
||
| describe('scale execution', () => { | ||
| let terasliceHarness: TerasliceHarness; | ||
| let job: Ex; | ||
|
|
||
| beforeAll(async () => { | ||
| terasliceHarness = new TerasliceHarness(); | ||
| await terasliceHarness.init(); | ||
| await terasliceHarness.resetState(); | ||
|
|
||
| const jobSpec = terasliceHarness.newJob('generator'); | ||
| // Set resource constraints on workers within CI | ||
| if (TEST_PLATFORM === 'kubernetes' || TEST_PLATFORM === 'kubernetesV2') { | ||
| jobSpec.resources_requests_cpu = 0.1; | ||
| } | ||
| jobSpec.name = 'scale execution'; | ||
|
|
||
| jobSpec.workers = DEFAULT_WORKERS; | ||
|
|
||
| job = await terasliceHarness.submitAndStart(jobSpec); | ||
| }); | ||
|
|
||
| afterAll(async () => { | ||
| await job.stop(); | ||
| }); | ||
|
|
||
| it(`should start with ${DEFAULT_WORKERS} workers`, async () => { | ||
| let runningWorkers: WorkerNode[] = []; | ||
| while (runningWorkers.length === 0) { | ||
| await pDelay(100); | ||
| runningWorkers = await job.workers(); | ||
| } | ||
| expect(runningWorkers).toBeArrayOfSize(DEFAULT_WORKERS); | ||
| }); | ||
|
|
||
| it('should add 1 worker', async () => { | ||
| const runningWorkers = await terasliceHarness.addWorkers(job.id(), 1); | ||
| expect(runningWorkers).toBe(DEFAULT_WORKERS + 1); | ||
| }); | ||
|
|
||
| it('should remove 2 workers', async () => { | ||
| const runningWorkers = await terasliceHarness.removeWorkers(job.id(), 2); | ||
| expect(runningWorkers).toBe(DEFAULT_WORKERS - 1); | ||
| }); | ||
|
|
||
| it('should set workers to 2', async () => { | ||
| const runningWorkers = await terasliceHarness.setWorkers(job.id(), 2); | ||
| expect(runningWorkers).toBe(2); | ||
| }); | ||
| }); |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.