diff --git a/e2e/test/cases/assets/simple-spec.ts b/e2e/test/cases/assets/simple-spec.ts index e437ac3dbe2..fe5226b1478 100644 --- a/e2e/test/cases/assets/simple-spec.ts +++ b/e2e/test/cases/assets/simple-spec.ts @@ -13,7 +13,7 @@ import { import { Teraslice } from '@terascope/types'; import { pWhile } from '@terascope/utils'; import crypto from 'crypto'; -import { TerasliceHarness, JobFixtureNames } from '../../teraslice-harness.js'; +import { TerasliceHarness } from '../../teraslice-harness.js'; import { ASSET_STORAGE_CONNECTION_TYPE, MINIO_ACCESS_KEY, MINIO_HOST, MINIO_SECRET_KEY, TEST_PLATFORM } from '../../config.js'; @@ -27,42 +27,6 @@ describe('assets', () => { await terasliceHarness.resetState(); }); - /** - * Uploads the specified asset file and then submits the specified job config - * it then waits for the job to enter the running state, then waits for - * the requested number of workers to enter the joined state. Then that has - * happened it tests to see that the number of workers joined is the number - * of workers requested. This reasoning is a bit circular, but a worker - * won't enter the `joined` state if it fails to load its assets. - * - * @param {string} jobSpecName the name of job to run - * @param {string} assetPath the relative path to the asset file - */ - async function submitAndValidateAssetJob(jobSpecName: JobFixtureNames, assetPath: string) { - const fileStream = fs.createReadStream(assetPath); - const jobSpec = terasliceHarness.newJob(jobSpecName); - // Set resource constraints on workers within CI - if (TEST_PLATFORM === 'kubernetes') { - jobSpec.resources_requests_cpu = 0.1; - } - const { workers } = jobSpec; // save for comparison - - const result = await terasliceHarness.teraslice.assets.upload( - fileStream, - { blocking: true } - ); - // NOTE: In this case, the asset is referenced by the ID - // assigned by teraslice and not it's name. - jobSpec.assets = [result._id, 'standard', 'elasticsearch']; - - const ex = await terasliceHarness.submitAndStart(jobSpec); - - const r = await terasliceHarness.forWorkersJoined(ex.id(), workers as number, 25); - expect(r).toEqual(workers); - - await ex.stop({ blocking: true }); - } - it('after uploading an asset, it can be deleted', async () => { const testStream = fs.createReadStream('test/fixtures/assets/example_asset_1.zip'); @@ -96,31 +60,16 @@ describe('assets', () => { } }); - // Type 1 Asset - asset.json at top level of zipfile - // example_assets/ - // example_assets/drop_property/ - // example_assets/drop_property/index.js - // asset.json - xit('after starting a job with a Type 1 asset specified by ID should eventually have all workers joined', async () => { - const assetPath = 'test/fixtures/assets/example_asset_1.zip'; - - await submitAndValidateAssetJob('generator-asset', assetPath); - }); - - // Type 2 Asset - asset.json in subdirectory of zipfile - // example_assets/ - // example_assets/asset.json - // example_assets/drop_property/ - // example_assets/drop_property/index.js - xit('after starting a job with a Type 2 asset specified by ID should eventually have all workers joined', async () => { - const assetPath = 'test/fixtures/assets/example_asset_2.zip'; - await submitAndValidateAssetJob('generator-asset', assetPath); - }); - - xit('can update an asset bundle and use the new asset', async () => { - const assetPath = 'test/fixtures/assets/example_asset_1updated.zip'; + it('can update an asset bundle and use the new asset', async () => { + const olderAssetPath = 'test/fixtures/assets/example_asset_1.zip'; + const olderStream = fs.createReadStream(olderAssetPath); + await terasliceHarness.teraslice.assets.upload( + olderStream, + { blocking: true } + ); - const fileStream = fs.createReadStream(assetPath); + const newerAssetPath = 'test/fixtures/assets/example_asset_1updated.zip'; + const fileStream = fs.createReadStream(newerAssetPath); // the asset on this job already points to 'ex1' so it should use the latest available asset const jobSpec = terasliceHarness.newJob('generator-asset'); // Set resource constraints on workers within CI @@ -149,16 +98,18 @@ describe('assets', () => { await ex.stop({ blocking: true }); }); - xit('can directly ask for the new asset to be used', async () => { + it('can directly ask for a specific asset version to be used', async () => { const jobSpec = terasliceHarness.newJob('generator-asset'); // Set resource constraints on workers within CI if (TEST_PLATFORM === 'kubernetes') { jobSpec.resources_requests_cpu = 0.1; } - jobSpec.assets = ['ex1:0.1.1', 'standard', 'elasticsearch']; + // the previous test confirms the newer version will be used by default + // now we test to see if we can select the older version + jobSpec.assets = ['ex1:0.0.1', 'standard', 'elasticsearch']; const { workers } = jobSpec; - const assetResponse = await terasliceHarness.teraslice.assets.getAsset('ex1', '0.1.1'); + const assetResponse = await terasliceHarness.teraslice.assets.getAsset('ex1', '0.0.1'); const assetId = assetResponse[0].id; const ex = await terasliceHarness.submitAndStart(jobSpec); @@ -221,7 +172,7 @@ describe('s3 asset storage', () => { const response = await getS3Object(s3client, { Bucket: bucketName, Key: `${assetId}.zip` }); const base64 = await response.Body?.transformToString('base64'); - expect(base64).toStartWith('UEsDBAoAAAAAAAs6O'); + expect(base64).toStartWith('UEsDBBQAAgAIAFBl0'); }); it('does not create the "blob" field when storing asset metadata in ES', async () => { diff --git a/e2e/test/cases/cluster/api-spec.ts b/e2e/test/cases/cluster/api-spec.ts index 518b4f988f8..cc07d842e73 100644 --- a/e2e/test/cases/cluster/api-spec.ts +++ b/e2e/test/cases/cluster/api-spec.ts @@ -13,7 +13,7 @@ describe('cluster api', () => { await terasliceHarness.resetState(); }); - xit('submitted jobs are not saved in validated form', async () => { + it('submitted jobs are not saved in validated form', async () => { const assetPath = 'test/fixtures/assets/example_asset_1.zip'; const testStream = createReadStream(assetPath); const jobSpec = terasliceHarness.newJob('generator-asset'); @@ -30,7 +30,7 @@ describe('cluster api', () => { expect(jobConfig).toMatchObject(jobSpec); }); - xit('should update job config', async () => { + it('should update job config', async () => { // NOTE that this relies on the asset loaded in the test above const jobSpec = terasliceHarness.newJob('generator-asset'); const { workers, slicers } = jobSpec; @@ -114,7 +114,7 @@ describe('cluster api', () => { expect(response[0]).toHaveProperty('version'); }); - xit('api end point /assets/assetName should return an array of json objects of asset metadata', async () => { + it('api end point /assets/assetName should return an array of json objects of asset metadata', async () => { const response = await terasliceHarness.teraslice.cluster.get('/assets/ex1'); expect(response).toBeArray(); @@ -124,7 +124,7 @@ describe('cluster api', () => { expect(response[0]).toHaveProperty('version'); }); - xit('api end point /assets/assetName/version should return an array of json objects of asset metadata', async () => { + it('api end point /assets/assetName/version should return an array of json objects of asset metadata', async () => { const response = await terasliceHarness.teraslice.cluster.get('/assets/ex1/0.0.1'); expect(response).toBeArray(); @@ -139,12 +139,12 @@ describe('cluster api', () => { expect(response).toBeString(); }); - xit('api end point /txt/assets/assetName should return a text table', async () => { + it('api end point /txt/assets/assetName should return a text table', async () => { const response = await terasliceHarness.teraslice.cluster.txt('assets/ex1'); expect(response).toBeString(); }); - xit('api end point /txt/assets/assetName/version should return a text table', async () => { + it('api end point /txt/assets/assetName/version should return a text table', async () => { const response = await terasliceHarness.teraslice.cluster.txt('assets/ex1/0.0.1'); expect(response).toBeString(); }); diff --git a/e2e/test/fixtures/assets/example_asset_1.zip b/e2e/test/fixtures/assets/example_asset_1.zip index 32a4e168f3f..dd0b5cd21c1 100644 Binary files a/e2e/test/fixtures/assets/example_asset_1.zip and b/e2e/test/fixtures/assets/example_asset_1.zip differ diff --git a/e2e/test/fixtures/assets/example_asset_1updated.zip b/e2e/test/fixtures/assets/example_asset_1updated.zip index 5c4e473c256..99743ac0635 100644 Binary files a/e2e/test/fixtures/assets/example_asset_1updated.zip and b/e2e/test/fixtures/assets/example_asset_1updated.zip differ diff --git a/e2e/test/fixtures/assets/example_asset_2.zip b/e2e/test/fixtures/assets/example_asset_2.zip deleted file mode 100644 index 50a2a982219..00000000000 Binary files a/e2e/test/fixtures/assets/example_asset_2.zip and /dev/null differ diff --git a/packages/teraslice/test/storage/assets_storage-spec.ts b/packages/teraslice/test/storage/assets_storage-spec.ts index 14ab7c3e97c..2a5cfce62eb 100644 --- a/packages/teraslice/test/storage/assets_storage-spec.ts +++ b/packages/teraslice/test/storage/assets_storage-spec.ts @@ -63,14 +63,14 @@ describe('AssetsStorage using S3 backend', () => { const filePath = 'e2e/test/fixtures/assets/example_asset_1.zip'; const buffer = fs.readFileSync(filePath); const result = await storage.save(buffer); - expect(result.assetId).toBe('2909ec5fd38466cf6276cc14ede25096f1f34ee9'); + expect(result.assetId).toBe('caf0e5ce7cf1edc864f306b1d9edbad0f7060545'); }); it('can grab asset info from S3', async () => { const list = await storage.grabS3Info(); expect(list).toEqual([{ - File: '2909ec5fd38466cf6276cc14ede25096f1f34ee9.zip', - Size: 2759 + File: 'caf0e5ce7cf1edc864f306b1d9edbad0f7060545.zip', + Size: 162711 }]); }); @@ -78,13 +78,13 @@ describe('AssetsStorage using S3 backend', () => { /// create a buffer copy of example_asset_1.zip to test if it equals what s3 sends back const filePath = 'e2e/test/fixtures/assets/example_asset_1.zip'; const buffer = fs.readFileSync(filePath); - const assetRecord = await storage.get('2909ec5fd38466cf6276cc14ede25096f1f34ee9'); + const assetRecord = await storage.get('caf0e5ce7cf1edc864f306b1d9edbad0f7060545'); expect(buffer.equals(assetRecord.blob as Buffer)).toBe(true); expect(assetRecord.name).toBe('ex1'); }); it('can delete an asset from S3', async () => { - await storage.remove('2909ec5fd38466cf6276cc14ede25096f1f34ee9'); + await storage.remove('caf0e5ce7cf1edc864f306b1d9edbad0f7060545'); const list = await storage.grabS3Info(); expect(list).toBeEmpty(); }); diff --git a/packages/teraslice/test/storage/s3_store-spec.ts b/packages/teraslice/test/storage/s3_store-spec.ts index 4032c8da4c5..2ff83b29fff 100644 --- a/packages/teraslice/test/storage/s3_store-spec.ts +++ b/packages/teraslice/test/storage/s3_store-spec.ts @@ -109,14 +109,14 @@ describe('S3 backend test', () => { }); it('should be able to download asset', async () => { - const filePath = 'e2e/test/fixtures/assets/example_asset_2.zip'; + const filePath = 'e2e/test/fixtures/assets/example_asset_1.zip'; const fileBuffer = fse.readFileSync(filePath); - await s3Backend.save('ex2', fileBuffer, 30000); + await s3Backend.save('ex1', fileBuffer, 30000); - const result = await s3Backend.get('ex2'); + const result = await s3Backend.get('ex1'); expect(result.equals(fileBuffer)).toBe(true); - await s3Backend.remove('ex2'); + await s3Backend.remove('ex1'); }); });