Rework e2e test assets (#3666)

This PR makes the following changes:
- Replace old style assets in e2e tests with bundled assets.
- Re-enable tests that were failing because of those assets.
- Remove 2 tests that checked if different types of old style assets
could be used.

ref: #3653

e2e/test/cases/assets/simple-spec.ts

@@ -13,7 +13,7 @@ import {
 import { Teraslice } from '@terascope/types';
 import { pWhile } from '@terascope/utils';
 import crypto from 'crypto';
-import { TerasliceHarness, JobFixtureNames } from '../../teraslice-harness.js';
+import { TerasliceHarness } from '../../teraslice-harness.js';
 import {
     ASSET_STORAGE_CONNECTION_TYPE, MINIO_ACCESS_KEY, MINIO_HOST, MINIO_SECRET_KEY, TEST_PLATFORM
 } from '../../config.js';
@@ -27,42 +27,6 @@ describe('assets', () => {
         await terasliceHarness.resetState();
     });
 
-    /**
-     * Uploads the specified asset file and then submits the specified job config
-     * it then waits for the job to enter the running state, then waits for
-     * the requested number of workers to enter the joined state.  Then that has
-     * happened it tests to see that the number of workers joined is the number
-     * of workers requested.  This reasoning is a bit circular, but a worker
-     * won't enter the `joined` state if it fails to load its assets.
-     *
-     * @param {string}   jobSpecName the name of job to run
-     * @param {string}   assetPath   the relative path to the asset file
-     */
-    async function submitAndValidateAssetJob(jobSpecName: JobFixtureNames, assetPath: string) {
-        const fileStream = fs.createReadStream(assetPath);
-        const jobSpec = terasliceHarness.newJob(jobSpecName);
-        // Set resource constraints on workers within CI
-        if (TEST_PLATFORM === 'kubernetes') {
-            jobSpec.resources_requests_cpu = 0.1;
-        }
-        const { workers } = jobSpec; // save for comparison
-
-        const result = await terasliceHarness.teraslice.assets.upload(
-            fileStream,
-            { blocking: true }
-        );
-        // NOTE: In this case, the asset is referenced by the ID
-        // assigned by teraslice and not it's name.
-        jobSpec.assets = [result._id, 'standard', 'elasticsearch'];
-
-        const ex = await terasliceHarness.submitAndStart(jobSpec);
-
-        const r = await terasliceHarness.forWorkersJoined(ex.id(), workers as number, 25);
-        expect(r).toEqual(workers);
-
-        await ex.stop({ blocking: true });
-    }
-
     it('after uploading an asset, it can be deleted', async () => {
         const testStream = fs.createReadStream('test/fixtures/assets/example_asset_1.zip');
 
@@ -96,31 +60,16 @@ describe('assets', () => {
         }
     });
 
-    // Type 1 Asset - asset.json at top level of zipfile
-    // example_assets/
-    // example_assets/drop_property/
-    // example_assets/drop_property/index.js
-    // asset.json
-    xit('after starting a job with a Type 1 asset specified by ID should eventually have all workers joined', async () => {
-        const assetPath = 'test/fixtures/assets/example_asset_1.zip';
-
-        await submitAndValidateAssetJob('generator-asset', assetPath);
-    });
-
-    // Type 2 Asset - asset.json in subdirectory of zipfile
-    // example_assets/
-    // example_assets/asset.json
-    // example_assets/drop_property/
-    // example_assets/drop_property/index.js
-    xit('after starting a job with a Type 2 asset specified by ID should eventually have all workers joined', async () => {
-        const assetPath = 'test/fixtures/assets/example_asset_2.zip';
-        await submitAndValidateAssetJob('generator-asset', assetPath);
-    });
-
-    xit('can update an asset bundle and use the new asset', async () => {
-        const assetPath = 'test/fixtures/assets/example_asset_1updated.zip';
+    it('can update an asset bundle and use the new asset', async () => {
+        const olderAssetPath = 'test/fixtures/assets/example_asset_1.zip';
+        const olderStream = fs.createReadStream(olderAssetPath);
+        await terasliceHarness.teraslice.assets.upload(
+            olderStream,
+            { blocking: true }
+        );
 
-        const fileStream = fs.createReadStream(assetPath);
+        const newerAssetPath = 'test/fixtures/assets/example_asset_1updated.zip';
+        const fileStream = fs.createReadStream(newerAssetPath);
         // the asset on this job already points to 'ex1' so it should use the latest available asset
         const jobSpec = terasliceHarness.newJob('generator-asset');
         // Set resource constraints on workers within CI
@@ -149,16 +98,18 @@ describe('assets', () => {
         await ex.stop({ blocking: true });
     });
 
-    xit('can directly ask for the new asset to be used', async () => {
+    it('can directly ask for a specific asset version to be used', async () => {
         const jobSpec = terasliceHarness.newJob('generator-asset');
         // Set resource constraints on workers within CI
         if (TEST_PLATFORM === 'kubernetes') {
             jobSpec.resources_requests_cpu = 0.1;
         }
-        jobSpec.assets = ['ex1:0.1.1', 'standard', 'elasticsearch'];
+        // the previous test confirms the newer version will be used by default
+        // now we test to see if we can select the older version
+        jobSpec.assets = ['ex1:0.0.1', 'standard', 'elasticsearch'];
         const { workers } = jobSpec;
 
-        const assetResponse = await terasliceHarness.teraslice.assets.getAsset('ex1', '0.1.1');
+        const assetResponse = await terasliceHarness.teraslice.assets.getAsset('ex1', '0.0.1');
         const assetId = assetResponse[0].id;
 
         const ex = await terasliceHarness.submitAndStart(jobSpec);
@@ -221,7 +172,7 @@ describe('s3 asset storage', () => {
 
             const response = await getS3Object(s3client, { Bucket: bucketName, Key: `${assetId}.zip` });
             const base64 = await response.Body?.transformToString('base64');
-            expect(base64).toStartWith('UEsDBAoAAAAAAAs6O');
+            expect(base64).toStartWith('UEsDBBQAAgAIAFBl0');
         });
 
         it('does not create the "blob" field when storing asset metadata in ES', async () => {

e2e/test/cases/cluster/api-spec.ts

@@ -13,7 +13,7 @@ describe('cluster api', () => {
         await terasliceHarness.resetState();
     });
 
-    xit('submitted jobs are not saved in validated form', async () => {
+    it('submitted jobs are not saved in validated form', async () => {
         const assetPath = 'test/fixtures/assets/example_asset_1.zip';
         const testStream = createReadStream(assetPath);
         const jobSpec = terasliceHarness.newJob('generator-asset');
@@ -30,7 +30,7 @@ describe('cluster api', () => {
         expect(jobConfig).toMatchObject(jobSpec);
     });
 
-    xit('should update job config', async () => {
+    it('should update job config', async () => {
         // NOTE that this relies on the asset loaded in the test above
         const jobSpec = terasliceHarness.newJob('generator-asset');
         const { workers, slicers } = jobSpec;
@@ -114,7 +114,7 @@ describe('cluster api', () => {
         expect(response[0]).toHaveProperty('version');
     });
 
-    xit('api end point /assets/assetName should return an array of json objects of asset metadata', async () => {
+    it('api end point /assets/assetName should return an array of json objects of asset metadata', async () => {
         const response = await terasliceHarness.teraslice.cluster.get('/assets/ex1');
 
         expect(response).toBeArray();
@@ -124,7 +124,7 @@ describe('cluster api', () => {
         expect(response[0]).toHaveProperty('version');
     });
 
-    xit('api end point /assets/assetName/version should return an array of json objects of asset metadata', async () => {
+    it('api end point /assets/assetName/version should return an array of json objects of asset metadata', async () => {
         const response = await terasliceHarness.teraslice.cluster.get('/assets/ex1/0.0.1');
 
         expect(response).toBeArray();
@@ -139,12 +139,12 @@ describe('cluster api', () => {
         expect(response).toBeString();
     });
 
-    xit('api end point /txt/assets/assetName should return a text table', async () => {
+    it('api end point /txt/assets/assetName should return a text table', async () => {
         const response = await terasliceHarness.teraslice.cluster.txt('assets/ex1');
         expect(response).toBeString();
     });
 
-    xit('api end point /txt/assets/assetName/version should return a text table', async () => {
+    it('api end point /txt/assets/assetName/version should return a text table', async () => {
         const response = await terasliceHarness.teraslice.cluster.txt('assets/ex1/0.0.1');
         expect(response).toBeString();
     });

packages/teraslice/test/storage/assets_storage-spec.ts

@@ -63,28 +63,28 @@ describe('AssetsStorage using S3 backend', () => {
         const filePath = 'e2e/test/fixtures/assets/example_asset_1.zip';
         const buffer = fs.readFileSync(filePath);
         const result = await storage.save(buffer);
-        expect(result.assetId).toBe('2909ec5fd38466cf6276cc14ede25096f1f34ee9');
+        expect(result.assetId).toBe('caf0e5ce7cf1edc864f306b1d9edbad0f7060545');
     });
 
     it('can grab asset info from S3', async () => {
         const list = await storage.grabS3Info();
         expect(list).toEqual([{
-            File: '2909ec5fd38466cf6276cc14ede25096f1f34ee9.zip',
-            Size: 2759
+            File: 'caf0e5ce7cf1edc864f306b1d9edbad0f7060545.zip',
+            Size: 162711
         }]);
     });
 
     it('can get an asset from S3', async () => {
         /// create a buffer copy of example_asset_1.zip to test if it equals what s3 sends back
         const filePath = 'e2e/test/fixtures/assets/example_asset_1.zip';
         const buffer = fs.readFileSync(filePath);
-        const assetRecord = await storage.get('2909ec5fd38466cf6276cc14ede25096f1f34ee9');
+        const assetRecord = await storage.get('caf0e5ce7cf1edc864f306b1d9edbad0f7060545');
         expect(buffer.equals(assetRecord.blob as Buffer)).toBe(true);
         expect(assetRecord.name).toBe('ex1');
     });
 
     it('can delete an asset from S3', async () => {
-        await storage.remove('2909ec5fd38466cf6276cc14ede25096f1f34ee9');
+        await storage.remove('caf0e5ce7cf1edc864f306b1d9edbad0f7060545');
         const list = await storage.grabS3Info();
         expect(list).toBeEmpty();
     });

packages/teraslice/test/storage/s3_store-spec.ts

@@ -109,14 +109,14 @@ describe('S3 backend test', () => {
         });
 
         it('should be able to download asset', async () => {
-            const filePath = 'e2e/test/fixtures/assets/example_asset_2.zip';
+            const filePath = 'e2e/test/fixtures/assets/example_asset_1.zip';
             const fileBuffer = fse.readFileSync(filePath);
-            await s3Backend.save('ex2', fileBuffer, 30000);
+            await s3Backend.save('ex1', fileBuffer, 30000);
 
-            const result = await s3Backend.get('ex2');
+            const result = await s3Backend.get('ex1');
 
             expect(result.equals(fileBuffer)).toBe(true);
-            await s3Backend.remove('ex2');
+            await s3Backend.remove('ex1');
         });
     });