Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Rework e2e test assets #3666

Merged
merged 4 commits into from
Jun 28, 2024
Merged

Rework e2e test assets #3666

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
a69e0f5
remove old assets
busma13 Jun 21, 2024
99f1e9b
add new assets
busma13 Jun 21, 2024
de37e0a
rework tests
busma13 Jun 21, 2024
857cfce
Fix base64 string for s3 test
busma13 Jun 21, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
81 changes: 16 additions & 65 deletions e2e/test/cases/assets/simple-spec.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ import {
import { Teraslice } from '@terascope/types';
import { pWhile } from '@terascope/utils';
import crypto from 'crypto';
import { TerasliceHarness, JobFixtureNames } from '../../teraslice-harness.js';
import { TerasliceHarness } from '../../teraslice-harness.js';
import {
ASSET_STORAGE_CONNECTION_TYPE, MINIO_ACCESS_KEY, MINIO_HOST, MINIO_SECRET_KEY, TEST_PLATFORM
} from '../../config.js';
Expand All @@ -27,42 +27,6 @@ describe('assets', () => {
await terasliceHarness.resetState();
});

/**
* Uploads the specified asset file and then submits the specified job config
* it then waits for the job to enter the running state, then waits for
* the requested number of workers to enter the joined state. Then that has
* happened it tests to see that the number of workers joined is the number
* of workers requested. This reasoning is a bit circular, but a worker
* won't enter the `joined` state if it fails to load its assets.
*
* @param {string} jobSpecName the name of job to run
* @param {string} assetPath the relative path to the asset file
*/
async function submitAndValidateAssetJob(jobSpecName: JobFixtureNames, assetPath: string) {
const fileStream = fs.createReadStream(assetPath);
const jobSpec = terasliceHarness.newJob(jobSpecName);
// Set resource constraints on workers within CI
if (TEST_PLATFORM === 'kubernetes') {
jobSpec.resources_requests_cpu = 0.1;
}
const { workers } = jobSpec; // save for comparison

const result = await terasliceHarness.teraslice.assets.upload(
fileStream,
{ blocking: true }
);
// NOTE: In this case, the asset is referenced by the ID
// assigned by teraslice and not it's name.
jobSpec.assets = [result._id, 'standard', 'elasticsearch'];

const ex = await terasliceHarness.submitAndStart(jobSpec);

const r = await terasliceHarness.forWorkersJoined(ex.id(), workers as number, 25);
expect(r).toEqual(workers);

await ex.stop({ blocking: true });
}

it('after uploading an asset, it can be deleted', async () => {
const testStream = fs.createReadStream('test/fixtures/assets/example_asset_1.zip');

Expand Down Expand Up @@ -96,31 +60,16 @@ describe('assets', () => {
}
});

// Type 1 Asset - asset.json at top level of zipfile
// example_assets/
// example_assets/drop_property/
// example_assets/drop_property/index.js
// asset.json
xit('after starting a job with a Type 1 asset specified by ID should eventually have all workers joined', async () => {
const assetPath = 'test/fixtures/assets/example_asset_1.zip';

await submitAndValidateAssetJob('generator-asset', assetPath);
});

// Type 2 Asset - asset.json in subdirectory of zipfile
// example_assets/
// example_assets/asset.json
// example_assets/drop_property/
// example_assets/drop_property/index.js
xit('after starting a job with a Type 2 asset specified by ID should eventually have all workers joined', async () => {
const assetPath = 'test/fixtures/assets/example_asset_2.zip';
await submitAndValidateAssetJob('generator-asset', assetPath);
});

xit('can update an asset bundle and use the new asset', async () => {
const assetPath = 'test/fixtures/assets/example_asset_1updated.zip';
it('can update an asset bundle and use the new asset', async () => {
const olderAssetPath = 'test/fixtures/assets/example_asset_1.zip';
const olderStream = fs.createReadStream(olderAssetPath);
await terasliceHarness.teraslice.assets.upload(
olderStream,
{ blocking: true }
);

const fileStream = fs.createReadStream(assetPath);
const newerAssetPath = 'test/fixtures/assets/example_asset_1updated.zip';
const fileStream = fs.createReadStream(newerAssetPath);
// the asset on this job already points to 'ex1' so it should use the latest available asset
const jobSpec = terasliceHarness.newJob('generator-asset');
// Set resource constraints on workers within CI
Expand Down Expand Up @@ -149,16 +98,18 @@ describe('assets', () => {
await ex.stop({ blocking: true });
});

xit('can directly ask for the new asset to be used', async () => {
it('can directly ask for a specific asset version to be used', async () => {
const jobSpec = terasliceHarness.newJob('generator-asset');
// Set resource constraints on workers within CI
if (TEST_PLATFORM === 'kubernetes') {
jobSpec.resources_requests_cpu = 0.1;
}
jobSpec.assets = ['ex1:0.1.1', 'standard', 'elasticsearch'];
// the previous test confirms the newer version will be used by default
// now we test to see if we can select the older version
jobSpec.assets = ['ex1:0.0.1', 'standard', 'elasticsearch'];
const { workers } = jobSpec;

const assetResponse = await terasliceHarness.teraslice.assets.getAsset('ex1', '0.1.1');
const assetResponse = await terasliceHarness.teraslice.assets.getAsset('ex1', '0.0.1');
const assetId = assetResponse[0].id;

const ex = await terasliceHarness.submitAndStart(jobSpec);
Expand Down Expand Up @@ -221,7 +172,7 @@ describe('s3 asset storage', () => {

const response = await getS3Object(s3client, { Bucket: bucketName, Key: `${assetId}.zip` });
const base64 = await response.Body?.transformToString('base64');
expect(base64).toStartWith('UEsDBAoAAAAAAAs6O');
expect(base64).toStartWith('UEsDBBQAAgAIAFBl0');
});

it('does not create the "blob" field when storing asset metadata in ES', async () => {
Expand Down
12 changes: 6 additions & 6 deletions e2e/test/cases/cluster/api-spec.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ describe('cluster api', () => {
await terasliceHarness.resetState();
});

xit('submitted jobs are not saved in validated form', async () => {
it('submitted jobs are not saved in validated form', async () => {
const assetPath = 'test/fixtures/assets/example_asset_1.zip';
const testStream = createReadStream(assetPath);
const jobSpec = terasliceHarness.newJob('generator-asset');
Expand All @@ -30,7 +30,7 @@ describe('cluster api', () => {
expect(jobConfig).toMatchObject(jobSpec);
});

xit('should update job config', async () => {
it('should update job config', async () => {
// NOTE that this relies on the asset loaded in the test above
const jobSpec = terasliceHarness.newJob('generator-asset');
const { workers, slicers } = jobSpec;
Expand Down Expand Up @@ -114,7 +114,7 @@ describe('cluster api', () => {
expect(response[0]).toHaveProperty('version');
});

xit('api end point /assets/assetName should return an array of json objects of asset metadata', async () => {
it('api end point /assets/assetName should return an array of json objects of asset metadata', async () => {
const response = await terasliceHarness.teraslice.cluster.get('/assets/ex1');

expect(response).toBeArray();
Expand All @@ -124,7 +124,7 @@ describe('cluster api', () => {
expect(response[0]).toHaveProperty('version');
});

xit('api end point /assets/assetName/version should return an array of json objects of asset metadata', async () => {
it('api end point /assets/assetName/version should return an array of json objects of asset metadata', async () => {
const response = await terasliceHarness.teraslice.cluster.get('/assets/ex1/0.0.1');

expect(response).toBeArray();
Expand All @@ -139,12 +139,12 @@ describe('cluster api', () => {
expect(response).toBeString();
});

xit('api end point /txt/assets/assetName should return a text table', async () => {
it('api end point /txt/assets/assetName should return a text table', async () => {
const response = await terasliceHarness.teraslice.cluster.txt('assets/ex1');
expect(response).toBeString();
});

xit('api end point /txt/assets/assetName/version should return a text table', async () => {
it('api end point /txt/assets/assetName/version should return a text table', async () => {
const response = await terasliceHarness.teraslice.cluster.txt('assets/ex1/0.0.1');
expect(response).toBeString();
});
Expand Down
Binary file modified e2e/test/fixtures/assets/example_asset_1.zip
View file
Open in desktop
Binary file not shown.
Binary file modified e2e/test/fixtures/assets/example_asset_1updated.zip
View file
Open in desktop
Binary file not shown.
Binary file removed e2e/test/fixtures/assets/example_asset_2.zip
View file
Open in desktop
Binary file not shown.
10 changes: 5 additions & 5 deletions packages/teraslice/test/storage/assets_storage-spec.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -63,28 +63,28 @@ describe('AssetsStorage using S3 backend', () => {
const filePath = 'e2e/test/fixtures/assets/example_asset_1.zip';
const buffer = fs.readFileSync(filePath);
const result = await storage.save(buffer);
expect(result.assetId).toBe('2909ec5fd38466cf6276cc14ede25096f1f34ee9');
expect(result.assetId).toBe('caf0e5ce7cf1edc864f306b1d9edbad0f7060545');
});

it('can grab asset info from S3', async () => {
const list = await storage.grabS3Info();
expect(list).toEqual([{
File: '2909ec5fd38466cf6276cc14ede25096f1f34ee9.zip',
Size: 2759
File: 'caf0e5ce7cf1edc864f306b1d9edbad0f7060545.zip',
Size: 162711
}]);
});

it('can get an asset from S3', async () => {
/// create a buffer copy of example_asset_1.zip to test if it equals what s3 sends back
const filePath = 'e2e/test/fixtures/assets/example_asset_1.zip';
const buffer = fs.readFileSync(filePath);
const assetRecord = await storage.get('2909ec5fd38466cf6276cc14ede25096f1f34ee9');
const assetRecord = await storage.get('caf0e5ce7cf1edc864f306b1d9edbad0f7060545');
expect(buffer.equals(assetRecord.blob as Buffer)).toBe(true);
expect(assetRecord.name).toBe('ex1');
});

it('can delete an asset from S3', async () => {
await storage.remove('2909ec5fd38466cf6276cc14ede25096f1f34ee9');
await storage.remove('caf0e5ce7cf1edc864f306b1d9edbad0f7060545');
const list = await storage.grabS3Info();
expect(list).toBeEmpty();
});
Expand Down
8 changes: 4 additions & 4 deletions packages/teraslice/test/storage/s3_store-spec.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -109,14 +109,14 @@ describe('S3 backend test', () => {
});

it('should be able to download asset', async () => {
const filePath = 'e2e/test/fixtures/assets/example_asset_2.zip';
const filePath = 'e2e/test/fixtures/assets/example_asset_1.zip';
const fileBuffer = fse.readFileSync(filePath);
await s3Backend.save('ex2', fileBuffer, 30000);
await s3Backend.save('ex1', fileBuffer, 30000);

const result = await s3Backend.get('ex2');
const result = await s3Backend.get('ex1');

expect(result.equals(fileBuffer)).toBe(true);
await s3Backend.remove('ex2');
await s3Backend.remove('ex1');
});
});

Expand Down
Loading