Merge pull request #860 from terascope/migrate-to-esm

Migrate to esm

.eslintrc

@@ -1,3 +1,14 @@
 {
-    "extends": "@terascope"
+    "extends": "@terascope",
+    "parserOptions": {
+        "ecmaVersion": "latest",
+        "sourceType": "module"
+    },
+    "rules": {
+        "@typescript-eslint/naming-convention": "off",
+        "@typescript-eslint/no-duplicate-enum-values": "warn",
+        "import/extensions": "off",
+        "import/no-import-module-exports": "off"
+    },
+    "ignorePatterns":[]
 }

.yarnrc

@@ -3,4 +3,4 @@
 
 
 lastUpdateCheck 1695653572310
-yarn-path ".yarn/releases/yarn-1.22.19.js"
+yarn-path ".yarn/releases/yarn-1.22.19.cjs"

asset/asset.json

@@ -1,5 +1,5 @@
 {
     "name": "standard",
-    "version": "0.25.1",
+    "version": "1.0.0",
     "description": "Teraslice standard processor asset bundle"
 }

asset/package.json

@@ -1,14 +1,15 @@
 {
     "name": "standard",
     "displayName": "Asset",
-    "version": "0.25.1",
+    "version": "1.0.0",
     "private": true,
     "description": "Teraslice standard processor asset bundle",
     "repository": {
         "type": "git",
         "url": "git+https://github.com/terascope/standard-assets.git"
     },
     "license": "MIT",
+    "type": "module",
     "workspaces": {
         "nohoist": [
             "**"
@@ -21,9 +22,9 @@
     },
     "dependencies": {
         "@faker-js/faker": "^8.4.1",
-        "@terascope/job-components": "^0.75.1",
+        "@terascope/job-components": "^1.0.1",
         "@terascope/standard-asset-apis": "^0.7.2",
-        "@terascope/utils": "^0.59.1",
+        "@terascope/utils": "^0.59.3",
         "@types/chance": "^1.1.4",
         "@types/express": "^4.17.19",
         "chance": "^1.1.11",
@@ -33,7 +34,7 @@
         "randexp": "^0.5.3",
         "short-unique-id": "^5.2.0",
         "timsort": "^0.3.0",
-        "ts-transforms": "^0.85.2",
+        "ts-transforms": "^0.85.3",
         "tslib": "^2.6.3"
     },
     "engines": {

asset/src/__lib/accumulator-key.ts

@@ -1,7 +1,7 @@
 import { DataEntity } from '@terascope/job-components';
-import Accumulator from './accumulator';
-import DataWindow from './data-window';
-import { AccumulateByKeyConfig } from '../accumulate_by_key/interfaces';
+import Accumulator from './accumulator.js';
+import DataWindow from './data-window.js';
+import { AccumulateByKeyConfig } from '../accumulate_by_key/interfaces.js';
 
 export default class AccumulatorByKey extends Accumulator {
     buckets = new Map();

asset/src/__lib/utils.ts

@@ -1,6 +1,6 @@
-import path from 'path';
+import path from 'node:path';
 import { get, getTime as tsGetTime } from '@terascope/job-components';
-import { PhaseConfig } from '../transform/interfaces';
+import { PhaseConfig } from '../transform/interfaces.js';
 
 export enum Order {
     asc = 'asc',
@@ -52,11 +52,11 @@ export async function loadResources(opConfig: PhaseConfig, getPaths: GetPathFn):
     if (opConfig.plugins) {
         const pluginPaths = await formatPaths(getPaths, opConfig.plugins);
         Object.assign(opConfig, { plugins: pluginPaths });
-        plugins = pluginPaths.map((pPath) => {
-            const myPlugin = require(pPath);
+        plugins = await Promise.all(pluginPaths.map(async (pPath) => {
+            const myPlugin = await import(pPath);
             // if es6 import default, else use regular node required obj
             return get(myPlugin, 'default', myPlugin);
-        });
+        }));
     }
     return { opConfig, plugins };
 }

asset/src/accumulate/interfaces.ts

@@ -1,4 +1,4 @@
-import { OpConfig } from '@terascope/job-components';
+import { OpConfig } from '@terascope/types';
 
 export interface AccumulateConfig extends OpConfig {
     empty_after: number;

asset/src/accumulate/processor.ts

@@ -1,16 +1,17 @@
 import {
-    BatchProcessor, DataEntity, WorkerContext, ExecutionConfig
+    BatchProcessor, DataEntity, Context
 } from '@terascope/job-components';
-import { AccumulateConfig } from './interfaces';
-import DataWindow from '../__lib/data-window';
-import Accumulator from '../__lib/accumulator';
+import { ExecutionConfig } from '@terascope/types';
+import { AccumulateConfig } from './interfaces.js';
+import DataWindow from '../__lib/data-window.js';
+import Accumulator from '../__lib/accumulator.js';
 
 export default class Accumulate extends BatchProcessor<AccumulateConfig> {
     flushData = false;
     shuttingDown = false;
     accumulator: Accumulator;
 
-    constructor(context: WorkerContext, opConfig: AccumulateConfig, exConfig: ExecutionConfig) {
+    constructor(context: Context, opConfig: AccumulateConfig, exConfig: ExecutionConfig) {
         super(context, opConfig, exConfig);
         this.accumulator = new Accumulator(this.opConfig.empty_after);
     }

asset/src/accumulate/schema.ts

@@ -1,7 +1,7 @@
 import {
     ConvictSchema, isNumber, getTypeOf, isBoolean
 } from '@terascope/job-components';
-import { AccumulateConfig } from './interfaces';
+import { AccumulateConfig } from './interfaces.js';
 
 export default class Schema extends ConvictSchema<AccumulateConfig> {
     build(): Record<string, any> {

asset/src/accumulate_by_key/interfaces.ts

@@ -1,5 +1,5 @@
-import { OpConfig } from '@terascope/job-components';
-import { AccumulateConfig } from '../accumulate/interfaces';
+import { OpConfig } from '@terascope/types';
+import { AccumulateConfig } from '../accumulate/interfaces.js';
 
 export interface AccumulateByKeyConfig extends AccumulateConfig, OpConfig {
     key_field: string;

asset/src/accumulate_by_key/processor.ts

@@ -1,15 +1,16 @@
 import {
-    BatchProcessor, WorkerContext, ExecutionConfig, DataEntity
+    BatchProcessor, Context, DataEntity
 } from '@terascope/job-components';
-import { AccumulateByKeyConfig } from './interfaces';
-import AccumulatorByKey from '../__lib/accumulator-key';
+import { ExecutionConfig } from '@terascope/types';
+import { AccumulateByKeyConfig } from './interfaces.js';
+import AccumulatorByKey from '../__lib/accumulator-key.js';
 
 export default class AccumulateByKey extends BatchProcessor<AccumulateByKeyConfig> {
     flushData = false;
     accumulator: AccumulatorByKey;
 
     constructor(
-        context: WorkerContext, opConfig: AccumulateByKeyConfig, exConfig: ExecutionConfig
+        context: Context, opConfig: AccumulateByKeyConfig, exConfig: ExecutionConfig
     ) {
         super(context, opConfig, exConfig);
         const { empty_after: emptyAfter } = opConfig;

asset/src/accumulate_by_key/schema.ts

@@ -1,5 +1,5 @@
 import { ConvictSchema, isInteger } from '@terascope/job-components';
-import { AccumulateByKeyConfig } from './interfaces';
+import { AccumulateByKeyConfig } from './interfaces.js';
 
 export default class Schema extends ConvictSchema<AccumulateByKeyConfig> {
     build(): Record<string, any> {

asset/src/add_key/processor.ts

@@ -21,7 +21,7 @@ import {
 } from '@terascope/types';
 
 import crypto from 'crypto';
-import DataWindow from '../__lib/data-window';
+import DataWindow from '../__lib/data-window.js';
 
 export default class AddKey extends BatchProcessor {
     async onBatch(data: DataEntity[] | DataWindow[]) {

asset/src/add_short_id/interfaces.ts

@@ -1,4 +1,4 @@
-import { OpConfig } from '@terascope/job-components';
+import { OpConfig } from '@terascope/types';
 
 export interface UniqueIdOpConfig extends OpConfig {
     length: number;

asset/src/add_short_id/processor.ts

@@ -1,14 +1,12 @@
 import {
     MapProcessor,
     DataEntity,
-    WorkerContext,
-    OpConfig,
-    ExecutionConfig,
-
+    Context
 } from '@terascope/job-components';
+import { ExecutionConfig, OpConfig } from '@terascope/types';
 import ShortUniqueId from 'short-unique-id';
-import DataWindow from '../__lib/data-window';
-import { UniqueIdOpConfig } from './interfaces';
+import DataWindow from '../__lib/data-window.js';
+import { UniqueIdOpConfig } from './interfaces.js';
 
 /**
  * Adds a short unique ID of a specified length to the specified field
@@ -18,7 +16,7 @@ import { UniqueIdOpConfig } from './interfaces';
 export default class AddShortId extends MapProcessor<OpConfig> {
     uniqueId: ShortUniqueId;
 
-    constructor(context: WorkerContext, opConfig: UniqueIdOpConfig, exConfig: ExecutionConfig) {
+    constructor(context: Context, opConfig: UniqueIdOpConfig, exConfig: ExecutionConfig) {
         super(context, opConfig, exConfig);
 
         this.uniqueId = new ShortUniqueId({

asset/src/copy_field/processor.ts

@@ -4,7 +4,7 @@ import {
     DataEntity,
 } from '@terascope/job-components';
 import { get, set } from '@terascope/utils';
-import DataWindow from '../__lib/data-window';
+import DataWindow from '../__lib/data-window.js';
 
 export default class CopyField extends MapProcessor<OpConfig> {
     map(doc: DataEntity): DataEntity {

asset/src/count_by_field/interfaces.ts

@@ -1,4 +1,4 @@
-import { OpConfig } from '@terascope/job-components';
+import { OpConfig } from '@terascope/types';
 
 export interface CountByField {
     op_name: string;

asset/src/count_by_field/processor.ts

@@ -1,7 +1,7 @@
 import {
     MapProcessor, DataEntity, isPromAvailable
 } from '@terascope/job-components';
-import { CountByFieldConfig } from './interfaces';
+import { CountByFieldConfig } from './interfaces.js';
 
 type Counters = {
     [valueAsString: string]: {

asset/src/count_by_field/schema.ts

@@ -1,5 +1,5 @@
 import { ConvictSchema } from '@terascope/job-components';
-import { CountByFieldConfig } from './interfaces';
+import { CountByFieldConfig } from './interfaces.js';
 
 export default class Schema extends ConvictSchema<CountByFieldConfig> {
     build(): Record<string, any> {

asset/src/data_generator/counter.ts

@@ -1,4 +1,4 @@
-import { CounterResults } from './interfaces';
+import { CounterResults } from './interfaces.js';
 
 export default class Counter {
     isPersistent: boolean;

asset/src/data_generator/data-schema.ts

@@ -1,6 +1,6 @@
 import moment from 'moment';
 import { AnyObject, isEmpty } from '@terascope/utils';
-import { DataGenerator, DateOptions, IDType } from './interfaces';
+import { DataGenerator, DateOptions, IDType } from './interfaces.js';
 
 // "2016-01-19T13:33:09.356-07:00"
 export const dateFormat = 'YYYY-MM-DDTHH:mm:ss.SSSZ';
@@ -106,7 +106,6 @@ export default function getSchema(opConfig: DataGenerator, otherSchema: AnyObjec
         schema[opConfig.date_key] = schema.created;
         delete schema.created;
     }
-
     if (opConfig.format) {
         const dataConfig = schema[dateKey];
         const newFn = getFormatFunction(opConfig.format, { start, diff });
@@ -122,6 +121,5 @@ export default function getSchema(opConfig: DataGenerator, otherSchema: AnyObjec
         const reg = schema.id.randexp;
         schema.id.randexp = `${opConfig.id_start_key}${reg}`;
     }
-
     return schema;
 }

asset/src/data_generator/fetcher.ts

@@ -1,30 +1,31 @@
 import {
-    Fetcher, WorkerContext, ExecutionConfig, TSError, AnyObject
+    Fetcher, Context, TSError, AnyObject
 } from '@terascope/job-components';
-import mocker from 'mocker-data-generator';
+import { ExecutionConfig } from '@terascope/types';
+import { Mocker } from 'mocker-data-generator';
 import { faker } from '@faker-js/faker';
 import Randexp from 'randexp';
 import Chance from 'chance';
-import path from 'path';
-import { existsSync } from 'fs';
-import { DataGenerator, CounterResults } from './interfaces';
-import defaultSchema from './data-schema';
+import path from 'node:path';
+import { existsSync } from 'node:fs';
+import { DataGenerator, CounterResults } from './interfaces.js';
+import defaultSchema from './data-schema.js';
 
 const chance = new Chance();
 export default class DataGeneratorFetcher extends Fetcher<DataGenerator> {
     dataSchema: AnyObject;
 
-    constructor(context: WorkerContext, opConfig: DataGenerator, exConfig: ExecutionConfig) {
+    constructor(context: Context, opConfig: DataGenerator, exConfig: ExecutionConfig) {
         super(context, opConfig, exConfig);
         this.dataSchema = parsedSchema(opConfig);
     }
 
     async fetch(slice?: CounterResults): Promise<AnyObject[]> {
+        const mocker = new Mocker();
         if (slice == null) return [];
         const { count } = slice;
-
         if (this.opConfig.stress_test) {
-            return mocker()
+            return mocker
                 .addGenerator('faker', faker)
                 .addGenerator('chance', chance)
                 .addGenerator('randexp', Randexp, (Generator, input) => new Generator(input).gen())
@@ -40,8 +41,7 @@ export default class DataGeneratorFetcher extends Fetcher<DataGenerator> {
                 })
                 .catch((err) => Promise.reject(new TSError(err, { reason: 'could not generate mocked data' })));
         }
-
-        return mocker()
+        return mocker
             .addGenerator('faker', faker)
             .addGenerator('chance', chance)
             .addGenerator('randexp', Randexp, (Generator, input) => new Generator(input).gen())
@@ -54,16 +54,15 @@ export default class DataGeneratorFetcher extends Fetcher<DataGenerator> {
 
 function parsedSchema(opConfig: DataGenerator) {
     let dataSchema = {};
-
     if (opConfig.json_schema) {
         const firstPath = opConfig.json_schema;
         const nextPath = path.join(process.cwd(), opConfig.json_schema);
 
         try {
             if (existsSync(firstPath)) {
-                dataSchema = require(firstPath);
+                dataSchema = import(firstPath);
             } else {
-                dataSchema = require(nextPath);
+                dataSchema = import(nextPath);
             }
             return dataSchema;
         } catch (err) {

asset/src/data_generator/interfaces.ts

@@ -1,4 +1,4 @@
-import { OpConfig } from '@terascope/job-components';
+import { OpConfig } from '@terascope/types';
 
 export enum DateOptions {
     dateNow = 'dateNow',

asset/src/data_generator/schema.ts

@@ -7,7 +7,7 @@ import {
     getTypeOf,
     isString
 } from '@terascope/job-components';
-import { DataGenerator, IDType, DateOptions } from './interfaces';
+import { DataGenerator, IDType, DateOptions } from './interfaces.js';
 
 export default class Schema extends ConvictSchema<DataGenerator> {
     validateJob(job: ValidatedJobConfig): void {

asset/src/data_generator/slicer.ts

@@ -1,6 +1,6 @@
 import { Slicer, get, SlicerRecoveryData } from '@terascope/job-components';
-import { DataGenerator, CounterResults } from './interfaces';
-import Counter from './counter';
+import { DataGenerator, CounterResults } from './interfaces.js';
+import Counter from './counter.js';
 
 export default class DataGeneratorSlicer extends Slicer<DataGenerator> {
     countHandle!: () => Promise<CounterResults>;

asset/src/data_window_to_array/processor.ts

@@ -1,5 +1,5 @@
 import { BatchProcessor, DataEntity, OpConfig } from '@terascope/job-components';
-import DataWindow from '../__lib/data-window';
+import DataWindow from '../__lib/data-window.js';
 
 export default class DataWindowToArray extends BatchProcessor<OpConfig> {
     async onBatch(dataArray: DataWindow[]): Promise<DataEntity[]> {

asset/src/date_router/processor.ts

@@ -1,13 +1,14 @@
 import {
-    MapProcessor, DataEntity, WorkerContext, OpConfig, ExecutionConfig
+    MapProcessor, DataEntity, Context
 } from '@terascope/job-components';
+import { ExecutionConfig, OpConfig } from '@terascope/types';
 import { DateRouter, DateRouterConfig } from '@terascope/standard-asset-apis';
 
 export default class DateRouterProcessor extends MapProcessor<DateRouterConfig> {
     router: DateRouter;
 
     constructor(
-        context: WorkerContext,
+        context: Context,
         opConfig: DateRouterConfig & OpConfig,
         exConfig: ExecutionConfig
     ) {