diff --git a/.gitignore b/.gitignore index 6c759bc..fc76209 100644 --- a/.gitignore +++ b/.gitignore @@ -25,4 +25,3 @@ coverage .vscode datajs/test/_babel* -node_modules diff --git a/package.json b/package.json index 75d83fd..a6a5faa 100644 --- a/package.json +++ b/package.json @@ -8,12 +8,12 @@ }, "scripts": { "test": "yarn run test:node && yarn run test:browser", - "test:browser": "karma start karma.conf.js --browsers=ChromeHeadless", + "test:browser": "yarn build:browser && karma start karma.conf.js --browsers=ChromeHeadless", "test:node": "nyc mocha --require @babel/register test/*", "build:browser": "webpack --mode production", - "build:node": "babel ./src/ -d ./dist/node --no-comments", + "build:browser:watch": "webpack --watch", + "build:node": "rm -rf ./dist/node && babel ./src/ -d ./dist/node --no-comments", "build": "yarn run build:node && yarn run build:browser", - "build:watch": "webpack --watch", "coveralls": "cat ./coverage/lcov.info | ./node_modules/.bin/coveralls", "coverage": "nyc report --reporter=text-lcov | coveralls && nyc report --reporter=lcov", "lint": "eslint ./src" @@ -83,4 +83,4 @@ "text" ] } -} \ No newline at end of file +} diff --git a/src/browser-utils/index.js b/src/browser-utils/index.js index 444dd30..703a04e 100644 --- a/src/browser-utils/index.js +++ b/src/browser-utils/index.js @@ -1,7 +1,6 @@ if (typeof window !== 'undefined') module.exports = { ...require('./utils') } else module.exports = { - toNodeStream: (reader, size, returnChunk) => {}, isFileFromBrowser: (file) => {}, webToNodeStream: (reader) => {}, } diff --git a/src/browser-utils/utils.js b/src/browser-utils/utils.js index f73bfbc..c249c6d 100644 --- a/src/browser-utils/utils.js +++ b/src/browser-utils/utils.js @@ -6,9 +6,8 @@ import { ReadableWebToNodeStream } from 'readable-web-to-node-stream' * Transform browser's Reader to string, then create a nodejs stream from it * @param {object} stream A browser file stream * @param {number} size size of file to return - * @param {boolean} return_chunk whether to return a chunk in string format or a node stream */ -export async function webToNodeStream(stream, size, returnChunk = false) { +export async function webToNodeStream(stream, size) { if (size == undefined || size == -1) { return new ReadableWebToNodeStream(stream) } else { @@ -18,7 +17,6 @@ export async function webToNodeStream(stream, size, returnChunk = false) { let lineCounter = 0 let lastString = '' - let chunkText = '' const decoder = new TextDecoder() let reader = stream.getReader() @@ -33,7 +31,6 @@ export async function webToNodeStream(stream, size, returnChunk = false) { // Decode the current chunk to string and prepend the last string const string = `${lastString}${decoder.decode(value)}` - chunkText += string // Extract lines from chunk const lines = string.split(/\r\n|[\r\n]/g) @@ -54,11 +51,6 @@ export async function webToNodeStream(stream, size, returnChunk = false) { nodeStream.push(null) - //return a chunk of the file. Chunk is used when parsing large files in CSV modeule - if (returnChunk) { - return chunkText - } - return nodeStream } } diff --git a/src/parser/csv.js b/src/parser/csv.js index f53445a..4e6272b 100644 --- a/src/parser/csv.js +++ b/src/parser/csv.js @@ -2,9 +2,8 @@ import parse from 'csv-parse' const CSVSniffer = require('csv-sniffer')() import toString from 'stream-to-string' import { decodeStream } from 'iconv-lite' -import {webToNodeStream } from '../browser-utils/utils' -export async function csvParser(file, { keyed = false, size} = {}) { +export async function csvParser(file, { keyed = false, size } = {}) { const parseOptions = await getParseOptions(file, keyed) let stream = await file.stream(size) if (file.descriptor.encoding.toLowerCase().replace('-', '') === 'utf8') { @@ -27,8 +26,8 @@ export async function guessParseOptions(file) { const stream = await file.stream({ end: 50000 }) text = await toString(stream) } else if (file.displayName === 'FileInterface') { - let reader = file.descriptor.stream() - text = await webToNodeStream(reader, 10, true) + let stream = await file.stream(10) + text = await toString(stream) } else if (file.displayName === 'FileRemote') { const stream = await file.stream({ size: 100 }) let bytes = 0