Merge pull request #104 from frictionlessdata/refactoruing/web-to-nod…

…e-stream

Do not use webToNodeStream() outside of File like classes and make so…

.gitignore

@@ -25,4 +25,3 @@ coverage
 .vscode
 
 datajs/test/_babel*
-node_modules

package.json

@@ -8,12 +8,12 @@
   },
   "scripts": {
     "test": "yarn run test:node && yarn run test:browser",
-    "test:browser": "karma start karma.conf.js --browsers=ChromeHeadless",
+    "test:browser": "yarn build:browser && karma start karma.conf.js --browsers=ChromeHeadless",
     "test:node": "nyc mocha --require @babel/register test/*",
     "build:browser": "webpack --mode production",
-    "build:node": "babel ./src/ -d ./dist/node --no-comments",
+    "build:browser:watch": "webpack --watch",
+    "build:node": "rm -rf ./dist/node && babel ./src/ -d ./dist/node --no-comments",
     "build": "yarn run build:node && yarn run build:browser",
-    "build:watch": "webpack --watch",
     "coveralls": "cat ./coverage/lcov.info | ./node_modules/.bin/coveralls",
     "coverage": "nyc report --reporter=text-lcov | coveralls && nyc report --reporter=lcov",
     "lint": "eslint ./src"
@@ -83,4 +83,4 @@
       "text"
     ]
   }
-}
+}

src/browser-utils/index.js

@@ -1,7 +1,6 @@
 if (typeof window !== 'undefined') module.exports = { ...require('./utils') }
 else
   module.exports = {
-    toNodeStream: (reader, size, returnChunk) => {},
     isFileFromBrowser: (file) => {},
     webToNodeStream: (reader) => {},
   }

src/browser-utils/utils.js

@@ -6,9 +6,8 @@ import { ReadableWebToNodeStream } from 'readable-web-to-node-stream'
  * Transform browser's Reader to string, then create a nodejs stream from it
  * @param {object} stream A browser file stream
  * @param {number} size size of file to return
- * @param {boolean} return_chunk whether to return a chunk in string format or a node stream
  */
-export async function webToNodeStream(stream, size, returnChunk = false) {
+export async function webToNodeStream(stream, size) {
   if (size == undefined || size == -1) {
     return new ReadableWebToNodeStream(stream)
   } else {
@@ -18,7 +17,6 @@ export async function webToNodeStream(stream, size, returnChunk = false) {
 
     let lineCounter = 0
     let lastString = ''
-    let chunkText = ''
 
     const decoder = new TextDecoder()
     let reader = stream.getReader()
@@ -33,7 +31,6 @@ export async function webToNodeStream(stream, size, returnChunk = false) {
 
       // Decode the current chunk to string and prepend the last string
       const string = `${lastString}${decoder.decode(value)}`
-      chunkText += string
 
       // Extract lines from chunk
       const lines = string.split(/\r\n|[\r\n]/g)
@@ -54,11 +51,6 @@ export async function webToNodeStream(stream, size, returnChunk = false) {
 
     nodeStream.push(null)
 
-    //return a chunk of the file. Chunk is used when parsing large files in CSV modeule
-    if (returnChunk) {
-      return chunkText
-    }
-
     return nodeStream
   }
 }

src/parser/csv.js

@@ -2,9 +2,8 @@ import parse from 'csv-parse'
 const CSVSniffer = require('csv-sniffer')()
 import toString from 'stream-to-string'
 import { decodeStream } from 'iconv-lite'
-import {webToNodeStream } from '../browser-utils/utils'
 
-export async function csvParser(file, { keyed = false, size} = {}) {
+export async function csvParser(file, { keyed = false, size } = {}) {
   const parseOptions = await getParseOptions(file, keyed)
   let stream = await file.stream(size)
   if (file.descriptor.encoding.toLowerCase().replace('-', '') === 'utf8') {
@@ -27,8 +26,8 @@ export async function guessParseOptions(file) {
     const stream = await file.stream({ end: 50000 })
     text = await toString(stream)
   } else if (file.displayName === 'FileInterface') {
-    let reader = file.descriptor.stream()
-    text = await webToNodeStream(reader, 10, true)
+    let stream = await file.stream(10)
+    text = await toString(stream)
   } else if (file.displayName === 'FileRemote') {
     const stream = await file.stream({ size: 100 })
     let bytes = 0