Skip to content

Commit

Permalink
single thread
Browse files Browse the repository at this point in the history
  • Loading branch information
Baunsgaard committed Oct 21, 2024
1 parent b163f56 commit 83f81a1
Showing 1 changed file with 6 additions and 2 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -62,14 +62,18 @@ protected void readCSVFrameFromHDFS( Path path, JobConf job, FileSystem fs,
if(HDFSTool.isDirectory(fs, path))
splits = IOUtilFunctions.sortInputSplits(splits);

final ExecutorService pool = CommonThreadPool.get(numThreads);

final ExecutorService pool = CommonThreadPool.get(numThreads);
try {
if(splits.length == 1){
new ReadRowsTask(splits[0], informat, job, dest, 0, true).call();
return;
}
//compute num rows per split
ArrayList<Future<Long>> cret = new ArrayList<>();
for( int i=0; i<splits.length - 1; i++ )
cret.add(pool.submit(new CountRowsTask(splits[i], informat, job, _props.hasHeader() && i==0)));


//compute row offset per split via cumsum on row counts
int offset = 0;
ArrayList<Future<Object>> tasks2 = new ArrayList<>();
Expand Down

0 comments on commit 83f81a1

Please sign in to comment.