diff --git a/lib/mosql/streamer.rb b/lib/mosql/streamer.rb index 78812c7..621788a 100644 --- a/lib/mosql/streamer.rb +++ b/lib/mosql/streamer.rb @@ -1,3 +1,5 @@ +require 'Parallel' + module MoSQL class Streamer include MoSQL::Logging @@ -106,7 +108,7 @@ def initial_import db = @mongo.db(dbname) collections = db.collections.select { |c| spec.key?(c.name) } - collections.each do |collection| + Parallel.each(collections, in_threads: 8) do |collection| ns = "#{dbname}.#{collection.name}" import_collection(ns, collection, spec[collection.name][:meta][:filter]) exit(0) if @done @@ -156,7 +158,7 @@ def import_collection(ns, collection, filter) if count % BATCH == 0 sql_time += upsert_all_batches(batches, ns) elapsed = Time.now - start - log.info("Imported #{count} rows (#{elapsed}s, #{sql_time}s SQL)...") + log.info("Imported #{count} rows into #{collection} (#{elapsed}s, #{sql_time}s SQL)...") exit(0) if @done end end diff --git a/mosql.gemspec b/mosql.gemspec index 8089b30..2bb14b3 100644 --- a/mosql.gemspec +++ b/mosql.gemspec @@ -21,6 +21,7 @@ Gem::Specification.new do |gem| gem.add_runtime_dependency "rake" gem.add_runtime_dependency "log4r" gem.add_runtime_dependency "json" + gem.add_runtime_dependency "parallel" gem.add_runtime_dependency "mongoriver", "0.4"