diff options
author | Alex Legler <alex@a3li.li> | 2015-02-23 21:58:05 +0100 |
---|---|---|
committer | Alex Legler <alex@a3li.li> | 2015-02-23 21:58:05 +0100 |
commit | 88d9781c469e27ca77901a05326f598821715b33 (patch) | |
tree | 111b6c5cd06e4e059089be7019bea7ccde9ccae2 | |
parent | Silence encoding warnings (diff) | |
download | backend-88d9781c469e27ca77901a05326f598821715b33.tar.gz backend-88d9781c469e27ca77901a05326f598821715b33.tar.bz2 backend-88d9781c469e27ca77901a05326f598821715b33.zip |
Only occupy 3/4 of the processors
-rwxr-xr-x | ag | 6 | ||||
-rw-r--r-- | lib/storage.rb | 6 | ||||
-rw-r--r-- | lib/utils.rb | 8 |
3 files changed, 14 insertions, 6 deletions
@@ -115,13 +115,13 @@ def do_full messages = $maildir.list(:cur) - Parallel.each(messages, progress: "Importing #{$options.name}") do |maildir_message| + Parallel.each(messages, in_processes: Ag::Utils.proc_count, progress: "Importing #{$options.name}") do |maildir_message| mail = maildir_message.data begin Ag::Storage.store($options.name, mail, maildir_message.unique_name) rescue => e - $stderr.puts "Cannot save message #{mail.message_id}: #{e.message}" + $stderr.puts "Cannot save message #{mail.message_id}: (#{e.class}) #{e.message}" if $options.debug next end end @@ -132,7 +132,7 @@ end def do_incremental messages = $maildir.list(:cur) - Parallel.each(messages, progress: "Importing #{$options.name}") do |maildir_message| + Parallel.each(messages, in_processes: Ag::Utils.proc_count, progress: "Importing #{$options.name}") do |maildir_message| mail = maildir_message.data begin diff --git a/lib/storage.rb b/lib/storage.rb index 6409df2..5b360f5 100644 --- a/lib/storage.rb +++ b/lib/storage.rb @@ -201,9 +201,9 @@ module Ag::Storage def fix_threading(list, pass) result = $es.search( index: 'ml-' + list, - size: 100000, + size: 5000, body: { - size: 100000, + size: 5000, query: { filtered: { filter: { @@ -225,7 +225,7 @@ module Ag::Storage } ) - Parallel.each(result['hits']['hits'], progress: "Calculating Threading (Pass #{pass})") do |hit| + Parallel.each(result['hits']['hits'], in_processes: Ag::Utils.proc_count, progress: "Calculating Threading (Pass #{pass})") do |hit| msg = resolve_message_id(list, hit['_source']['raw_parent']) unless msg == nil diff --git a/lib/utils.rb b/lib/utils.rb index d621a2e..0213c6d 100644 --- a/lib/utils.rb +++ b/lib/utils.rb @@ -1,5 +1,9 @@ require 'charlock_holmes' +module Parallel::ProcessorCount + module_function :processor_count +end + module Ag module Utils @@ -62,5 +66,9 @@ module Ag rescue ArgumentError '' end + + def proc_count + (Parallel::ProcessorCount.processor_count.to_f * 0.75).floor + end end end
\ No newline at end of file |