aboutsummaryrefslogtreecommitdiff
path: root/lib
diff options
context:
space:
mode:
authorAlex Legler <alex@a3li.li>2015-02-23 21:58:05 +0100
committerAlex Legler <alex@a3li.li>2015-02-23 21:58:05 +0100
commit88d9781c469e27ca77901a05326f598821715b33 (patch)
tree111b6c5cd06e4e059089be7019bea7ccde9ccae2 /lib
parentSilence encoding warnings (diff)
downloadbackend-88d9781c469e27ca77901a05326f598821715b33.tar.gz
backend-88d9781c469e27ca77901a05326f598821715b33.tar.bz2
backend-88d9781c469e27ca77901a05326f598821715b33.zip
Only occupy 3/4 of the processors
Diffstat (limited to 'lib')
-rw-r--r--lib/storage.rb6
-rw-r--r--lib/utils.rb8
2 files changed, 11 insertions, 3 deletions
diff --git a/lib/storage.rb b/lib/storage.rb
index 6409df2..5b360f5 100644
--- a/lib/storage.rb
+++ b/lib/storage.rb
@@ -201,9 +201,9 @@ module Ag::Storage
def fix_threading(list, pass)
result = $es.search(
index: 'ml-' + list,
- size: 100000,
+ size: 5000,
body: {
- size: 100000,
+ size: 5000,
query: {
filtered: {
filter: {
@@ -225,7 +225,7 @@ module Ag::Storage
}
)
- Parallel.each(result['hits']['hits'], progress: "Calculating Threading (Pass #{pass})") do |hit|
+ Parallel.each(result['hits']['hits'], in_processes: Ag::Utils.proc_count, progress: "Calculating Threading (Pass #{pass})") do |hit|
msg = resolve_message_id(list, hit['_source']['raw_parent'])
unless msg == nil
diff --git a/lib/utils.rb b/lib/utils.rb
index d621a2e..0213c6d 100644
--- a/lib/utils.rb
+++ b/lib/utils.rb
@@ -1,5 +1,9 @@
require 'charlock_holmes'
+module Parallel::ProcessorCount
+ module_function :processor_count
+end
+
module Ag
module Utils
@@ -62,5 +66,9 @@ module Ag
rescue ArgumentError
''
end
+
+ def proc_count
+ (Parallel::ProcessorCount.processor_count.to_f * 0.75).floor
+ end
end
end \ No newline at end of file