Skip to content

Commit

Permalink
Don't rely on index from future
Browse files Browse the repository at this point in the history
  • Loading branch information
nampas committed Oct 28, 2024
1 parent 3f06972 commit 9cc21f7
Showing 1 changed file with 21 additions and 22 deletions.
43 changes: 21 additions & 22 deletions lib/ruby_snowflake/client/threaded_in_memory_strategy.rb
Original file line number Diff line number Diff line change
Expand Up @@ -9,31 +9,30 @@ def self.result(statement_json_body, retreive_proc, num_threads)
result[0] = statement_json_body["data"]

thread_pool = Concurrent::FixedThreadPool.new(num_threads)
futures = []
partitions.each_with_index do |partition, index|
next if index == 0 # already have the first partition
futures << Concurrent::Future.execute(executor: thread_pool) do
[index, retreive_proc.call(index)]
partitions
.each_with_index do |partition, index|
next if index == 0 # already have the first partition
futures << [index, Concurrent::Future.execute(executor: thread_pool) { retreive_proc.call(index) }]
end
end
futures.each do |future|
if future.rejected?
if future.reason.is_a? RubySnowflake::Error
raise future.reason
else
raise ConnectionStarvedError.new(
"A partition request timed out. This is usually do to using the client in" \
"multiple threads. The client uses a connection thread pool and if too many" \
"requests are all done in threads at the same time, threads can get starved" \
"of access to connections. The solution for this is to either increase the " \
"max_connections parameter on the client or create a new client instance" \
"with it's own connection pool to snowflake per thread. Rejection reason: #{future.reason.message}"
)
.compact
.each do |entry|
index, future = entry
if future.rejected?
if future.reason.is_a? RubySnowflake::Error
raise future.reason
else
raise ConnectionStarvedError.new(
"A partition request timed out. This is usually do to using the client in" \
"multiple threads. The client uses a connection thread pool and if too many" \
"requests are all done in threads at the same time, threads can get starved" \
"of access to connections. The solution for this is to either increase the " \
"max_connections parameter on the client or create a new client instance" \
"with it's own connection pool to snowflake per thread. Rejection reason: #{future.reason.message}"
)
end
end
result[index] = future.value
end
index, partition_data = future.value
result[index] = partition_data
end
result
end
end
Expand Down

0 comments on commit 9cc21f7

Please sign in to comment.