diff --git a/Gemfile.lock b/Gemfile.lock index 130da6d..57129a8 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,7 +1,7 @@ PATH remote: . specs: - little_monster (0.1.22) + little_monster (0.1.24) activesupport multi_json thor @@ -14,51 +14,120 @@ PATH GEM remote: https://rubygems.org/ specs: - activesupport (6.0.3.2) + activesupport (6.1.5) concurrent-ruby (~> 1.0, >= 1.0.2) - i18n (>= 0.7, < 2) - minitest (~> 5.1) - tzinfo (~> 1.1) - zeitwerk (~> 2.2, >= 2.2.2) + i18n (>= 1.6, < 2) + minitest (>= 5.1) + tzinfo (~> 2.0) + zeitwerk (~> 2.3) addressable (2.8.0) public_suffix (>= 2.0.2, < 5.0) ast (2.3.0) - aws-eventstream (1.1.0) - aws-partitions (1.380.0) - aws-sdk-core (3.109.0) + aws-eventstream (1.2.0) + aws-partitions (1.576.0) + aws-sdk-core (3.130.1) aws-eventstream (~> 1, >= 1.0.2) - aws-partitions (~> 1, >= 1.239.0) + aws-partitions (~> 1, >= 1.525.0) aws-sigv4 (~> 1.1) jmespath (~> 1.0) - aws-sdk-sqs (1.34.0) - aws-sdk-core (~> 3, >= 3.109.0) + aws-sdk-sqs (1.51.0) + aws-sdk-core (~> 3, >= 3.127.0) aws-sigv4 (~> 1.1) - aws-sigv4 (1.2.2) + aws-sigv4 (1.4.0) aws-eventstream (~> 1, >= 1.0.2) byebug (9.0.5) codeclimate-test-reporter (0.6.0) simplecov (>= 0.7.1, < 1.0.0) coderay (1.1.1) - concurrent-ruby (1.1.6) + concurrent-ruby (1.1.10) concurrent-ruby-edge (0.6.0) concurrent-ruby (~> 1.1.6) - crack (0.4.3) - safe_yaml (~> 1.0.0) + crack (0.4.5) + rexml diff-lcs (1.2.5) docile (1.1.5) - ethon (0.12.0) - ffi (>= 1.3.0) - ffi (1.13.1) + ethon (0.15.0) + ffi (>= 1.15.0) + faraday (1.10.0) + faraday-em_http (~> 1.0) + faraday-em_synchrony (~> 1.0) + faraday-excon (~> 1.1) + faraday-httpclient (~> 1.0) + faraday-multipart (~> 1.0) + faraday-net_http (~> 1.0) + faraday-net_http_persistent (~> 1.0) + faraday-patron (~> 1.0) + faraday-rack (~> 1.0) + faraday-retry (~> 1.0) + ruby2_keywords (>= 0.0.4) + faraday-em_http (1.0.0) + faraday-em_synchrony (1.0.0) + faraday-excon (1.1.0) + faraday-httpclient (1.0.1) + faraday-multipart (1.0.3) + multipart-post (>= 1.2, < 3) + faraday-net_http (1.0.1) + faraday-net_http_persistent (1.2.0) + faraday-patron (1.0.0) + faraday-rack (1.0.0) + faraday-retry (1.0.3) + ffi (1.15.5) + gapic-common (0.8.0) + faraday (~> 1.3) + google-protobuf (~> 3.14) + googleapis-common-protos (>= 1.3.11, < 2.a) + googleapis-common-protos-types (>= 1.0.6, < 2.a) + googleauth (>= 0.17.0, < 2.a) + grpc (~> 1.36) + google-cloud-core (1.6.0) + google-cloud-env (~> 1.0) + google-cloud-errors (~> 1.0) + google-cloud-env (1.6.0) + faraday (>= 0.17.3, < 3.0) + google-cloud-errors (1.2.0) + google-cloud-pubsub (2.9.1) + concurrent-ruby (~> 1.1) + google-cloud-core (~> 1.5) + google-cloud-pubsub-v1 (~> 0.0) + google-cloud-pubsub-v1 (0.8.0) + gapic-common (>= 0.7, < 2.a) + google-cloud-errors (~> 1.0) + grpc-google-iam-v1 (>= 0.6.10, < 2.a) + google-protobuf (3.20.0) + googleapis-common-protos (1.3.12) + google-protobuf (~> 3.14) + googleapis-common-protos-types (~> 1.2) + grpc (~> 1.27) + googleapis-common-protos-types (1.3.1) + google-protobuf (~> 3.14) + googleauth (1.1.2) + faraday (>= 0.17.3, < 3.a) + jwt (>= 1.4, < 3.0) + memoist (~> 0.16) + multi_json (~> 1.11) + os (>= 0.9, < 2.0) + signet (>= 0.16, < 2.a) + grpc (1.45.0) + google-protobuf (~> 3.19) + googleapis-common-protos-types (~> 1.0) + grpc-google-iam-v1 (1.1.0) + google-protobuf (~> 3.14) + googleapis-common-protos (>= 1.3.12, < 2.0) + grpc (~> 1.27) hashdiff (1.0.1) - i18n (1.8.5) + i18n (1.10.0) concurrent-ruby (~> 1.0) - jmespath (1.4.0) + jmespath (1.6.1) json (2.3.1) + jwt (2.3.0) + memoist (0.16.2) method_source (0.8.2) - minitest (5.14.1) + minitest (5.15.0) multi_json (1.15.0) + multipart-post (2.1.1) newrelic_rpm (4.0.0.332) oj (3.10.8) + os (1.1.4) parallel (1.12.0) parser (2.4.0.2) ast (~> 2.3) @@ -67,11 +136,12 @@ GEM coderay (~> 1.1.0) method_source (~> 0.8.1) slop (~> 3.4) - public_suffix (4.0.6) + public_suffix (4.0.7) rainbow (2.2.2) rake rake (13.0.1) require_all (1.3.3) + rexml (3.2.5) rspec (3.5.0) rspec-core (~> 3.5.0) rspec-expectations (~> 3.5.0) @@ -93,31 +163,36 @@ GEM ruby-progressbar (~> 1.7) unicode-display_width (~> 1.0, >= 1.0.1) ruby-progressbar (1.9.0) - safe_yaml (1.0.5) + ruby2_keywords (0.0.5) + signet (0.16.1) + addressable (~> 2.8) + faraday (>= 0.17.5, < 3.0) + jwt (>= 1.5, < 3.0) + multi_json (~> 1.10) simplecov (0.12.0) docile (~> 1.1.0) json (>= 1.8, < 3) simplecov-html (~> 0.10.0) simplecov-html (0.10.0) slop (3.6.0) - thor (1.0.1) - thread_safe (0.3.6) + thor (1.2.1) tilt (2.0.10) - toiler (0.6.0) + toiler (0.7.0) aws-sdk-sqs (>= 1.0.0, < 2.0.0) concurrent-ruby (~> 1.0, >= 1.0.0) concurrent-ruby-edge (~> 0.3, >= 0.3) + google-cloud-pubsub (~> 2.9, >= 2.9.1) typhoeus (1.4.0) ethon (>= 0.9.0) - tzinfo (1.2.7) - thread_safe (~> 0.1) + tzinfo (2.0.4) + concurrent-ruby (~> 1.0) unicode-display_width (1.3.0) - vcr (6.0.0) - webmock (3.8.3) - addressable (>= 2.3.6) + vcr (6.1.0) + webmock (3.14.0) + addressable (>= 2.8.0) crack (>= 0.3.2) hashdiff (>= 0.4.0, < 2.0.0) - zeitwerk (2.4.0) + zeitwerk (2.5.4) PLATFORMS ruby @@ -137,4 +212,4 @@ DEPENDENCIES simplecov BUNDLED WITH - 2.1.2 + 2.3.10 diff --git a/lib/little_monster.rb b/lib/little_monster.rb index 0f1d8e2..433918d 100644 --- a/lib/little_monster.rb +++ b/lib/little_monster.rb @@ -42,6 +42,7 @@ def default_config_values api_url: 'http://little_monster_api_url.com/', worker_concurrency: 200, worker_queue: nil, + worker_provider: :aws, request_timeout: 3, default_request_retries: 4, default_request_retry_wait: 1, diff --git a/lib/little_monster/config.rb b/lib/little_monster/config.rb index fcfb705..3bd15c5 100644 --- a/lib/little_monster/config.rb +++ b/lib/little_monster/config.rb @@ -4,6 +4,7 @@ class Config attr_accessor :worker_concurrency attr_accessor :worker_queue + attr_accessor :worker_provider attr_accessor :formatter diff --git a/lib/little_monster/core/job_orchrestator.rb b/lib/little_monster/core/job_orchrestator.rb index 5c0b68f..4abb593 100644 --- a/lib/little_monster/core/job_orchrestator.rb +++ b/lib/little_monster/core/job_orchrestator.rb @@ -172,7 +172,14 @@ def abort_job(error) def handle_error(error) raise error if LittleMonster.env.development? - logger.error "[type:error] [error_type:#{error.class}][message:#{error.message.dump}] \n #{error.backtrace.to_a.join("\n\t")}" + if error.cause.nil? + logger.error "[type:error] [error_type:#{error.class}][message:#{error.message.dump}] \n #{error.backtrace.to_a.join("\n\t")}" + else + logger.error "[type:error] [error_type:#{error.class}][message:#{error.message.dump}]"\ + "\n\t#{error.backtrace.to_a.join("\n\t")}"\ + "\nCaused by:"\ + "\n\t#{error.cause.backtrace.to_a.join("\n\t")}" + end @job.error = @job.serialize_error error diff --git a/lib/little_monster/generators/templates/config/application.rb b/lib/little_monster/generators/templates/config/application.rb index 4f2be30..2125326 100644 --- a/lib/little_monster/generators/templates/config/application.rb +++ b/lib/little_monster/generators/templates/config/application.rb @@ -6,6 +6,8 @@ conf.default_request_retries = 3 # conf.worker_concurrency = 200 # conf.worker_queue = 'my_sqs_queue' + # conf.worker_provider = :aws + # conf.worker_provider = :gcp end require_relative "environments/#{LittleMonster.env}" diff --git a/lib/little_monster/generators/templates/config/toiler.yml b/lib/little_monster/generators/templates/config/toiler.yml index 2de5d59..f2bbcbf 100644 --- a/lib/little_monster/generators/templates/config/toiler.yml +++ b/lib/little_monster/generators/templates/config/toiler.yml @@ -1,3 +1,5 @@ aws: region: us-east-1 +# gcp: +# project_id: my-project wait: 20 diff --git a/lib/little_monster/version.rb b/lib/little_monster/version.rb index d7f83e6..35ea9a9 100644 --- a/lib/little_monster/version.rb +++ b/lib/little_monster/version.rb @@ -1,3 +1,3 @@ module LittleMonster - VERSION = '0.1.24'.freeze + VERSION = '0.1.25'.freeze end diff --git a/lib/little_monster/worker.rb b/lib/little_monster/worker.rb index 079cf17..b9f2e97 100644 --- a/lib/little_monster/worker.rb +++ b/lib/little_monster/worker.rb @@ -6,20 +6,26 @@ class Worker include ::Toiler::Worker toiler_options queue: LittleMonster.worker_queue, - concurrency: LittleMonster.worker_concurrency + concurrency: LittleMonster.worker_concurrency, + provider: LittleMonster.worker_provider - toiler_options auto_visibility_timeout: true, + toiler_options deadline_extension: true, auto_delete: true - toiler_options parser: MultiJson + toiler_options parser: ->(msg){ MultiJson.load msg.body, symbolize_keys: true } def self.update_attributes toiler_options queue: LittleMonster.worker_queue, - concurrency: LittleMonster.worker_concurrency + concurrency: LittleMonster.worker_concurrency, + provider: LittleMonster.worker_provider end def perform(_sqs_msg, body) - message = MultiJson.load body['Message'], symbolize_keys: true + if LittleMonster.worker_provider == :gcp + message = body + else + message = MultiJson.load body[:Message], symbolize_keys: true + end LittleMonster::Runner.new(message).run end diff --git a/spec/lib/little_monster/config_spec.rb b/spec/lib/little_monster/config_spec.rb index c1a577f..c604a01 100644 --- a/spec/lib/little_monster/config_spec.rb +++ b/spec/lib/little_monster/config_spec.rb @@ -9,6 +9,7 @@ api_request_retries: 100, worker_concurrency: 200, worker_queue: nil, + worker_provider: :aws, key: :value } end diff --git a/spec/lib/little_monster/worker_spec.rb b/spec/lib/little_monster/worker_spec.rb index 02ed56c..534963c 100644 --- a/spec/lib/little_monster/worker_spec.rb +++ b/spec/lib/little_monster/worker_spec.rb @@ -15,7 +15,8 @@ allow(described_class).to receive(:toiler_options) described_class.update_attributes expect(described_class).to have_received(:toiler_options).with(queue: LittleMonster.worker_queue, - concurrency: LittleMonster.worker_concurrency) + concurrency: LittleMonster.worker_concurrency, + provider: LittleMonster.worker_provider) end end