Support for GCP PubSub (#67)

* support for gcp pubsub * update toiler * for gcp the message is not wrapped * fix aws message get * remove provider config * update toiler * use new toiler parameter * log exception causes * update lm gem * update toiler * use toiler 0.7.0 * increment version
mercadolibre · Apr 13, 2022 · 53ef2d6 · 53ef2d6
1 parent d1610a1
commit 53ef2d6
Show file tree

Hide file tree

Showing 10 changed files with 138 additions and 42 deletions.
diff --git a/Gemfile.lock b/Gemfile.lock
@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    little_monster (0.1.22)
+    little_monster (0.1.24)
       activesupport
       multi_json
       thor
@@ -14,51 +14,120 @@ PATH
 GEM
   remote: https://rubygems.org/
   specs:
-    activesupport (6.0.3.2)
+    activesupport (6.1.5)
       concurrent-ruby (~> 1.0, >= 1.0.2)
-      i18n (>= 0.7, < 2)
-      minitest (~> 5.1)
-      tzinfo (~> 1.1)
-      zeitwerk (~> 2.2, >= 2.2.2)
+      i18n (>= 1.6, < 2)
+      minitest (>= 5.1)
+      tzinfo (~> 2.0)
+      zeitwerk (~> 2.3)
     addressable (2.8.0)
       public_suffix (>= 2.0.2, < 5.0)
     ast (2.3.0)
-    aws-eventstream (1.1.0)
-    aws-partitions (1.380.0)
-    aws-sdk-core (3.109.0)
+    aws-eventstream (1.2.0)
+    aws-partitions (1.576.0)
+    aws-sdk-core (3.130.1)
       aws-eventstream (~> 1, >= 1.0.2)
-      aws-partitions (~> 1, >= 1.239.0)
+      aws-partitions (~> 1, >= 1.525.0)
       aws-sigv4 (~> 1.1)
       jmespath (~> 1.0)
-    aws-sdk-sqs (1.34.0)
-      aws-sdk-core (~> 3, >= 3.109.0)
+    aws-sdk-sqs (1.51.0)
+      aws-sdk-core (~> 3, >= 3.127.0)
       aws-sigv4 (~> 1.1)
-    aws-sigv4 (1.2.2)
+    aws-sigv4 (1.4.0)
       aws-eventstream (~> 1, >= 1.0.2)
     byebug (9.0.5)
     codeclimate-test-reporter (0.6.0)
       simplecov (>= 0.7.1, < 1.0.0)
     coderay (1.1.1)
-    concurrent-ruby (1.1.6)
+    concurrent-ruby (1.1.10)
     concurrent-ruby-edge (0.6.0)
       concurrent-ruby (~> 1.1.6)
-    crack (0.4.3)
-      safe_yaml (~> 1.0.0)
+    crack (0.4.5)
+      rexml
     diff-lcs (1.2.5)
     docile (1.1.5)
-    ethon (0.12.0)
-      ffi (>= 1.3.0)
-    ffi (1.13.1)
+    ethon (0.15.0)
+      ffi (>= 1.15.0)
+    faraday (1.10.0)
+      faraday-em_http (~> 1.0)
+      faraday-em_synchrony (~> 1.0)
+      faraday-excon (~> 1.1)
+      faraday-httpclient (~> 1.0)
+      faraday-multipart (~> 1.0)
+      faraday-net_http (~> 1.0)
+      faraday-net_http_persistent (~> 1.0)
+      faraday-patron (~> 1.0)
+      faraday-rack (~> 1.0)
+      faraday-retry (~> 1.0)
+      ruby2_keywords (>= 0.0.4)
+    faraday-em_http (1.0.0)
+    faraday-em_synchrony (1.0.0)
+    faraday-excon (1.1.0)
+    faraday-httpclient (1.0.1)
+    faraday-multipart (1.0.3)
+      multipart-post (>= 1.2, < 3)
+    faraday-net_http (1.0.1)
+    faraday-net_http_persistent (1.2.0)
+    faraday-patron (1.0.0)
+    faraday-rack (1.0.0)
+    faraday-retry (1.0.3)
+    ffi (1.15.5)
+    gapic-common (0.8.0)
+      faraday (~> 1.3)
+      google-protobuf (~> 3.14)
+      googleapis-common-protos (>= 1.3.11, < 2.a)
+      googleapis-common-protos-types (>= 1.0.6, < 2.a)
+      googleauth (>= 0.17.0, < 2.a)
+      grpc (~> 1.36)
+    google-cloud-core (1.6.0)
+      google-cloud-env (~> 1.0)
+      google-cloud-errors (~> 1.0)
+    google-cloud-env (1.6.0)
+      faraday (>= 0.17.3, < 3.0)
+    google-cloud-errors (1.2.0)
+    google-cloud-pubsub (2.9.1)
+      concurrent-ruby (~> 1.1)
+      google-cloud-core (~> 1.5)
+      google-cloud-pubsub-v1 (~> 0.0)
+    google-cloud-pubsub-v1 (0.8.0)
+      gapic-common (>= 0.7, < 2.a)
+      google-cloud-errors (~> 1.0)
+      grpc-google-iam-v1 (>= 0.6.10, < 2.a)
+    google-protobuf (3.20.0)
+    googleapis-common-protos (1.3.12)
+      google-protobuf (~> 3.14)
+      googleapis-common-protos-types (~> 1.2)
+      grpc (~> 1.27)
+    googleapis-common-protos-types (1.3.1)
+      google-protobuf (~> 3.14)
+    googleauth (1.1.2)
+      faraday (>= 0.17.3, < 3.a)
+      jwt (>= 1.4, < 3.0)
+      memoist (~> 0.16)
+      multi_json (~> 1.11)
+      os (>= 0.9, < 2.0)
+      signet (>= 0.16, < 2.a)
+    grpc (1.45.0)
+      google-protobuf (~> 3.19)
+      googleapis-common-protos-types (~> 1.0)
+    grpc-google-iam-v1 (1.1.0)
+      google-protobuf (~> 3.14)
+      googleapis-common-protos (>= 1.3.12, < 2.0)
+      grpc (~> 1.27)
     hashdiff (1.0.1)
-    i18n (1.8.5)
+    i18n (1.10.0)
       concurrent-ruby (~> 1.0)
-    jmespath (1.4.0)
+    jmespath (1.6.1)
     json (2.3.1)
+    jwt (2.3.0)
+    memoist (0.16.2)
     method_source (0.8.2)
-    minitest (5.14.1)
+    minitest (5.15.0)
     multi_json (1.15.0)
+    multipart-post (2.1.1)
     newrelic_rpm (4.0.0.332)
     oj (3.10.8)
+    os (1.1.4)
     parallel (1.12.0)
     parser (2.4.0.2)
       ast (~> 2.3)
@@ -67,11 +136,12 @@ GEM
       coderay (~> 1.1.0)
       method_source (~> 0.8.1)
       slop (~> 3.4)
-    public_suffix (4.0.6)
+    public_suffix (4.0.7)
     rainbow (2.2.2)
       rake
     rake (13.0.1)
     require_all (1.3.3)
+    rexml (3.2.5)
     rspec (3.5.0)
       rspec-core (~> 3.5.0)
       rspec-expectations (~> 3.5.0)
@@ -93,31 +163,36 @@ GEM
       ruby-progressbar (~> 1.7)
       unicode-display_width (~> 1.0, >= 1.0.1)
     ruby-progressbar (1.9.0)
-    safe_yaml (1.0.5)
+    ruby2_keywords (0.0.5)
+    signet (0.16.1)
+      addressable (~> 2.8)
+      faraday (>= 0.17.5, < 3.0)
+      jwt (>= 1.5, < 3.0)
+      multi_json (~> 1.10)
     simplecov (0.12.0)
       docile (~> 1.1.0)
       json (>= 1.8, < 3)
       simplecov-html (~> 0.10.0)
     simplecov-html (0.10.0)
     slop (3.6.0)
-    thor (1.0.1)
-    thread_safe (0.3.6)
+    thor (1.2.1)
     tilt (2.0.10)
-    toiler (0.6.0)
+    toiler (0.7.0)
       aws-sdk-sqs (>= 1.0.0, < 2.0.0)
       concurrent-ruby (~> 1.0, >= 1.0.0)
       concurrent-ruby-edge (~> 0.3, >= 0.3)
+      google-cloud-pubsub (~> 2.9, >= 2.9.1)
     typhoeus (1.4.0)
       ethon (>= 0.9.0)
-    tzinfo (1.2.7)
-      thread_safe (~> 0.1)
+    tzinfo (2.0.4)
+      concurrent-ruby (~> 1.0)
     unicode-display_width (1.3.0)
-    vcr (6.0.0)
-    webmock (3.8.3)
-      addressable (>= 2.3.6)
+    vcr (6.1.0)
+    webmock (3.14.0)
+      addressable (>= 2.8.0)
       crack (>= 0.3.2)
       hashdiff (>= 0.4.0, < 2.0.0)
-    zeitwerk (2.4.0)
+    zeitwerk (2.5.4)
 
 PLATFORMS
   ruby
@@ -137,4 +212,4 @@ DEPENDENCIES
   simplecov
 
 BUNDLED WITH
-   2.1.2
+   2.3.10
diff --git a/lib/little_monster.rb b/lib/little_monster.rb
@@ -42,6 +42,7 @@ def default_config_values
       api_url: 'http://little_monster_api_url.com/',
       worker_concurrency: 200,
       worker_queue: nil,
+      worker_provider: :aws,
       request_timeout: 3,
       default_request_retries: 4,
       default_request_retry_wait: 1,

diff --git a/lib/little_monster/config.rb b/lib/little_monster/config.rb
@@ -4,6 +4,7 @@ class Config
 
     attr_accessor :worker_concurrency
     attr_accessor :worker_queue
+    attr_accessor :worker_provider
 
     attr_accessor :formatter
 

diff --git a/lib/little_monster/core/job_orchrestator.rb b/lib/little_monster/core/job_orchrestator.rb
@@ -172,7 +172,14 @@ def abort_job(error)
 
     def handle_error(error)
       raise error if LittleMonster.env.development?
-      logger.error "[type:error] [error_type:#{error.class}][message:#{error.message.dump}] \n #{error.backtrace.to_a.join("\n\t")}"
+      if error.cause.nil?
+        logger.error "[type:error] [error_type:#{error.class}][message:#{error.message.dump}] \n #{error.backtrace.to_a.join("\n\t")}"
+      else
+        logger.error "[type:error] [error_type:#{error.class}][message:#{error.message.dump}]"\
+                     "\n\t#{error.backtrace.to_a.join("\n\t")}"\
+                     "\nCaused by:"\
+                     "\n\t#{error.cause.backtrace.to_a.join("\n\t")}"
+      end
 
       @job.error = @job.serialize_error error
 

diff --git a/lib/little_monster/generators/templates/config/application.rb b/lib/little_monster/generators/templates/config/application.rb
@@ -6,6 +6,8 @@
   conf.default_request_retries = 3
   # conf.worker_concurrency = 200
   # conf.worker_queue = 'my_sqs_queue'
+  # conf.worker_provider = :aws
+  # conf.worker_provider = :gcp
 end
 
 require_relative "environments/#{LittleMonster.env}"

diff --git a/lib/little_monster/generators/templates/config/toiler.yml b/lib/little_monster/generators/templates/config/toiler.yml
@@ -1,3 +1,5 @@
 aws: 
   region: us-east-1
+# gcp:
+#   project_id: my-project
 wait: 20
diff --git a/lib/little_monster/version.rb b/lib/little_monster/version.rb
@@ -1,3 +1,3 @@
 module LittleMonster
-  VERSION = '0.1.24'.freeze
+  VERSION = '0.1.25'.freeze
 end
diff --git a/lib/little_monster/worker.rb b/lib/little_monster/worker.rb
@@ -6,20 +6,26 @@ class Worker
     include ::Toiler::Worker
 
     toiler_options queue: LittleMonster.worker_queue,
-                   concurrency: LittleMonster.worker_concurrency
+                   concurrency: LittleMonster.worker_concurrency,
+                   provider: LittleMonster.worker_provider
 
-    toiler_options auto_visibility_timeout: true,
+    toiler_options deadline_extension: true,
                    auto_delete: true
 
-    toiler_options parser: MultiJson
+    toiler_options parser: ->(msg){ MultiJson.load msg.body, symbolize_keys: true }
 
     def self.update_attributes
       toiler_options queue: LittleMonster.worker_queue,
-                     concurrency: LittleMonster.worker_concurrency
+                     concurrency: LittleMonster.worker_concurrency,
+                     provider: LittleMonster.worker_provider
     end
 
     def perform(_sqs_msg, body)
-      message = MultiJson.load body['Message'], symbolize_keys: true
+      if LittleMonster.worker_provider == :gcp 
+        message = body
+      else
+        message = MultiJson.load body[:Message], symbolize_keys: true
+      end
 
       LittleMonster::Runner.new(message).run
     end

diff --git a/spec/lib/little_monster/config_spec.rb b/spec/lib/little_monster/config_spec.rb
@@ -9,6 +9,7 @@
           api_request_retries: 100,
           worker_concurrency: 200,
           worker_queue: nil,
+          worker_provider: :aws,
           key: :value
         }
       end

diff --git a/spec/lib/little_monster/worker_spec.rb b/spec/lib/little_monster/worker_spec.rb
@@ -15,7 +15,8 @@
       allow(described_class).to receive(:toiler_options)
       described_class.update_attributes
       expect(described_class).to have_received(:toiler_options).with(queue: LittleMonster.worker_queue,
-                                                                     concurrency: LittleMonster.worker_concurrency)
+                                                                     concurrency: LittleMonster.worker_concurrency,
+                                                                     provider: LittleMonster.worker_provider)
     end
   end