Merge pull request #1122 from PRX/limit-error-interface

Limit error interface

app/jobs/application_job.rb

@@ -21,7 +21,7 @@ def s3_bucket
     ENV["FEEDER_STORAGE_BUCKET"]
   end
 
-  def s3_client
+  def self.s3_client
     if Rails.env.test? || ENV["AWS_ACCESS_KEY_ID"].present?
       Aws::S3::Client.new(
         credentials: Aws::Credentials.new(ENV["AWS_ACCESS_KEY_ID"], ENV["AWS_SECRET_ACCESS_KEY"]),
@@ -31,4 +31,8 @@ def s3_client
       Aws::S3::Client.new
     end
   end
+
+  def s3_client
+    self.class.s3_client
+  end
 end

app/jobs/publish_apple_job.rb

@@ -5,12 +5,16 @@ def self.publish_to_apple(apple_config)
     apple_config.build_publisher.publish!
   end
 
-  def perform(apple_config)
+  def self.do_perform(apple_config)
     if !apple_config.publish_to_apple?
       logger.info "Skipping publish to apple for #{apple_config.class.name} #{apple_config.id}"
       return
     end
 
-    self.class.publish_to_apple(apple_config)
+    publish_to_apple(apple_config)
+  end
+
+  def perform(apple_config)
+    self.class.do_perform(apple_config)
   end
 end

app/jobs/publish_feed_job.rb

@@ -18,30 +18,47 @@ def perform(podcast, pub_item)
     podcast.feeds.each { |feed| publish_apple(podcast, feed) }
     podcast.feeds.each { |feed| publish_rss(podcast, feed) }
     PublishingPipelineState.complete!(podcast)
+  rescue Apple::AssetStateTimeoutError => e
+    fail_state(podcast, "apple_timeout", e)
   rescue => e
-    PublishingPipelineState.error!(podcast)
-    Rails.logger.error("Error publishing podcast", {podcast_id: podcast.id, error: e.message, backtrace: e.backtrace.join("\n")})
-    raise e
+    fail_state(podcast, "error", e)
   ensure
     PublishingPipelineState.settle_remaining!(podcast)
   end
 
   def publish_apple(podcast, feed)
     return unless feed.publish_to_apple?
-    res = PublishAppleJob.perform_now(feed.apple_config)
+
+    res = PublishAppleJob.do_perform(podcast.apple_config)
     PublishingPipelineState.publish_apple!(podcast)
     res
   rescue => e
-    NewRelic::Agent.notice_error(e)
-    res = PublishingPipelineState.error_apple!(podcast)
-    raise e if feed.apple_config.sync_blocks_rss?
-    res
+    if podcast.apple_config.sync_blocks_rss
+      fail_state(podcast, "apple", e)
+    else
+      Rails.logger.error("Error publishing to Apple, but continuing to publish RSS", {podcast_id: podcast.id, error: e.message})
+      PublishingPipelineState.error_apple!(podcast)
+    end
   end
 
   def publish_rss(podcast, feed)
     res = save_file(podcast, feed)
     PublishingPipelineState.publish_rss!(podcast)
     res
+  rescue => e
+    fail_state(podcast, "rss", e)
+  end
+
+  def fail_state(podcast, type, error)
+    (pipeline_method, log_level) = case type
+    when "apple" then [:error_apple!, :warn]
+    when "rss" then [:error_rss!, :warn]
+    when "apple_timeout", "error" then [:error!, :error]
+    end
+
+    PublishingPipelineState.public_send(pipeline_method, podcast)
+    Rails.logger.send(log_level, error.message, {podcast_id: podcast.id})
+    raise error
   end
 
   def save_file(podcast, feed, options = {})

app/models/apple/asset_state_timeout_error.rb

@@ -0,0 +1,16 @@
+module Apple
+  class AssetStateTimeoutError < RuntimeError
+    attr_reader :episodes, :attempts, :asset_wait_duration
+
+    def initialize(episodes)
+      @episodes = episodes
+      @attempts = episodes.map { |ep| ep.apple_episode_delivery_status.asset_processing_attempts }.max
+      @asset_wait_duration = episodes.map { |ep| ep.feeder_episode.measure_asset_processing_duration }.compact.max
+      super("Timeout waiting for asset state change: Episodes: #{episode_ids}, Attempts: #{attempts}, Asset Wait Duration: #{asset_wait_duration}")
+    end
+
+    def episode_ids
+      episodes.map(&:feeder_id)
+    end
+  end
+end

app/models/apple/publisher.rb

@@ -242,12 +242,11 @@ def wait_for_asset_state(eps)
       Rails.logger.tagged("##{__method__}") do
         eps = eps.filter { |e| e.podcast_delivery_files.any?(&:api_marked_as_uploaded?) }
 
-        (waiting_timed_out, _) = Apple::Episode.wait_for_asset_state(api, eps)
+        (waiting_timed_out, remaining_eps) = Apple::Episode.wait_for_asset_state(api, eps)
         if waiting_timed_out
-          attempts = eps.map { |ep| ep.apple_episode_delivery_status.asset_processing_attempts }.max
-          max_duration = eps.map { |ep| ep.feeder_episode.measure_asset_processing_duration }.compact.max
-          Rails.logger.info("Timed out waiting for asset state", {attempts: attempts, duration: max_duration, episode_count: eps.length})
-          raise "Timed out waiting for asset state"
+          e = Apple::AssetStateTimeoutError.new(remaining_eps)
+          Rails.logger.info("Timed out waiting for asset state", {attempts: e.attempts, episode_count: e.episodes.length, asset_wait_duration: e.asset_wait_duration})
+          raise e
         end
       end
     end

app/models/concerns/apple_delivery.rb

@@ -58,4 +58,24 @@ def measure_asset_processing_duration
 
     Time.now - start_status.created_at
   end
+
+  def apple_prepare_for_delivery!
+    # remove the previous delivery attempt (soft delete)
+    apple_podcast_deliveries.map(&:destroy)
+    apple_podcast_deliveries.reset
+    apple_podcast_delivery_files.reset
+    apple_podcast_container&.podcast_deliveries&.reset
+  end
+
+  def apple_mark_for_reupload!
+    apple_needs_delivery!
+  end
+
+  def apple_episode
+    return nil if !persisted? || !publish_to_apple?
+
+    if (show = podcast.apple_config&.build_publisher&.show)
+      Apple::Episode.new(api: show.api, show: show, feeder_episode: self)
+    end
+  end
 end

app/models/episode.rb

@@ -243,26 +243,6 @@ def copy_media(force = false)
     uncut&.copy_media(force)
   end
 
-  def apple_prepare_for_delivery!
-    # remove the previous delivery attempt (soft delete)
-    apple_podcast_deliveries.map(&:destroy)
-    apple_podcast_deliveries.reset
-    apple_podcast_delivery_files.reset
-    apple_podcast_container&.podcast_deliveries&.reset
-  end
-
-  def apple_mark_for_reupload!
-    apple_needs_delivery!
-  end
-
-  def apple_episode
-    return nil if !persisted? || !publish_to_apple?
-
-    if (show = podcast.apple_config&.build_publisher&.show)
-      Apple::Episode.new(api: show.api, show: show, feeder_episode: self)
-    end
-  end
-
   def publish!
     Rails.logger.tagged("Episode#publish!") do
       apple_mark_for_reupload!

app/models/publishing_pipeline_state.rb

@@ -52,7 +52,8 @@ class PublishingPipelineState < ApplicationRecord
     :complete,
     :error,
     :expired,
-    :error_apple
+    :error_apple,
+    :error_rss
   ]
 
   validate :podcast_ids_match
@@ -153,6 +154,10 @@ def self.error_apple!(podcast)
     state_transition(podcast, :error_apple)
   end
 
+  def self.error_rss!(podcast)
+    state_transition(podcast, :error_rss)
+  end
+
   def self.complete!(podcast)
     state_transition(podcast, :complete)
   end