From 7570b1fdecba1d713d8265a63b2826e973b56db3 Mon Sep 17 00:00:00 2001
From: Sam Vevang <sam.vevang@gmail.com>
Date: Tue, 22 Oct 2024 08:54:39 -0500
Subject: [PATCH 1/3] Asset processing count (#1116)

---
 app/models/apple/episode_delivery_status.rb   |  22 +-
 app/models/apple/publisher.rb                 |  27 ++-
 app/models/concerns/apple_delivery.rb         |  23 +-
 ...essing_count_to_episode_delivery_status.rb |   5 +
 db/schema.rb                                  |   4 +-
 .../apple_episode_delivery_status_factory.rb  |  14 ++
 test/factories/apple_episode_factory.rb       |  12 +-
 .../apple/episode_delivery_status_test.rb     | 199 ++++++++++++++++++
 test/models/apple/episode_test.rb             |   2 +-
 test/models/apple/publisher_test.rb           | 111 +++++++++-
 test/models/episode_test.rb                   |  46 +++-
 test/test_helper.rb                           |  13 ++
 12 files changed, 460 insertions(+), 18 deletions(-)
 create mode 100644 db/migrate/20241002215949_add_asset_processing_count_to_episode_delivery_status.rb
 create mode 100644 test/factories/apple_episode_delivery_status_factory.rb
 create mode 100644 test/models/apple/episode_delivery_status_test.rb

diff --git a/app/models/apple/episode_delivery_status.rb b/app/models/apple/episode_delivery_status.rb
index db1fcbb95..5781c180c 100644
--- a/app/models/apple/episode_delivery_status.rb
+++ b/app/models/apple/episode_delivery_status.rb
@@ -1,5 +1,25 @@
 module Apple
   class EpisodeDeliveryStatus < ApplicationRecord
-    belongs_to :episode, class_name: "::Episode"
+    belongs_to :episode, -> { with_deleted }, class_name: "::Episode"
+
+    def self.update_status(episode, attrs)
+      new_status = (episode.apple_episode_delivery_status&.dup || default_status(episode))
+      new_status.assign_attributes(attrs)
+      new_status.save!
+      episode.apple_episode_delivery_statuses.reset
+      new_status
+    end
+
+    def self.default_status(episode)
+      new(episode: episode)
+    end
+
+    def increment_asset_wait
+      self.class.update_status(episode, asset_processing_attempts: (asset_processing_attempts || 0) + 1)
+    end
+
+    def reset_asset_wait
+      self.class.update_status(episode, asset_processing_attempts: 0)
+    end
   end
 end
diff --git a/app/models/apple/publisher.rb b/app/models/apple/publisher.rb
index bfb84474a..a46ab425f 100644
--- a/app/models/apple/publisher.rb
+++ b/app/models/apple/publisher.rb
@@ -148,9 +148,12 @@ def deliver_and_publish!(eps)
           mark_delivery_files_uploaded!(eps)
 
           wait_for_upload_processing(eps)
+
+          increment_asset_wait!(eps)
           wait_for_asset_state(eps)
 
           publish_drafting!(eps)
+          reset_asset_wait!(eps)
 
           raise_delivery_processing_errors(eps)
         end
@@ -226,12 +229,26 @@ def wait_for_upload_processing(eps)
       end
     end
 
+    def increment_asset_wait!(eps)
+      Rails.logger.tagged("##{__method__}") do
+        eps = eps.filter { |e| e.podcast_delivery_files.any?(&:api_marked_as_uploaded?) }
+
+        # Mark the episodes as waiting again for asset processing
+        eps.each { |ep| ep.apple_episode_delivery_status.increment_asset_wait }
+      end
+    end
+
     def wait_for_asset_state(eps)
       Rails.logger.tagged("##{__method__}") do
         eps = eps.filter { |e| e.podcast_delivery_files.any?(&:api_marked_as_uploaded?) }
 
         (waiting_timed_out, _) = Apple::Episode.wait_for_asset_state(api, eps)
-        raise "Timed out waiting for asset state" if waiting_timed_out
+        if waiting_timed_out
+          attempts = eps.map { |ep| ep.apple_episode_delivery_status.asset_processing_attempts }.max
+          max_duration = eps.map { |ep| ep.feeder_episode.measure_asset_processing_duration }.compact.max
+          Rails.logger.info("Timed out waiting for asset state", {attempts: attempts, duration: max_duration, episode_count: eps.length})
+          raise "Timed out waiting for asset state"
+        end
       end
     end
 
@@ -363,10 +380,18 @@ def publish_drafting!(eps)
         eps = eps.select { |ep| ep.drafting? && ep.container_upload_complete? }
 
         res = Apple::Episode.publish(api, show, eps)
+        eps.each { |ep| ep.apple_episode_delivery_status.reset_asset_wait }
+
         Rails.logger.info("Published #{res.length} drafting episodes.")
       end
     end
 
+    def reset_asset_wait!(eps)
+      Rails.logger.tagged("##{__method__}") do
+        eps.each { |ep| ep.apple_episode_delivery_status.reset_asset_wait }
+      end
+    end
+
     # Not used in any of the polling or publish routines, but useful for
     # debugging.  This removes the audio container reference from the episode,
     # but leaves the podcast container intact.
diff --git a/app/models/concerns/apple_delivery.rb b/app/models/concerns/apple_delivery.rb
index 25a76c282..e1ffc3859 100644
--- a/app/models/concerns/apple_delivery.rb
+++ b/app/models/concerns/apple_delivery.rb
@@ -22,16 +22,15 @@ def publish_to_apple?
   end
 
   def apple_update_delivery_status(attrs)
-    new_status = (apple_episode_delivery_status&.dup || apple_episode_delivery_statuses.build)
-    new_status.assign_attributes(**attrs)
-    new_status.save!
+    Apple::EpisodeDeliveryStatus.update_status(self, attrs)
+  end
 
-    apple_episode_delivery_statuses.reset
-    new_status
+  def build_initial_delivery_status
+    Apple::EpisodeDeliveryStatus.default_status(self)
   end
 
   def apple_episode_delivery_status
-    apple_episode_delivery_statuses.order(created_at: :desc).first
+    apple_episode_delivery_statuses.order(created_at: :desc).first || build_initial_delivery_status
   end
 
   def apple_needs_delivery?
@@ -47,4 +46,16 @@ def apple_needs_delivery!
   def apple_has_delivery!
     apple_update_delivery_status(delivered: true)
   end
+
+  def measure_asset_processing_duration
+    statuses = apple_episode_delivery_statuses.to_a
+
+    last_status = statuses.shift
+    return nil unless last_status&.asset_processing_attempts.to_i.positive?
+
+    start_status = statuses.find { |status| status.asset_processing_attempts.to_i.zero? }
+    return nil unless start_status
+
+    Time.now - start_status.created_at
+  end
 end
diff --git a/db/migrate/20241002215949_add_asset_processing_count_to_episode_delivery_status.rb b/db/migrate/20241002215949_add_asset_processing_count_to_episode_delivery_status.rb
new file mode 100644
index 000000000..56aaa8151
--- /dev/null
+++ b/db/migrate/20241002215949_add_asset_processing_count_to_episode_delivery_status.rb
@@ -0,0 +1,5 @@
+class AddAssetProcessingCountToEpisodeDeliveryStatus < ActiveRecord::Migration[7.2]
+  def change
+    add_column :apple_episode_delivery_statuses, :asset_processing_attempts, :integer, default: 0, null: false
+  end
+end
diff --git a/db/schema.rb b/db/schema.rb
index ac678ceb8..235f715d7 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -10,9 +10,10 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema[7.2].define(version: 2024_09_28_150204) do
+ActiveRecord::Schema[7.2].define(version: 2024_10_02_215949) do
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
+  enable_extension "uuid-ossp"
 
   create_table "apple_configs", force: :cascade do |t|
     t.bigint "feed_id", null: false
@@ -35,6 +36,7 @@
     t.text "enclosure_url"
     t.integer "source_fetch_count", default: 0
     t.bigint "source_media_version_id"
+    t.integer "asset_processing_attempts", default: 0, null: false
     t.index ["episode_id", "created_at"], name: "index_apple_episode_delivery_statuses_on_episode_id_created_at", include: ["delivered", "id"]
     t.index ["episode_id"], name: "index_apple_episode_delivery_statuses_on_episode_id"
   end
diff --git a/test/factories/apple_episode_delivery_status_factory.rb b/test/factories/apple_episode_delivery_status_factory.rb
new file mode 100644
index 000000000..45a927b33
--- /dev/null
+++ b/test/factories/apple_episode_delivery_status_factory.rb
@@ -0,0 +1,14 @@
+FactoryBot.define do
+  factory :apple_episode_delivery_status, class: "Apple::EpisodeDeliveryStatus" do
+    association :episode
+
+    delivered { false }
+    asset_processing_attempts { 0 }
+    source_url { "http://example.com/audio.mp3" }
+    source_size { 1_048_576 } # 1 MB
+    source_filename { "episode_audio.mp3" }
+    enclosure_url { "http://cdn.example.com/audio.mp3" }
+    source_media_version_id { 1 }
+    source_fetch_count { 0 }
+  end
+end
diff --git a/test/factories/apple_episode_factory.rb b/test/factories/apple_episode_factory.rb
index 04d9c612b..ef8458758 100644
--- a/test/factories/apple_episode_factory.rb
+++ b/test/factories/apple_episode_factory.rb
@@ -11,7 +11,11 @@
 
     # set a complete episode factory varient
     factory :uploaded_apple_episode do
-      feeder_episode { create(:episode, apple_episode_delivery_statuses: [Apple::EpisodeDeliveryStatus.new(delivered: true)]) }
+      feeder_episode do
+        ep = create(:episode)
+        ep.apple_has_delivery!
+        ep
+      end
       transient do
         api_response do
           build(:apple_episode_api_response,
@@ -22,7 +26,11 @@
       after(:build) do |apple_episode, evaluator|
         container = create(:apple_podcast_container, episode: apple_episode.feeder_episode)
         delivery = create(:apple_podcast_delivery, episode: apple_episode.feeder_episode, podcast_container: container)
-        _delivery_file = create(:apple_podcast_delivery_file, delivery: delivery, episode: apple_episode.feeder_episode)
+        _delivery_file = create(:apple_podcast_delivery_file,
+          delivery: delivery,
+          episode: apple_episode.feeder_episode,
+          api_marked_as_uploaded: true,
+          upload_operations_complete: true)
 
         create(:content, episode: apple_episode.feeder_episode, position: 1, status: "complete")
         create(:content, episode: apple_episode.feeder_episode, position: 2, status: "complete")
diff --git a/test/models/apple/episode_delivery_status_test.rb b/test/models/apple/episode_delivery_status_test.rb
new file mode 100644
index 000000000..04cf8e3e0
--- /dev/null
+++ b/test/models/apple/episode_delivery_status_test.rb
@@ -0,0 +1,199 @@
+require "test_helper"
+
+class Apple::EpisodeDeliveryStatusTest < ActiveSupport::TestCase
+  describe Apple::EpisodeDeliveryStatus do
+    let(:episode) { create(:episode) }
+    let(:delivery_status) { create(:apple_episode_delivery_status, episode: episode) }
+
+    describe "associations" do
+      it "belongs to an episode" do
+        assert_equal episode, delivery_status.episode
+      end
+
+      it "can belong to deleted episodes" do
+        episode.destroy
+        assert_equal episode, delivery_status.episode
+        assert_difference "Apple::EpisodeDeliveryStatus.count", +1 do
+          episode.apple_needs_delivery!
+        end
+        assert_equal episode, episode.apple_episode_delivery_statuses.first.episode
+      end
+    end
+
+    describe "scopes" do
+      it "orders by created_at desc by default" do
+        old_status = create(:apple_episode_delivery_status, episode: episode, created_at: 2.days.ago)
+        new_status = create(:apple_episode_delivery_status, episode: episode, created_at: 1.day.ago)
+
+        assert_equal [new_status, old_status], episode.apple_episode_delivery_statuses.to_a
+      end
+    end
+
+    describe "default values" do
+      it "sets asset_processing_attempts to 0 by default" do
+        new_status = Apple::EpisodeDeliveryStatus.new
+        assert_equal 0, new_status.asset_processing_attempts
+      end
+    end
+
+    describe ".update_status" do
+      it "creates a new status when none exists" do
+        episode.apple_episode_delivery_statuses.destroy_all
+        assert_difference "Apple::EpisodeDeliveryStatus.count", 1 do
+          Apple::EpisodeDeliveryStatus.update_status(episode, delivered: true)
+        end
+      end
+
+      it "creates a new status even when one already exists" do
+        _existing_status = create(:apple_episode_delivery_status, episode: episode)
+        assert_difference "Apple::EpisodeDeliveryStatus.count", 1 do
+          Apple::EpisodeDeliveryStatus.update_status(episode, delivered: false)
+        end
+      end
+
+      it "updates attributes of the new status" do
+        new_status = Apple::EpisodeDeliveryStatus.update_status(episode,
+          delivered: true,
+          source_url: "http://example.com/audio.mp3",
+          source_size: 1024,
+          source_filename: "audio.mp3")
+
+        assert new_status.delivered
+        assert_equal "http://example.com/audio.mp3", new_status.source_url
+        assert_equal 1024, new_status.source_size
+        assert_equal "audio.mp3", new_status.source_filename
+      end
+
+      it "resets the episode's apple_episode_delivery_statuses association" do
+        episode.apple_episode_delivery_statuses.load
+        Apple::EpisodeDeliveryStatus.update_status(episode, delivered: true)
+        refute episode.apple_episode_delivery_statuses.loaded?
+      end
+    end
+
+    describe "Episode#apple_episode_delivery_status" do
+      it "returns the most recent status" do
+        _old_status = create(:apple_episode_delivery_status, episode: episode, created_at: 2.days.ago)
+        new_status = create(:apple_episode_delivery_status, episode: episode, created_at: 1.day.ago)
+
+        assert_equal new_status, episode.apple_episode_delivery_status
+      end
+    end
+
+    describe "Asset waits and counting" do
+      describe "#increment_asset_wait" do
+        it "increments the asset_processing_attempts count" do
+          initial_count = delivery_status.asset_processing_attempts || 0
+          new_status = delivery_status.increment_asset_wait
+          assert_equal initial_count + 1, new_status.asset_processing_attempts
+        end
+
+        it "creates a new status entry" do
+          assert delivery_status.asset_processing_attempts.zero?
+          assert_difference "Apple::EpisodeDeliveryStatus.count", 1 do
+            delivery_status.increment_asset_wait
+          end
+        end
+
+        it "maintains other attributes" do
+          delivery_status.update(delivered: true, source_url: "http://example.com/audio.mp3")
+          new_status = delivery_status.increment_asset_wait
+          assert new_status.delivered
+          assert_equal "http://example.com/audio.mp3", new_status.source_url
+        end
+      end
+
+      describe "#reset_asset_wait" do
+        it "resets the asset_processing_attempts count to zero" do
+          delivery_status.update(asset_processing_attempts: 5)
+          new_status = delivery_status.reset_asset_wait
+          assert_equal 0, new_status.asset_processing_attempts
+        end
+
+        it "creates a new status entry" do
+          assert delivery_status.asset_processing_attempts.zero?
+          assert_difference "Apple::EpisodeDeliveryStatus.count", 1 do
+            delivery_status.reset_asset_wait
+          end
+        end
+
+        it "maintains other attributes" do
+          delivery_status.update(delivered: true, source_url: "http://example.com/audio.mp3")
+          new_status = delivery_status.reset_asset_wait
+          assert new_status.delivered
+          assert_equal "http://example.com/audio.mp3", new_status.source_url
+        end
+      end
+    end
+  end
+
+  describe "#measure_asset_processing_duration" do
+    let(:episode) { create(:episode) }
+
+    before do
+      travel_to Time.now
+    end
+
+    after do
+      travel_back
+    end
+
+    it "returns nil when there are no delivery statuses" do
+      assert_nil episode.measure_asset_processing_duration
+    end
+
+    it "returns nil when the latest status has zero attempts" do
+      create(:apple_episode_delivery_status, episode: episode, asset_processing_attempts: 0, created_at: 1.hour.ago)
+      assert_nil episode.measure_asset_processing_duration
+    end
+
+    it "measures duration for contiguous increments" do
+      create(:apple_episode_delivery_status, episode: episode, asset_processing_attempts: 0, created_at: 5.hours.ago)
+      create(:apple_episode_delivery_status, episode: episode, asset_processing_attempts: 1, created_at: 4.hours.ago)
+      create(:apple_episode_delivery_status, episode: episode, asset_processing_attempts: 2, created_at: 3.hours.ago)
+      create(:apple_episode_delivery_status, episode: episode, asset_processing_attempts: 3, created_at: 2.hours.ago)
+      create(:apple_episode_delivery_status, episode: episode, asset_processing_attempts: 4, created_at: 1.hour.ago)
+
+      assert_equal 5, episode.reload.measure_asset_processing_duration / 1.hour
+    end
+
+    it "measures duration for non-contiguous increments" do
+      create(:apple_episode_delivery_status, episode: episode, asset_processing_attempts: 0, created_at: 3.hours.ago)
+      create(:apple_episode_delivery_status, episode: episode, asset_processing_attempts: 4, created_at: 2.hours.ago)
+      create(:apple_episode_delivery_status, episode: episode, asset_processing_attempts: 5, created_at: 1.hour.ago)
+
+      assert_equal 3, episode.measure_asset_processing_duration / 1.hour
+    end
+
+    it "handles reset attempts correctly" do
+      create(:apple_episode_delivery_status, episode: episode, asset_processing_attempts: 0, created_at: 5.hours.ago)
+      create(:apple_episode_delivery_status, episode: episode, asset_processing_attempts: 1, created_at: 4.hours.ago)
+      create(:apple_episode_delivery_status, episode: episode, asset_processing_attempts: 2, created_at: 3.hours.ago)
+      create(:apple_episode_delivery_status, episode: episode, asset_processing_attempts: 0, created_at: 2.hours.ago)  # reset
+      create(:apple_episode_delivery_status, episode: episode, asset_processing_attempts: 1, created_at: 1.hour.ago)
+
+      assert_equal 2, episode.measure_asset_processing_duration / 1.hour
+    end
+
+    it "returns nil when all attempts are zero" do
+      create(:apple_episode_delivery_status, episode: episode, asset_processing_attempts: 0, created_at: 2.hours.ago)
+      create(:apple_episode_delivery_status, episode: episode, asset_processing_attempts: 0, created_at: 1.hour.ago)
+
+      assert_nil episode.measure_asset_processing_duration
+    end
+
+    it "handles nil asset_processing_attempts correctly" do
+      create(:apple_episode_delivery_status, episode: episode, asset_processing_attempts: 1, created_at: 1.hour.ago)
+
+      assert_nil episode.measure_asset_processing_duration
+    end
+
+    it "returns correct duration when latest attempt is zero" do
+      create(:apple_episode_delivery_status, episode: episode, asset_processing_attempts: 0, created_at: 3.hours.ago)
+      create(:apple_episode_delivery_status, episode: episode, asset_processing_attempts: 1, created_at: 2.hours.ago)
+      create(:apple_episode_delivery_status, episode: episode, asset_processing_attempts: 0, created_at: 1.hour.ago)
+
+      assert_nil episode.measure_asset_processing_duration
+    end
+  end
+end
diff --git a/test/models/apple/episode_test.rb b/test/models/apple/episode_test.rb
index ca43fa524..30e8d8feb 100644
--- a/test/models/apple/episode_test.rb
+++ b/test/models/apple/episode_test.rb
@@ -178,7 +178,7 @@
     }
     it "should be true if the delivery status is nil or has nil attrs" do
       assert apple_episode.delivery_statuses.destroy_all
-      assert apple_episode.delivery_status.nil?
+      assert apple_episode.delivery_status.source_media_version_id.nil?
 
       assert_equal true, apple_episode.needs_media_version?
     end
diff --git a/test/models/apple/publisher_test.rb b/test/models/apple/publisher_test.rb
index c8d3885db..48ff43e5a 100644
--- a/test/models/apple/publisher_test.rb
+++ b/test/models/apple/publisher_test.rb
@@ -1,5 +1,3 @@
-# frozen_string_literal: true
-
 require "test_helper"
 
 describe Apple::Publisher do
@@ -13,6 +11,8 @@
     Apple::Publisher.new(api: apple_api, public_feed: public_feed, private_feed: private_feed)
   end
 
+  let(:publisher) { apple_publisher }
+
   before do
     stub_request(:get, "https://api.podcastsconnect.apple.com/v1/countriesAndRegions?limit=200")
       .to_return(status: 200, body: json_file(:apple_countries_and_regions), headers: {})
@@ -343,19 +343,55 @@
   end
 
   describe "#publish_drafting!" do
+    let(:episode1) { build(:uploaded_apple_episode, show: apple_publisher.show, api_response: build(:apple_episode_api_response, publishing_state: "DRAFTING")) }
+    let(:episode2) { build(:uploaded_apple_episode, show: apple_publisher.show, api_response: build(:apple_episode_api_response, publishing_state: "DRAFTING")) }
+    let(:episodes) { [episode1, episode2] }
+
     it "should call the episode publish drafting class method" do
-      ep = OpenStruct.new(drafting?: true, container_upload_complete?: true)
       mock = Minitest::Mock.new
-      mock.expect(:call, [], [apple_publisher.api, apple_publisher.show, [ep]])
+      mock.expect(:call, [], [Apple::Api, Apple::Show, episodes])
 
       Apple::Episode.stub(:publish, mock) do
-        apple_publisher.publish_drafting!([ep])
+        apple_publisher.publish_drafting!(episodes)
       end
 
       assert mock.verify
     end
   end
 
+  describe "#reset_asset_wait!" do
+    let(:episode1) { build(:uploaded_apple_episode, show: apple_publisher.show) }
+    let(:episode2) { build(:uploaded_apple_episode, show: apple_publisher.show) }
+    let(:episodes) { [episode1, episode2] }
+
+    it "should reset the asset processing attempts" do
+      episodes.each do |ep|
+        ep.feeder_episode.apple_update_delivery_status(asset_processing_attempts: 3)
+      end
+      assert_equal 3, episode1.delivery_status.asset_processing_attempts
+      assert_equal 3, episode2.delivery_status.asset_processing_attempts
+
+      apple_publisher.reset_asset_wait!(episodes)
+
+      assert_equal 0, episode1.delivery_status.asset_processing_attempts
+      assert_equal 0, episode2.delivery_status.asset_processing_attempts
+    end
+
+    it "should reset the asset processing attempts for non-drafting episodes" do
+      episodes.each do |ep|
+        ep.feeder_episode.apple_update_delivery_status(asset_processing_attempts: 3)
+      end
+
+      assert_equal 3, episode1.delivery_status.asset_processing_attempts
+      assert_equal 3, episode2.delivery_status.asset_processing_attempts
+
+      apple_publisher.reset_asset_wait!(episodes)
+
+      assert_equal 0, episode1.delivery_status.asset_processing_attempts
+      assert_equal 0, episode2.delivery_status.asset_processing_attempts
+    end
+  end
+
   describe "#wait_for_upload_processing" do
     it "should poll the podcast container state" do
       mock = Minitest::Mock.new
@@ -369,6 +405,71 @@
     end
   end
 
+  describe "#increment_asset_wait!" do
+    let(:episode1) { build(:uploaded_apple_episode, show: apple_publisher.show) }
+    let(:episode2) { build(:uploaded_apple_episode, show: apple_publisher.show) }
+    let(:episodes) { [episode1, episode2] }
+
+    it "should increment asset wait count for each episode" do
+      episodes.each do |ep|
+        assert_equal 0, ep.apple_episode_delivery_status.asset_processing_attempts
+      end
+
+      apple_publisher.increment_asset_wait!(episodes)
+
+      episodes.each do |ep|
+        assert_equal 1, ep.apple_episode_delivery_status.asset_processing_attempts
+      end
+    end
+
+    it "should only increment the episodes that are still waiting" do
+      assert 1, episode1.podcast_delivery_files.length
+      assert 1, episode2.podcast_delivery_files.length
+
+      episode2.podcast_delivery_files.first.stub(:api_marked_as_uploaded?, false) do
+        episode1.podcast_delivery_files.first.stub(:api_marked_as_uploaded?, true) do
+          apple_publisher.increment_asset_wait!(episodes)
+        end
+      end
+
+      assert_equal [1, 0], [episode1, episode2].map { |ep| ep.apple_episode_delivery_status.asset_processing_attempts }
+    end
+
+    it "logs a timeout message with correct information" do
+      travel_to Time.now do
+        # Set up the delivery statuses
+        eps = [episode1, episode2]
+        eps.map { |e| e.feeder_episode.apple_episode_delivery_statuses.map(&:destroy) }
+
+        # Here is the log of attempts
+        create(:apple_episode_delivery_status, episode: episode1.feeder_episode, asset_processing_attempts: 0, created_at: 4.hour.ago)
+        create(:apple_episode_delivery_status, episode: episode1.feeder_episode, asset_processing_attempts: 1, created_at: 3.hour.ago)
+        create(:apple_episode_delivery_status, episode: episode1.feeder_episode, asset_processing_attempts: 2, created_at: 2.hour.ago)
+        create(:apple_episode_delivery_status, episode: episode1.feeder_episode, asset_processing_attempts: 3, created_at: 1.hour.ago)
+        eps.map(&:feeder_episode).map(&:reload)
+
+        # now simulate the asset timeout
+        assert publisher
+        logs = capture_json_logs do
+          Apple::Episode.stub :wait_for_asset_state, [true, eps] do
+            error = assert_raises(RuntimeError) do
+              publisher.wait_for_asset_state(eps)
+            end
+            assert_equal "Timed out waiting for asset state", error.message
+          end
+        end
+
+        # look at the logs
+        log = logs[0]
+        assert_equal "Timed out waiting for asset state", log[:msg]
+        assert_equal 30, log[:level]
+        assert_equal 3, log[:attempts]
+        assert_equal 4 * 60 * 60, log[:duration]
+        assert_equal 2, log[:episode_count]
+      end
+    end
+  end
+
   describe "#raise_delivery_processing_errors" do
     let(:apple_episode) { build(:apple_episode, show: apple_publisher.show) }
     let(:asset_processing_state) { "COMPLETED" }
diff --git a/test/models/episode_test.rb b/test/models/episode_test.rb
index ce5ca3130..0ef8f4d8e 100644
--- a/test/models/episode_test.rb
+++ b/test/models/episode_test.rb
@@ -458,7 +458,7 @@
   describe "#apple_needs_delivery?" do
     let(:episode) { create(:episode) }
     it "is true by default" do
-      assert_nil episode.apple_episode_delivery_status
+      refute episode.apple_episode_delivery_status.persisted?
       assert episode.apple_needs_delivery?
     end
 
@@ -493,4 +493,48 @@
       refute_nil episode.ready_transcript
     end
   end
+
+  describe "#increment_asset_wait" do
+    let(:episode) { create(:episode) }
+
+    it "creates a new status with incremented asset_processing_attempts" do
+      assert_difference -> { episode.apple_episode_delivery_statuses.count }, 1 do
+        new_status = episode.apple_status.increment_asset_wait
+        assert_equal 1, new_status.asset_processing_attempts
+      end
+    end
+
+    it "increments existing asset_processing_attempts" do
+      create(:apple_episode_delivery_status, episode: episode, asset_processing_attempts: 2)
+      new_status = episode.apple_status.increment_asset_wait
+      assert_equal 3, new_status.asset_processing_attempts
+    end
+
+    it "maintains other attributes when incrementing" do
+      create(:apple_episode_delivery_status,
+        episode: episode,
+        delivered: true,
+        source_url: "http://example.com/audio.mp3",
+        asset_processing_attempts: 1)
+
+      new_status = episode.apple_status.increment_asset_wait
+      assert_equal 2, new_status.asset_processing_attempts
+      assert new_status.delivered
+      assert_equal "http://example.com/audio.mp3", new_status.source_url
+    end
+
+    it "creates a new status with asset_processing_attempts set to 1 if no previous status exists" do
+      episode.apple_episode_delivery_statuses.destroy_all
+      assert_difference -> { episode.apple_episode_delivery_statuses.count }, 1 do
+        new_status = episode.apple_status.increment_asset_wait
+        assert_equal 1, new_status.asset_processing_attempts
+      end
+    end
+
+    it "returns the new status" do
+      result = episode.apple_status.increment_asset_wait
+      assert_instance_of Apple::EpisodeDeliveryStatus, result
+      assert_equal episode.apple_episode_delivery_statuses.last, result
+    end
+  end
 end
diff --git a/test/test_helper.rb b/test/test_helper.rb
index 0efc19100..5e27ee50f 100644
--- a/test/test_helper.rb
+++ b/test/test_helper.rb
@@ -49,6 +49,19 @@ class Minitest::Spec
   include FactoryBot::Syntax::Methods
 end
 
+def capture_json_logs(&block)
+  old_logger = Rails.logger
+  log = StringIO.new
+  feeder_logger = FeederLogger.new(log)
+  feeder_logger.formatter = Ougai::Formatters::Bunyan.new
+  Rails.logger = ActiveSupport::TaggedLogging.new(feeder_logger)
+  block.call
+  log.rewind
+  log.read.split("\n").map { |line| JSON.parse(line).with_indifferent_access }
+ensure
+  Rails.logger = old_logger
+end
+
 def json_file(name)
   test_file("/fixtures/#{name}.json")
 end

From bf63a528d07dc1c2702940de13fe7716b2a15535 Mon Sep 17 00:00:00 2001
From: cavis <ryancavis@gmail.com>
Date: Tue, 22 Oct 2024 08:13:50 -0600
Subject: [PATCH 2/3] API endpoints for old CMS data

---
 app/controllers/api/episodes_controller.rb |  5 +++++
 app/controllers/api/podcasts_controller.rb | 11 +++++++++++
 config/routes.rb                           |  3 +++
 3 files changed, 19 insertions(+)

diff --git a/app/controllers/api/episodes_controller.rb b/app/controllers/api/episodes_controller.rb
index c7d50431e..af1571e45 100644
--- a/app/controllers/api/episodes_controller.rb
+++ b/app/controllers/api/episodes_controller.rb
@@ -51,6 +51,11 @@ def show_resource
       resource = Episode.find_by_item_guid(params[:id])
       raise HalApi::Errors::NotFound.new if resource.nil?
 
+      @episode = resource
+    elsif params[:story_id]
+      resource = Episode.find_by(prx_uri: "/api/v1/stories/#{params[:story_id]}")
+      raise HalApi::Errors::NotFound.new if resource.nil?
+
       @episode = resource
     end
 
diff --git a/app/controllers/api/podcasts_controller.rb b/app/controllers/api/podcasts_controller.rb
index 4d6a333ea..f1c8f83b7 100644
--- a/app/controllers/api/podcasts_controller.rb
+++ b/app/controllers/api/podcasts_controller.rb
@@ -43,6 +43,17 @@ def visible?
 
   private
 
+  def show_resource
+    if params[:series_id]
+      resource = Podcast.find_by(prx_uri: "/api/v1/series/#{params[:series_id]}")
+      raise HalApi::Errors::NotFound.new if resource.nil?
+
+      @podcast = resource
+    end
+
+    super
+  end
+
   def find_base
     super.with_deleted
   end
diff --git a/config/routes.rb b/config/routes.rb
index 14ba8f1c3..13510e6d8 100644
--- a/config/routes.rb
+++ b/config/routes.rb
@@ -35,6 +35,9 @@
       resources :episodes, except: [:new, :edit]
       resources :feeds, only: [:index]
 
+      get "comatose/series/:series_id", to: "podcasts#show"
+      get "comatose/stories/:story_id", to: "episodes#show"
+
       root to: "base#entrypoint"
       match "*any", via: [:options], to: "base#options"
 

From 2376d8dd630713529f6ee6f6e0bf91609e097f54 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Wed, 23 Oct 2024 23:15:40 +0000
Subject: [PATCH 3/3] Bump rails from 7.2.1.1 to 7.2.1.2

Bumps [rails](https://github.com/rails/rails) from 7.2.1.1 to 7.2.1.2.
- [Release notes](https://github.com/rails/rails/releases)
- [Commits](https://github.com/rails/rails/compare/v7.2.1.1...v7.2.1.2)

---
updated-dependencies:
- dependency-name: rails
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 Gemfile.lock | 112 +++++++++++++++++++++++++--------------------------
 1 file changed, 56 insertions(+), 56 deletions(-)

diff --git a/Gemfile.lock b/Gemfile.lock
index 9d6238968..425ade286 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -14,29 +14,29 @@ GIT
 GEM
   remote: https://rubygems.org/
   specs:
-    actioncable (7.2.1.1)
-      actionpack (= 7.2.1.1)
-      activesupport (= 7.2.1.1)
+    actioncable (7.2.1.2)
+      actionpack (= 7.2.1.2)
+      activesupport (= 7.2.1.2)
       nio4r (~> 2.0)
       websocket-driver (>= 0.6.1)
       zeitwerk (~> 2.6)
-    actionmailbox (7.2.1.1)
-      actionpack (= 7.2.1.1)
-      activejob (= 7.2.1.1)
-      activerecord (= 7.2.1.1)
-      activestorage (= 7.2.1.1)
-      activesupport (= 7.2.1.1)
+    actionmailbox (7.2.1.2)
+      actionpack (= 7.2.1.2)
+      activejob (= 7.2.1.2)
+      activerecord (= 7.2.1.2)
+      activestorage (= 7.2.1.2)
+      activesupport (= 7.2.1.2)
       mail (>= 2.8.0)
-    actionmailer (7.2.1.1)
-      actionpack (= 7.2.1.1)
-      actionview (= 7.2.1.1)
-      activejob (= 7.2.1.1)
-      activesupport (= 7.2.1.1)
+    actionmailer (7.2.1.2)
+      actionpack (= 7.2.1.2)
+      actionview (= 7.2.1.2)
+      activejob (= 7.2.1.2)
+      activesupport (= 7.2.1.2)
       mail (>= 2.8.0)
       rails-dom-testing (~> 2.2)
-    actionpack (7.2.1.1)
-      actionview (= 7.2.1.1)
-      activesupport (= 7.2.1.1)
+    actionpack (7.2.1.2)
+      actionview (= 7.2.1.2)
+      activesupport (= 7.2.1.2)
       nokogiri (>= 1.8.5)
       racc
       rack (>= 2.2.4, < 3.2)
@@ -47,15 +47,15 @@ GEM
       useragent (~> 0.16)
     actionpack-action_caching (1.2.2)
       actionpack (>= 4.0.0)
-    actiontext (7.2.1.1)
-      actionpack (= 7.2.1.1)
-      activerecord (= 7.2.1.1)
-      activestorage (= 7.2.1.1)
-      activesupport (= 7.2.1.1)
+    actiontext (7.2.1.2)
+      actionpack (= 7.2.1.2)
+      activerecord (= 7.2.1.2)
+      activestorage (= 7.2.1.2)
+      activesupport (= 7.2.1.2)
       globalid (>= 0.6.0)
       nokogiri (>= 1.8.5)
-    actionview (7.2.1.1)
-      activesupport (= 7.2.1.1)
+    actionview (7.2.1.2)
+      activesupport (= 7.2.1.2)
       builder (~> 3.1)
       erubi (~> 1.11)
       rails-dom-testing (~> 2.2)
@@ -63,14 +63,14 @@ GEM
     active_link_to (1.0.5)
       actionpack
       addressable
-    activejob (7.2.1.1)
-      activesupport (= 7.2.1.1)
+    activejob (7.2.1.2)
+      activesupport (= 7.2.1.2)
       globalid (>= 0.3.6)
-    activemodel (7.2.1.1)
-      activesupport (= 7.2.1.1)
-    activerecord (7.2.1.1)
-      activemodel (= 7.2.1.1)
-      activesupport (= 7.2.1.1)
+    activemodel (7.2.1.2)
+      activesupport (= 7.2.1.2)
+    activerecord (7.2.1.2)
+      activemodel (= 7.2.1.2)
+      activesupport (= 7.2.1.2)
       timeout (>= 0.4.0)
     activerecord-session_store (2.0.0)
       actionpack (>= 5.2.4.1)
@@ -78,13 +78,13 @@ GEM
       multi_json (~> 1.11, >= 1.11.2)
       rack (>= 2.0.8, < 3)
       railties (>= 5.2.4.1)
-    activestorage (7.2.1.1)
-      actionpack (= 7.2.1.1)
-      activejob (= 7.2.1.1)
-      activerecord (= 7.2.1.1)
-      activesupport (= 7.2.1.1)
+    activestorage (7.2.1.2)
+      actionpack (= 7.2.1.2)
+      activejob (= 7.2.1.2)
+      activerecord (= 7.2.1.2)
+      activesupport (= 7.2.1.2)
       marcel (~> 1.0)
-    activesupport (7.2.1.1)
+    activesupport (7.2.1.2)
       base64
       bigdecimal
       concurrent-ruby (~> 1.0, >= 1.3.1)
@@ -298,7 +298,7 @@ GEM
     mutex_m (0.2.0)
     net-http (0.3.1)
       uri
-    net-imap (0.4.17)
+    net-imap (0.5.0)
       date
       net-protocol
     net-pop (0.1.2)
@@ -568,23 +568,23 @@ GEM
       rack (< 3)
     rack-test (2.1.0)
       rack (>= 1.3)
-    rackup (1.0.0)
+    rackup (1.0.1)
       rack (< 3)
       webrick
-    rails (7.2.1.1)
-      actioncable (= 7.2.1.1)
-      actionmailbox (= 7.2.1.1)
-      actionmailer (= 7.2.1.1)
-      actionpack (= 7.2.1.1)
-      actiontext (= 7.2.1.1)
-      actionview (= 7.2.1.1)
-      activejob (= 7.2.1.1)
-      activemodel (= 7.2.1.1)
-      activerecord (= 7.2.1.1)
-      activestorage (= 7.2.1.1)
-      activesupport (= 7.2.1.1)
+    rails (7.2.1.2)
+      actioncable (= 7.2.1.2)
+      actionmailbox (= 7.2.1.2)
+      actionmailer (= 7.2.1.2)
+      actionpack (= 7.2.1.2)
+      actiontext (= 7.2.1.2)
+      actionview (= 7.2.1.2)
+      activejob (= 7.2.1.2)
+      activemodel (= 7.2.1.2)
+      activerecord (= 7.2.1.2)
+      activestorage (= 7.2.1.2)
+      activesupport (= 7.2.1.2)
       bundler (>= 1.15.0)
-      railties (= 7.2.1.1)
+      railties (= 7.2.1.2)
     rails-controller-testing (1.0.5)
       actionpack (>= 5.0.1.rc1)
       actionview (>= 5.0.1.rc1)
@@ -596,9 +596,9 @@ GEM
     rails-html-sanitizer (1.6.0)
       loofah (~> 2.21)
       nokogiri (~> 1.14)
-    railties (7.2.1.1)
-      actionpack (= 7.2.1.1)
-      activesupport (= 7.2.1.1)
+    railties (7.2.1.2)
+      actionpack (= 7.2.1.2)
+      activesupport (= 7.2.1.2)
       irb (~> 1.13)
       rackup (>= 1.0.0)
       rake (>= 12.2)
@@ -738,7 +738,7 @@ GEM
     websocket-extensions (0.1.5)
     xpath (3.2.0)
       nokogiri (~> 1.8)
-    zeitwerk (2.7.0)
+    zeitwerk (2.7.1)
 
 PLATFORMS
   aarch64-linux