Fixed: Chart errors

lidarrmetadata/chart.py

@@ -16,7 +16,7 @@ async def _parse_itunes_chart(URL, count):
     async with aiohttp.ClientSession() as session:
         async with session.get(URL, timeout=aiohttp.ClientTimeout(total=5)) as response:
             json = await response.json()
-            results = filter(lambda r: r.get('kind', '') == 'album', json['feed']['results'])
+            results = filter(lambda r: r.get('kind', '') == 'albums', json['feed']['results'])
             search_provider = provider.get_providers_implementing(provider.AlbumNameSearchMixin)[0]
             search_results = []
             for result in results:
@@ -36,7 +36,7 @@ async def get_apple_music_top_albums_chart(count=10):
     :param count: Number of results to return. Defaults to 10
     :return: Chart response for itunes
     """
-    URL = 'https://rss.itunes.apple.com/api/v1/us/apple-music/top-albums/all/{count}/explicit.json'.format(
+    URL = 'https://rss.applemarketingtools.com/api/v2/us/music/most-played/{count}/albums.json'.format(
         count=4 * count)
     return await _parse_itunes_chart(URL, count)
 
@@ -138,23 +138,26 @@ async def get_lastfm_album_chart(count=10, user=None):
         # Try to stop lastfm from erroring out
         await asyncio.sleep(1)
 
-        # TODO Figure out a cleaner way to do this
-        rgid = await album_provider.map_query(
-            'SELECT release_group.gid '
-            'FROM release '
-            'JOIN release_group ON release_group.id = release.release_group '
-            'WHERE release.gid = $1 '
-            'LIMIT 1',
-            lastfm_album.item.get_mbid()
-        )
-
-        if rgid:
-            search_result = await _parse_album_search_result({'Id': rgid[0]['gid']})
-            if search_result:
-                albums.append(search_result)
-
-                if len(albums) == count:
-                    break
+        try:
+            # TODO Figure out a cleaner way to do this
+            rgid = await album_provider.map_query(
+                'SELECT release_group.gid '
+                'FROM release '
+                'JOIN release_group ON release_group.id = release.release_group '
+                'WHERE release.gid = $1 '
+                'LIMIT 1',
+                lastfm_album.item.get_mbid()
+            )
+
+            if rgid:
+                search_result = await _parse_album_search_result({'Id': rgid[0]['gid']})
+                if search_result:
+                    albums.append(search_result)
+
+                    if len(albums) == count:
+                        break
+        except:
+            pass
 
     if len(albums) > count:
         albums = albums[:count]

requirements.txt

@@ -6,7 +6,7 @@ async-timeout==3.0.1
 asyncpg==0.21.0
 attrs==20.3.0
 beautifulsoup4==4.9.3
-billboard.py==6.2.1
+billboard.py==7.0.0
 blinker==1.4
 certifi==2020.12.5
 chardet==3.0.4
@@ -26,7 +26,7 @@ Jinja2==3.0.0a1
 MarkupSafe==2.0.0a1
 multidict==5.1.0
 priority==1.3.0
-pylast==4.0.0
+pylast==4.3.0
 pytelegraf==0.3.3
 python-dateutil==2.8.1
 pytz==2020.4

tests/chart_test.py

@@ -0,0 +1,63 @@
+import pytest
+
+from lidarrmetadata import config
+from lidarrmetadata import chart
+from lidarrmetadata import provider
+
+class NullSearchProvider(provider.SolrSearchProvider):
+    """
+    This is a pretty sketchy way to test these charts, but some of them have a habit of changing. This will at
+    least allow us to ensure that they work without the complexity of end to end testing.
+    """
+    async def search_album_name(self, *args, **kwargs):
+        return []
+    async def search_artist_name(self, *args, **kwargs):
+            return []
+
+class NullRGProvider(provider.ReleaseGroupByIdMixin):
+    async def map_query(self, *args, **kwargs):
+        return []
+
+    async def get_release_group_id_from_spotify_id(self, spotify_id):
+        return None
+
+    async def get_release_groups_by_id(self, rgids):
+        return []
+
+    async def redirect_old_release_group_id(self, artist_id):
+        return None
+
+@pytest.fixture(scope='function')
+def patch_search_provider(monkeypatch):
+    before_patch = provider.get_providers_implementing
+    def patched_providers(x):
+        if x in NullSearchProvider.__mro__:
+            return [NullSearchProvider()]
+        elif x in NullRGProvider.__mro__:
+            return [NullRGProvider()]
+
+        return before_patch(x)
+
+    monkeypatch.setattr(provider, 'get_providers_implementing', patched_providers)
+
+@pytest.mark.asyncio
+async def test_billboard_200_albums_chart(patch_search_provider):
+    await chart.get_billboard_200_albums_chart()
+
+@pytest.mark.asyncio
+async def test_billboard_100_artists_chart(patch_search_provider):
+    await chart.get_billboard_100_artists_chart()
+
+@pytest.mark.asyncio
+async def test_apple_music_top_albums_chart(patch_search_provider):
+    await chart.get_apple_music_top_albums_chart()
+
+@pytest.mark.skipif(config.get_config().LASTFM_KEY == '', reason='No LastFM key available')
+@pytest.mark.asyncio
+async def test_lastfm_albums_chart(patch_search_provider):
+    await chart.get_lastfm_album_chart()
+
+@pytest.mark.skipif(config.get_config().LASTFM_KEY == '', reason='No LastFM key available')
+@pytest.mark.asyncio
+async def test_lastfm_artists_chart(patch_search_provider):
+    await chart.get_lastfm_artist_chart()

tests/e2e.js

@@ -0,0 +1,124 @@
+/**
+ * Post-deploy e2e testing by https://k6.io/. Once k6 is installed, just run
+ * k6 run tests/e2e.js
+ */
+import http from 'k6/http'
+import { check, group } from 'k6'
+
+const API_BASE = 'https://api.lidarr.audio/api/testing'
+
+function status200(response) {
+    return response.status == 200
+}
+
+function fetch(path) {
+    path = path === undefined ? '' : path
+    return http.get(`${API_BASE}/${path}`)
+}
+
+function testRoot() {
+    const res = fetch()
+    const data = res.json()
+    console.log(`Testing against version ${data.version} - ${data.branch}:${data.commit}`)
+    check(res, { 'API root returned HTTP 200': status200 })
+}
+
+function testBillboardTopAlbums() {
+    const res = fetch('chart/billboard/album/top')
+    check(res, { 'Charts - Billboard top albums returned HTTP 200': status200 })
+}
+
+function testBillboardTopArtists() {
+    const res = fetch('chart/billboard/artist/top')
+    check(res, { 'Charts - Billboard top artists returned HTTP 200': status200 })
+}
+
+function testBillboardCharts() {
+    testBillboardTopAlbums()
+    testBillboardTopArtists()
+}
+
+function testiTunesTopAlbums() {
+    const res = fetch('chart/itunes/album/top')
+    check(res, { 'Charts - iTunes top albums returned HTTP 200': status200 })
+}
+
+function testiTunesNewAlbums() {
+    const res = fetch('chart/itunes/album/new')
+    check(res, { 'Charts - iTunes new artists returned HTTP 200': status200 })
+}
+
+function testiTunesCharts() {
+    testiTunesTopAlbums()
+    testiTunesNewAlbums()
+}
+
+function testAppleMusicTopAlbums() {
+    const res = fetch('chart/apple-music/album/top')
+    check(res, { 'Charts - Apple Music top albums returned HTTP 200': status200 })
+}
+
+function testAppleMusicNewAlbums() {
+    const res = fetch('chart/apple-music/album/new')
+    check(res, { 'Charts - Apple Music new albums returned HTTP 200': status200 })
+}
+
+function testAppleMusicCharts() {
+    testAppleMusicTopAlbums()
+    testAppleMusicNewAlbums()
+}
+
+function testLastFMTopAlbums() {
+    const res = fetch('chart/lastfm/album/top')
+    check(res, { 'Charts - LastFM top albums returned HTTP 200': status200 })
+}
+
+function testLastFMTopArtists() {
+    const res = fetch('chart/lastfm/artist/top')
+    check(res, { 'Charts - LastFM top artists returned HTTP 200': status200 })
+}
+
+function testLastFMCharts() {
+    testLastFMTopAlbums()
+    testLastFMTopArtists()
+}
+
+function testCharts() {
+    testBillboardCharts()
+    testiTunesCharts()
+    testAppleMusicCharts()
+    testLastFMCharts()
+}
+
+function testArtist() {
+    const res = fetch('artist/0743b15a-3c32-48c8-ad58-cb325350befa')
+    check(res, { 'Artist - Fetch artist by MBID returned HTTP 200': status200 })
+}
+
+function testAlbum() {
+    const res = fetch('album/0a467c9c-c9f7-3a86-bce1-c013e3bacd5f')
+    check(res, { 'Album - Fetch album by MBID returned HTTP 200': status200 })
+}
+
+function testAlbumSearch() {
+    const res = fetch('search/album?query=cheshire%20cat')
+    check(res, { 'Search - Search artist returned HTTP 200': status200 })
+}
+
+function testArtistSearch() {
+    const res = fetch('search/artist?query=blink-182')
+    check(res, { 'Search - Search album returned HTTP 200': status200 })
+}
+
+function testSearch() {
+    testAlbumSearch()
+    testArtistSearch()
+}
+
+export default function () {
+    testRoot()
+    group('Charts', testCharts)
+    group('Artist', testArtist)
+    group('Album', testAlbum)
+    group('Search', testSearch)
+}