Add support for downloading Firefox on Android (Fenix) builds (#632)

.gitignore

@@ -4,5 +4,7 @@
 *.egg-info
 *.pyc
 .coverage
+.pytest_cache/
+.vscode/
 build
 dist

README.md

@@ -111,8 +111,8 @@ All those scraper instances allow you to retrieve the url which is used to downl
 ```python
 from mozdownload import FactoryScraper
 scraper = FactoryScraper('daily')
-print scraper.url
-print scraper.filename
+print(scraper.url)
+print(scraper.filename)
 ```
 
 To actually download the remote file the download() method has to be called:

mozdownload/factory.py

@@ -63,8 +63,8 @@ def __init__(self, scraper_type, **kwargs):
  if scraper_type in ('try') and not kwargs.get('revision'):
  raise ValueError('The revision of the build has to be specified.')
 
- if kwargs.get('application') == 'fennec' and scraper_type not in ('daily'):
- error_msg = '%s build is not yet supported for fennec' % scraper_type
+ if kwargs.get('application') == 'fenix' and scraper_type not in ('daily', 'release'):
+ error_msg = '%s build is not yet supported for fenix' % scraper_type
  raise NotSupportedError(error_msg)
 
  if (kwargs.get('application') == 'devedition'

mozdownload/scraper.py

@@ -24,44 +24,56 @@
 from mozdownload.timezones import PacificTimezone
 from mozdownload.utils import urljoin
 
-APPLICATIONS = ('devedition', 'firefox', 'fennec', 'thunderbird')
+APPLICATIONS = ('devedition', 'firefox', 'fenix', 'thunderbird')
 
 # Some applications contain all locales in a single build
-APPLICATIONS_MULTI_LOCALE = ('fennec')
+APPLICATIONS_MULTI_LOCALE = ('fenix')
 
-# Used if the application is named differently than the subfolder on the server
-APPLICATIONS_TO_FTP_DIRECTORY = {'fennec': 'mobile'}
-# Used if the application is named differently then the binary on the server
+# Used if the application is named differently than the binary on the server
 APPLICATIONS_TO_BINARY_NAME = {'devedition': 'firefox'}
 # Used when sorting versions
 APPLICATIONS_TO_VERSION_CLASS = {'devedition': 'DeveditionVersion',
  'firefox': 'FirefoxVersion',
- 'fennec': 'FennecVersion',
+ 'fenix': 'MobileVersion',
  'thunderbird': 'ThunderbirdVersion'}
 
+APPLICATION_BUILD_FILENAME = {
+ app: (
+ '%(TIMESTAMP)s-%(BRANCH)s-%(NAME)s' if app != 'fenix'
+ else '%(TIMESTAMP)s-%(NAME)s'
+ )
+ for app in APPLICATIONS
+}
+
+APPLICATION_REGEX = {
+ app: (
+ r'%(DATE)s-(\d+-)+%(BRANCH)s%(L10N)s%(PLATFORM)s$' if app != 'fenix'
+ else r'%(DATE)s-(\d+-){3}fenix-(\d+\.\d.\d)%(PLATFORM)s$'
+ )
+ for app in APPLICATIONS
+}
+
 # Base URL for the path to all builds
 BASE_URL = 'https://archive.mozilla.org/pub/'
 
 # Chunk size when downloading a file
 CHUNK_SIZE = 16 * 1024
 
-DEFAULT_FILE_EXTENSIONS = {'android-api-9': 'apk',
- 'android-api-11': 'apk',
- 'android-api-15': 'apk',
- 'android-api-16': 'apk',
+DEFAULT_FILE_EXTENSIONS = {'android-arm64-v8a': 'apk',
+ 'android-armeabi-v7a': 'apk',
  'android-x86': 'apk',
+ 'android-x86_64': 'apk',
  'linux': 'tar.bz2',
  'linux64': 'tar.bz2',
  'mac': 'dmg',
  'mac64': 'dmg',
  'win32': 'exe',
  'win64': 'exe'}
 
-PLATFORM_FRAGMENTS = {'android-api-9': r'android-arm',
- 'android-api-11': r'android-arm',
- 'android-api-15': r'android-arm',
- 'android-api-16': r'android-arm',
- 'android-x86': r'android-i386',
+PLATFORM_FRAGMENTS = {'android-arm64-v8a': r'android-arm64-v8a',
+ 'android-armeabi-v7a': r'android-armeabi-v7a',
+ 'android-x86': r'android-x86',
+ 'android-x86_64': r'android-x86_64',
  'linux': r'linux-i686',
  'linux64': r'linux-x86_64',
  'mac': r'mac',
@@ -138,10 +150,7 @@ def __init__(self, destination=None, platform=None,
 
  # build the base URL
  self.application = application
- self.base_url = '%s/' % urljoin(
- base_url,
- APPLICATIONS_TO_FTP_DIRECTORY.get(self.application, self.application)
- )
+ self.base_url = '%s/' % urljoin(base_url, self.application)
 
  if extension:
  self.extension = extension
@@ -377,10 +386,7 @@ def get_build_info(self):
  """Define additional build information."""
  # Retrieve build by revision
  if self.revision:
- th = treeherder.Treeherder(
- APPLICATIONS_TO_FTP_DIRECTORY.get(self.application, self.application),
- self.branch,
- self.platform)
+ th = treeherder.Treeherder(self.application, self.branch, self.platform)
  builds = th.query_builds_by_revision(
  self.revision,
  job_type_name='L10n Nightly' if self.locale_build else 'Nightly')
@@ -432,8 +438,23 @@ def get_build_info(self):
 
  def get_latest_build_date(self):
  """Return date of latest available nightly build."""
- if self.application not in ('fennec'):
+ if self.application not in ('fenix'):
  url = urljoin(self.base_url, 'nightly', 'latest-%s/' % self.branch)
+ elif self.application == 'fenix':
+ years = self._create_directory_parser(urljoin(self.base_url, 'nightly/'))
+ years.entries.sort()
+ months = self._create_directory_parser(urljoin(self.base_url, 'nightly',
+ years.entries[-1] + '/'))
+ months.entries.sort()
+
+ url = urljoin(self.base_url, 'nightly', years.entries[-1],
+ months.entries[-1] + '/')
+ parser = self._create_directory_parser(url)
+ parser.entries = parser.filter(r'.*%s' % self.platform_regex)
+ parser.entries.sort()
+
+ date = ''.join(parser.entries[-1].split('-')[:6])
+ return datetime.strptime(date, '%Y%m%d%H%M%S')
  else:
  url = urljoin(self.base_url, 'nightly', 'latest-%s-%s/' %
  (self.branch, self.platform))
@@ -487,15 +508,13 @@ def get_build_info_for_date(self, date, build_index=None):
 
  self.logger.info('Retrieving list of builds from %s' % url)
  parser = self._create_directory_parser(url)
- regex = r'%(DATE)s-(\d+-)+%(BRANCH)s%(L10N)s%(PLATFORM)s$' % {
+ regex = APPLICATION_REGEX[self.application] % {
  'DATE': date.strftime('%Y-%m-%d'),
  'BRANCH': self.branch,
  # ensure to select the correct subfolder for localized builds
  'L10N': '(-l10n)?' if self.locale_build else '',
- 'PLATFORM': '' if self.application not in (
- 'fennec') else '-' + self.platform
+ 'PLATFORM': '' if self.application not in ('fenix') else '-' + self.platform
  }
-
  parser.entries = parser.filter(regex)
  parser.entries = parser.filter(self.is_build_dir)
 
@@ -530,11 +549,10 @@ def binary_regex(self):
  """Return the regex for the binary."""
  regex_base_name = (r'^%(BINARY_NAME)s(\s%(STUB_NEW)s\.%(LOCALE)s|' +
  r'-.*\.%(LOCALE)s\.%(PLATFORM)s)')
- regex_suffix = {'android-api-9': r'\.%(EXT)s$',
- 'android-api-11': r'\.%(EXT)s$',
- 'android-api-15': r'\.%(EXT)s$',
- 'android-api-16': r'\.%(EXT)s$',
+ regex_suffix = {'android-arm64-v8a': r'\.%(EXT)s$',
+ 'android-armeabi-v7a': r'\.%(EXT)s$',
  'android-x86': r'\.%(EXT)s$',
+ 'android-x86_64': r'\.%(EXT)s$',
  'linux': r'\.%(EXT)s$',
  'linux64': r'\.%(EXT)s$',
  'mac': r'\.%(EXT)s$',
@@ -561,7 +579,7 @@ def build_filename(self, binary):
  # If it's not available use the build's date
  timestamp = self.date.strftime('%Y-%m-%d')
 
- return '%(TIMESTAMP)s-%(BRANCH)s-%(NAME)s' % {
+ return APPLICATION_BUILD_FILENAME[self.application] % {
  'TIMESTAMP': timestamp,
  'BRANCH': self.branch,
  'NAME': binary}
@@ -640,10 +658,16 @@ def binary_regex(self):
  r'^%(BINARY_NAME)s(%(STUB_NEW)s|(?:\sSetup\s|-)%(STUB)s%(VERSION)s)\.%(EXT)s$',
  'win64':
  r'^%(BINARY_NAME)s(%(STUB_NEW)s|(?:\sSetup\s|-)%(STUB)s%(VERSION)s)\.%(EXT)s$',
+ 'android-arm64-v8a': r'^%(BINARY_NAME)s-%(VERSION)s\.multi.%(PLATFORM)s\.%(EXT)s$',
+ 'android-armeabi-v7a':
+ r'^%(BINARY_NAME)s-%(VERSION)s\.multi.%(PLATFORM)s\.%(EXT)s$',
+ 'android-x86': r'^%(BINARY_NAME)s-%(VERSION)s\.multi.%(PLATFORM)s\.%(EXT)s$',
+ 'android-x86_64': r'^%(BINARY_NAME)s-%(VERSION)s\.multi.%(PLATFORM)s\.%(EXT)s$',
  }
  return regex[self.platform] % {
  'BINARY_NAME': APPLICATIONS_TO_BINARY_NAME.get(self.application, self.application),
  'EXT': self.extension,
+ 'PLATFORM': self.platform,
  'STUB': 'Stub ' if self.is_stub_installer else '',
  'STUB_NEW': ' Installer' if self.is_stub_installer else '',
  'VERSION': self.version,
@@ -652,7 +676,10 @@ def binary_regex(self):
  @property
  def path_regex(self):
  """Return the regex for the path to the build folder."""
- regex = r'releases/%(VERSION)s/%(PLATFORM)s/%(LOCALE)s/'
+ if self.application == "fenix":
+ regex = r'releases/%(VERSION)s/android/fenix-%(VERSION)s-%(PLATFORM)s/'
+ else:
+ regex = r'releases/%(VERSION)s/%(PLATFORM)s/%(LOCALE)s/'
  return regex % {'LOCALE': self.locale,
  'PLATFORM': self.platform_regex,
  'VERSION': self.version}
@@ -689,8 +716,8 @@ def query_versions(self, version=None):
  parser = self._create_directory_parser(url)
  if version:
  versions = parser.filter(latest_version_filter(version, self.application))
- from mozilla_version import gecko
- MozVersion = getattr(gecko, APPLICATIONS_TO_VERSION_CLASS[self.application])
+ import mozilla_version
+ MozVersion = getattr(mozilla_version, APPLICATIONS_TO_VERSION_CLASS[self.application])
  versions.sort(key=MozVersion.parse)
  return [versions[-1]]
  else:
@@ -795,10 +822,7 @@ def get_build_info(self):
  """Define additional build information."""
  # Retrieve build by revision
  if self.revision:
- th = treeherder.Treeherder(
- APPLICATIONS_TO_FTP_DIRECTORY.get(self.application, self.application),
- self.branch,
- self.platform)
+ th = treeherder.Treeherder(self.application, self.branch, self.platform)
  builds = th.query_builds_by_revision(
  self.revision, job_type_name='Build', debug_build=self.debug_build)
 
@@ -998,10 +1022,7 @@ def __init__(self, revision=None, debug_build=False, *args, **kwargs):
  def get_build_info(self):
  """Define additional build information."""
  # Retrieve build by revision
- th = treeherder.Treeherder(
- APPLICATIONS_TO_FTP_DIRECTORY.get(self.application, self.application),
- 'try',
- self.platform)
+ th = treeherder.Treeherder(self.application, 'try', self.platform)
  builds = th.query_builds_by_revision(
  self.revision, job_type_name='Build', debug_build=self.debug_build)
 

mozdownload/treeherder.py

@@ -13,10 +13,6 @@
 from mozdownload.errors import NotSupportedError
 
 PLATFORM_MAP = {
- 'android-api-9': {'build_platform': 'android-2-3-armv7-api9'},
- 'android-api-11': {'build_platform': 'android-4-0-armv7-api11'},
- 'android-api-15': {'build_platform': 'android-4-0-armv7-api15'},
- 'android-x86': {'build_platform': 'android-4-2-x86'},
  'linux': {'build_platform': 'linux32'},
  'linux64': {'build_platform': 'linux64'},
  'mac': {'build_os': 'mac', 'build_architecture': 'x86_64'},

tests/daily_scraper/test_daily_scraper.py

@@ -109,26 +109,22 @@
  'thunderbird/nightly/2013/10/2013-10-01-03-02-04-comm-aurora/thunderbird-27.0a1.en-US.win32.installer.exe'),
 ]
 
-fennec_tests = [
- ({'application': 'fennec', 'platform': 'android-api-9', 'branch': 'mozilla-central'},
- '2016-02-01-03-02-41-mozilla-central-fennec-47.0a1.multi.android-arm.apk',
- 'mobile/nightly/2016/02/2016-02-01-03-02-41-mozilla-central-android-api-9/fennec-47.0a1.multi.android-arm.apk'),
- ({'application': 'fennec', 'platform': 'android-api-11', 'branch': 'mozilla-central'},
- '2015-06-11-03-02-08-mozilla-central-fennec-41.0a1.multi.android-arm.apk',
- 'mobile/nightly/2015/06/2015-06-11-03-02-08-mozilla-central-android-api-11/fennec-41.0a1.multi.android-arm.apk'),
- ({'application': 'fennec', 'platform': 'android-api-15', 'branch': 'mozilla-central'},
- '2016-02-01-03-02-41-mozilla-central-fennec-47.0a1.multi.android-arm.apk',
- 'mobile/nightly/2016/02/2016-02-01-03-02-41-mozilla-central-android-api-15/fennec-47.0a1.multi.android-arm.apk'),
- ({'application': 'fennec', 'platform': 'android-x86', 'branch': 'mozilla-central'},
- '2016-02-01-03-02-41-mozilla-central-fennec-47.0a1.multi.android-i386.apk',
- 'mobile/nightly/2016/02/2016-02-01-03-02-41-mozilla-central-android-x86/fennec-47.0a1.multi.android-i386.apk'),
- ({'application': 'fennec', 'platform': 'android-api-15', 'branch': 'mozilla-aurora'},
- '2016-02-02-00-40-08-mozilla-aurora-fennec-46.0a2.multi.android-arm.apk',
- 'mobile/nightly/2016/02/2016-02-02-00-40-08-mozilla-aurora-android-api-15/fennec-46.0a2.multi.android-arm.apk'),
+fenix_tests = [
+ ({'application': 'fenix', 'platform': 'android-arm64-v8a', 'date': '2022-11-14'},
+ '2022-11-14-17-01-36-fenix-108.0b1.multi.android-arm64-v8a.apk',
+ 'fenix/nightly/2022/11/2022-11-14-17-01-36-fenix-108.0b1-android-arm64-v8a/fenix-108.0b1.multi.android-arm64-v8a.apk'),
+ ({'application': 'fenix', 'platform': 'android-x86', 'date': '2022-11-14'},
+ '2022-11-14-17-01-36-fenix-108.0b1.multi.android-x86.apk',
+ 'fenix/nightly/2022/11/2022-11-14-17-01-36-fenix-108.0b1-android-x86/fenix-108.0b1.multi.android-x86.apk'),
+ ({'application': 'fenix', 'platform': 'android-armeabi-v7a', 'date': '2022-11-14'},
+ '2022-11-14-17-01-36-fenix-108.0b1.multi.android-armeabi-v7a.apk',
+ 'fenix/nightly/2022/11/2022-11-14-17-01-36-fenix-108.0b1-android-armeabi-v7a/fenix-108.0b1.multi.android-armeabi-v7a.apk'),
+ ({'application': 'fenix', 'platform': 'android-x86_64', 'date': '2022-11-14'},
+ '2022-11-14-17-01-36-fenix-108.0b1.multi.android-x86_64.apk',
+ 'fenix/nightly/2022/11/2022-11-14-17-01-36-fenix-108.0b1-android-x86_64/fenix-108.0b1.multi.android-x86_64.apk'),
 ]
 
-
-@pytest.mark.parametrize("args,filename,url", firefox_tests + thunderbird_tests + fennec_tests)
+@pytest.mark.parametrize("args,filename,url", firefox_tests + thunderbird_tests + fenix_tests)
 def test_scraper(httpd, tmpdir, args, filename, url):
  """Testing various download scenarios for DailyScraper"""
 

tests/factory/test_missing_mandatory_options.py

@@ -22,14 +22,14 @@ def test_try_without_revision(httpd, tmpdir):
  FactoryScraper('try', destination=str(tmpdir), base_url=httpd.get_url())
 
 
-def test_non_daily_fennec(httpd, tmpdir):
- """Test that non-daily scraper_type for fennec raises exception"""
+def test_non_daily_fenix(httpd, tmpdir):
+ """Test that non-daily scraper_type for fenix raises exception"""
  with pytest.raises(NotSupportedError):
  FactoryScraper('candidate',
  destination=str(tmpdir),
  base_url=httpd.get_url(),
- application='fennec',
- version='60.0b1')
+ application='fenix',
+ version='110.0b1')
 
 
 def test_non_release_non_candidate_devedition(httpd, tmpdir):

tests/release_scraper/test_release_scraper.py

@@ -14,6 +14,18 @@
 
 
 @pytest.mark.parametrize("args,filename,url", [
+ ({'application': 'fenix', 'platform': 'android-arm64-v8a', 'version': '120.1.0'},
+ 'fenix-120.1.0.multi.android-arm64-v8a.apk',
+ 'fenix/releases/120.1.0/android/fenix-120.1.0-android-arm64-v8a/fenix-120.1.0.multi.android-arm64-v8a.apk'),
+ ({'application': 'fenix', 'platform': 'android-armeabi-v7a', 'version': '120.1.0'},
+ 'fenix-120.1.0.multi.android-armeabi-v7a.apk',
+ 'fenix/releases/120.1.0/android/fenix-120.1.0-android-armeabi-v7a/fenix-120.1.0.multi.android-armeabi-v7a.apk'),
+ ({'application': 'fenix', 'platform': 'android-x86', 'version': '120.1.0'},
+ 'fenix-120.1.0.multi.android-x86.apk',
+ 'fenix/releases/120.1.0/android/fenix-120.1.0-android-x86/fenix-120.1.0.multi.android-x86.apk'),
+ ({'application': 'fenix', 'platform': 'android-x86_64', 'version': '120.1.0'},
+ 'fenix-120.1.0.multi.android-x86_64.apk',
+ 'fenix/releases/120.1.0/android/fenix-120.1.0-android-x86_64/fenix-120.1.0.multi.android-x86_64.apk'),
  ({'platform': 'win32', 'version': '23.0.1'},
  'firefox-23.0.1.en-US.win32.exe',
  'firefox/releases/23.0.1/win32/en-US/Firefox Setup 23.0.1.exe'),

tests/release_scraper/test_release_scraper_latest.py

@@ -14,6 +14,30 @@
 
 
 @pytest.mark.parametrize("args,filename,url", [
+ ({'application': 'fenix', 'platform': 'android-arm64-v8a', 'version': 'latest'},
+ 'fenix-120.1.0.multi.android-arm64-v8a.apk',
+ 'fenix/releases/120.1.0/android/fenix-120.1.0-android-arm64-v8a/fenix-120.1.0.multi.android-arm64-v8a.apk'),
+ ({'application': 'fenix', 'platform': 'android-armeabi-v7a', 'version': 'latest'},
+ 'fenix-120.1.0.multi.android-armeabi-v7a.apk',
+ 'fenix/releases/120.1.0/android/fenix-120.1.0-android-armeabi-v7a/fenix-120.1.0.multi.android-armeabi-v7a.apk'),
+ ({'application': 'fenix', 'platform': 'android-x86', 'version': 'latest'},
+ 'fenix-120.1.0.multi.android-x86.apk',
+ 'fenix/releases/120.1.0/android/fenix-120.1.0-android-x86/fenix-120.1.0.multi.android-x86.apk'),
+ ({'application': 'fenix', 'platform': 'android-x86_64', 'version': 'latest'},
+ 'fenix-120.1.0.multi.android-x86_64.apk',
+ 'fenix/releases/120.1.0/android/fenix-120.1.0-android-x86_64/fenix-120.1.0.multi.android-x86_64.apk'),
+ ({'application': 'fenix', 'platform': 'android-arm64-v8a', 'version': 'latest-beta'},
+ 'fenix-120.0b9.multi.android-arm64-v8a.apk',
+ 'fenix/releases/120.0b9/android/fenix-120.0b9-android-arm64-v8a/fenix-120.0b9.multi.android-arm64-v8a.apk'),
+ ({'application': 'fenix', 'platform': 'android-armeabi-v7a', 'version': 'latest-beta'},
+ 'fenix-120.0b9.multi.android-armeabi-v7a.apk',
+ 'fenix/releases/120.0b9/android/fenix-120.0b9-android-armeabi-v7a/fenix-120.0b9.multi.android-armeabi-v7a.apk'),
+ ({'application': 'fenix', 'platform': 'android-x86', 'version': 'latest-beta'},
+ 'fenix-120.0b9.multi.android-x86.apk',
+ 'fenix/releases/120.0b9/android/fenix-120.0b9-android-x86/fenix-120.0b9.multi.android-x86.apk'),
+ ({'application': 'fenix', 'platform': 'android-x86_64', 'version': 'latest-beta'},
+ 'fenix-120.0b9.multi.android-x86_64.apk',
+ 'fenix/releases/120.0b9/android/fenix-120.0b9-android-x86_64/fenix-120.0b9.multi.android-x86_64.apk'),
  ({'application': 'firefox', 'platform': 'linux', 'version': 'latest'},
  'firefox-23.0.1.en-US.linux.tar.bz2',
  'firefox/releases/23.0.1/linux-i686/en-US/firefox-23.0.1.tar.bz2'),