Add CP.org Libraries Update Script

.gitignore

@@ -6,3 +6,4 @@ _build
 env.sh
 *.swp
 .libraries/*
+.cp_org/*

adabot/github_requests.py

@@ -27,9 +27,11 @@
 
 * Author(s): Scott Shawcroft
 """
-import os
 
+import datetime
+import os
 import requests
+import time
 
 
 def _fix_url(url):
@@ -58,6 +60,19 @@ def get(url, **kwargs):
     response = requests.get(_fix_url(url), timeout=30, **_fix_kwargs(kwargs))
     if "X-RateLimit-Remaining" in response.headers:
         remaining = int(response.headers["X-RateLimit-Remaining"])
+        if remaining <= 1:
+            rate_limit_reset = datetime.datetime.fromtimestamp(int(response.headers["X-RateLimit-Reset"]))
+            print("GitHub API Rate Limit reached. Pausing until Rate Limit reset.")
+            while datetime.datetime.now() < rate_limit_reset:
+                print("Rate Limit will reset at: {}".format(rate_limit_reset))
+                if "TRAVIS" in os.environ:
+                    # only pause for 5 minutes so that Travis doesn't timeout
+                    # due to idle console output.
+                    time.sleep(300)
+                else:
+                    reset_diff = rate_limit_reset - datetime.datetime.now()
+                    print("Sleeping {} seconds".format(reset_diff.seconds))
+                    time.sleep(reset_diff.seconds)
         if remaining % 100 == 0:
             print(remaining, "requests remaining this hour")
     return response

adabot/lib/circuitpython_library_validators.py

@@ -27,7 +27,7 @@
 from adabot import github_requests as github
 from adabot import travis_requests as travis
 from adabot import pypi_requests as pypi
-from adabot.lib.common_funcs import *
+from adabot.lib import common_funcs
 
 
 # Define constants for error strings to make checking against them more robust:
@@ -169,7 +169,7 @@ def validate_repo_state(self, repo):
             errors.append(ERROR_MISSING_LICENSE)
         if not repo["permissions"]["push"]:
             errors.append(ERROR_MISSING_LIBRARIANS)
-        if not is_repo_in_bundle(full_repo["clone_url"], self.bundle_submodules) and \
+        if not common_funcs.is_repo_in_bundle(full_repo["clone_url"], self.bundle_submodules) and \
            not repo["name"] in BUNDLE_IGNORE_LIST:  # Don't assume the bundle will
                                                     # bundle itself and possibly
                                                     # other repos.
@@ -337,10 +337,12 @@ def _validate_travis_yml(self, repo, travis_yml_file_info):
 
         if not pylint_version:
             errors.append(ERROR_PYLINT_VERSION_NOT_FIXED)
-        elif pylint_version.startswith("1."):
-            errors.append(ERROR_PYLINT_VERSION_VERY_OUTDATED)
-        elif pylint_version != self.latest_pylint:
-            errors.append(ERROR_PYLINT_VERSION_NOT_LATEST)
+        # disabling below for now, since we know all pylint versions are old
+        # will re-enable once efforts are underway to update pylint
+        #elif pylint_version.startswith("1."):
+        #    errors.append(ERROR_PYLINT_VERSION_VERY_OUTDATED)
+        #elif pylint_version != self.latest_pylint:
+        #    errors.append(ERROR_PYLINT_VERSION_NOT_LATEST)
 
         return errors
 
@@ -586,9 +588,9 @@ def validate_readthedocs(self, repo):
                 return [ERROR_RTD_SUBPROJECT_FAILED]
             rtd_subprojects = {}
             for subproject in rtd_response.json()["subprojects"]:
-                rtd_subprojects[sanitize_url(subproject["repo"])] = subproject
+                rtd_subprojects[common_funcs.sanitize_url(subproject["repo"])] = subproject
 
-        repo_url = sanitize_url(repo["clone_url"])
+        repo_url = common_funcs.sanitize_url(repo["clone_url"])
         if repo_url not in rtd_subprojects:
             return [ERROR_RTD_SUBPROJECT_MISSING]
 
@@ -608,9 +610,10 @@ def validate_readthedocs(self, repo):
             latest_release = github.get("/repos/{}/releases/latest".format(repo["full_name"]))
             if not latest_release.ok:
                 errors.append(ERROR_GITHUB_RELEASE_FAILED)
-            else:
-                if latest_release.json()["tag_name"] not in [tag["verbose_name"] for tag in valid_versions["versions"]]:
-                    errors.append(ERROR_RTD_MISSING_LATEST_RELEASE)
+            # disabling this for now, since it is ignored and always fails
+            #else:
+            #    if latest_release.json()["tag_name"] not in [tag["verbose_name"] for tag in valid_versions["versions"]]:
+            #        errors.append(ERROR_RTD_MISSING_LATEST_RELEASE)
 
         # There is no API which gives access to a list of builds for a project so we parse the html
         # webpage.
@@ -786,6 +789,6 @@ def validate_in_pypi(self, repo):
         if not (repo["owner"]["login"] == "adafruit" and
                 repo["name"].startswith("Adafruit_CircuitPython")):
             return []
-        if not repo_is_on_pypi(repo):
+        if not common_funcs.repo_is_on_pypi(repo):
             return [ERROR_NOT_ON_PYPI]
         return []