WIP: single function call to preload Astropy cache

docs/howto.rst

@@ -18,3 +18,4 @@ Please feel free to change this by writing more; there are also some entries on
    editing-documentation
    working-with-notebooks
    controlling-logging
+   working-offline

docs/working-offline.rst

@@ -0,0 +1,42 @@
+Working offline
+===============
+
+PINT takes advantage of the Internet to ensure that you are always working with
+up-to-date clock correction and other files. Normally this is very convenient,
+but sometimes one doesn't have access to the Internet. This could be because
+you were on an airplane or a mountaintop or inside a secure facility and can't
+wait to time your pulsar, or it could be because you are running PINT on the
+nodes of a cluster that don't have connections to the general Internet. PINT
+has some tools to support these situations.
+
+Here are some general guidelines:
+
+- You will need to ensure that you pre-download an appropriate collection of files before you disconnect from the Internet.
+- You can find yourself using out-of-date clock files; PINT will by default emit warnings if this happens.
+- You can find yourself using out-of-date IERS (Earth-rotation or leap-second) files. Newer versions of Astropy will warn you if this occurs, but older ones may not.
+
+The files that PINT and Astropy go looking for on the Internet get downloaded
+and stored in the "Astropy cache". This is a location, usually in your home
+directory, that depends on your operating system, version of Astropy, and
+possibly environment variable settings. If you need to know where this is, you
+can run ``...``. In general this directory will work and be accessible when you
+need it to; the Astropy documentation has some more details on this.
+
+Pre-loading the Astropy cache should be easy: simply call
+:func:`pint.utils.preload_cache` and it will download every file it thinks you
+might need in the cache. You can often get away with less; if you have been
+running your scripts fine, your cache probably already contains all the files
+you need.
+
+When you want to ensure that PINT and Astropy do not reach out to the Internet,
+you can call :func:`pint.utils.set_no_internet`; this will set several Astropy
+config options for the duration of the current Python session, and together
+they will ensure that your script does not reach out to the Internet and
+out-of-date files result in warnings.
+
+If you want a longer-lasting setup to disable Internet access, you can set up
+an Astropy config file. See ... for details on how to do this. Once the file
+exists, you can set some options to achieve the same effect as
+:func:`pint.utils.set_no_internet`. Unfortunately, which options are available
+differs between Astropy 4.0, 4.3, 5.0, and 5.1; support for disconnected
+operation has been gradually improving.

src/pint/utils.py

@@ -38,7 +38,7 @@
 import sys
 import textwrap
 from collections import OrderedDict
-from contextlib import contextmanager
+from contextlib import contextmanager, ExitStack
 from copy import deepcopy
 from io import StringIO
 from pathlib import Path
@@ -52,6 +52,7 @@
 import scipy.optimize.zeros as zeros
 from loguru import logger as log
 from scipy.special import fdtrc
+from astropy.utils.iers import IERS_Auto
 
 import pint
 import pint.pulsar_ecliptic
@@ -1095,7 +1096,7 @@ def weighted_mean(arrin, weights_in, inputmean=None, calcerr=False, sdev=False):
 def ELL1_check(
     A1: u.cm, E: u.dimensionless_unscaled, TRES: u.us, NTOA: int, outstring=True
 ):
-    """Check for validity of assumptions in ELL1 binary model
+    r"""Check for validity of assumptions in ELL1 binary model
 
     Checks whether the assumptions that allow ELL1 to be safely used are
     satisfied. To work properly, we should have:
@@ -1652,3 +1653,69 @@ def compute_hash(filename):
         while block := f.read(blocks * h.block_size):
             h.update(block)
     return h.digest()
+
+
+def preload_cache(extra_ephemerides=None):
+    """Ensure that all files PINT needs are in the Astropy cache.
+
+    This requests all clock corrections in the global repository, a list of
+    standard Solar System ephemerides, and up-to-date IERS tables. Once
+    complete, PINT should be able to function without an Internet connection.
+
+    Note that you may need to set some Astropy configuration options to prevent
+    Astropy from requesting new IERS data after a month.
+
+    For more information see http://docs.astropy.org/en/stable/utils/data.html#using-astropy-with-limited-or-no-internet-access
+    """
+    from pint.observatory.global_clock_corrections import update_all
+    from pint.solar_system_ephemerides import load_kernel
+
+    update_all()
+    IERS_Auto.open()
+    ephemerides = [
+        "de200",
+        "de405",
+        "de421",
+        "de430",
+        "de430t",
+        "de432s",
+        "de434",
+        "de436",
+        "de436t",
+        "de440",
+        "de440s",
+    ]
+    if extra_ephemerides is not None:
+        ephemerides.extend(extra_ephemerides)
+    for e in ephemerides:
+        load_kernel(e)
+
+
+def set_no_internet(mode="warn"):
+    """Set Astropy and PINT to run without Internet access.
+
+    The sets up a number of Astropy configuration options. If you want to achieve
+    this effect without having to add this line to your scripts, you can create
+    an Astropy config file and edit it to contain these same options. See
+    https://docs.astropy.org/en/stable/config/index.html#astropy-config
+    for details of how to do this.
+
+    Parameters
+    ----------
+    mode : 'warn' or 'ignore'
+        What to do when files appear to be out of date
+    """
+    import astropy.utils.data
+    import astropy.utils.iers
+
+    if hasattr(astropy.utils.data.conf, "allow_internet"):
+        astropy.utils.data.conf.allow_internet = False
+    astropy.utils.iers.conf.remote_timeout = 0
+    # if hasattr(astropy.utils.iers.conf, "auto_download"):
+    #    astropy.utils.iers.conf.auto_download = False
+    # else:
+    #    astropy.utils.iers.conf.remote_timeout = 0
+    if hasattr(astropy.utils.iers.conf, "iers_degraded_accuracy"):
+        astropy.utils.iers.conf.iers_degraded_accuracy = "warn"
+    # else:
+    #    astropy.utils.iers.conf.auto_max_age = None

tests/test_utils.py

@@ -6,10 +6,12 @@
 
 import astropy.constants as c
 import astropy.units as u
+import astropy.utils.data
 import numpy as np
 import pytest
 import scipy.stats
 from astropy.time import Time
+from astropy.config import set_temp_cache, set_temp_config
 from hypothesis import assume, example, given
 from hypothesis.extra.numpy import array_shapes, arrays, scalar_dtypes
 from hypothesis.strategies import (
@@ -51,6 +53,8 @@
     open_or_use,
     taylor_horner,
     taylor_horner_deriv,
+    preload_cache,
+    set_no_internet,
 )
 
 
@@ -765,3 +769,100 @@ def test_compute_hash_accepts_no_change(a):
         h_b = compute_hash(g)
 
     assert h_a == h_b
+
+
+def test_preload_cache_gives_all_expected_files(tmp_path):
+    with set_temp_cache():
+        preload_cache()
+        expected = [
+            # "ftp://ssd.jpl.nasa.gov/pub/eph/planets/bsp/de200.bsp",
+            "http://data.astropy.org/coordinates/sites.json",
+            "http://hpiers.obspm.fr/iers/eop/eopc04/eopc04_IAU2000.62-now",
+            # Ephemerides are sometimes pulled in via Astropy and sometimes via PINT
+            # Astropy
+            "https://naif.jpl.nasa.gov/pub/naif/generic_kernels/spk/planets/de430.bsp",
+            "https://naif.jpl.nasa.gov/pub/naif/generic_kernels/spk/planets/de432s.bsp",
+            "https://naif.jpl.nasa.gov/pub/naif/generic_kernels/spk/planets/de440.bsp",
+            "https://naif.jpl.nasa.gov/pub/naif/generic_kernels/spk/planets/de440s.bsp",
+            # PINT
+            "https://data.nanograv.org/static/data/ephem/de200.bsp",
+            "https://data.nanograv.org/static/data/ephem/de405.bsp",
+            "https://data.nanograv.org/static/data/ephem/de421.bsp",
+            "https://data.nanograv.org/static/data/ephem/de430t.bsp",
+            "https://data.nanograv.org/static/data/ephem/de434.bsp",
+            "https://data.nanograv.org/static/data/ephem/de436.bsp",
+            "https://data.nanograv.org/static/data/ephem/de436t.bsp",
+            # IERS A table
+            "https://datacenter.iers.org/data/9/finals2000A.all",
+            # "https://maia.usno.navy.mil/ser7/finals2000A.all",  # This is the mirror address, should never be needed I think?
+            # "ftp://anonymous:mail%[email protected]/pub/products/iers/finals2000A.all",  # Another mirror that shouldn't be needed
+            "https://hpiers.obspm.fr/iers/bul/bulc/Leap_Second.dat",
+            # Clock corrections
+            "https://raw.githubusercontent.com/ipta/pulsar-clock-corrections/main/T2runtime/clock/ao2gps.clk",
+            "https://raw.githubusercontent.com/ipta/pulsar-clock-corrections/main/T2runtime/clock/ao2gps_tempo2.clk",
+            "https://raw.githubusercontent.com/ipta/pulsar-clock-corrections/main/T2runtime/clock/ao2nist.clk",
+            "https://raw.githubusercontent.com/ipta/pulsar-clock-corrections/main/T2runtime/clock/chime2gps.clk",
+            "https://raw.githubusercontent.com/ipta/pulsar-clock-corrections/main/T2runtime/clock/eff2gps.clk",
+            "https://raw.githubusercontent.com/ipta/pulsar-clock-corrections/main/T2runtime/clock/effix2gps.clk",
+            "https://raw.githubusercontent.com/ipta/pulsar-clock-corrections/main/T2runtime/clock/gbt2gps.clk",
+            "https://raw.githubusercontent.com/ipta/pulsar-clock-corrections/main/T2runtime/clock/gbt2gps_tempo2.clk",
+            "https://raw.githubusercontent.com/ipta/pulsar-clock-corrections/main/T2runtime/clock/gmrt2gps.clk",
+            "https://raw.githubusercontent.com/ipta/pulsar-clock-corrections/main/T2runtime/clock/gps2utc.clk",
+            "https://raw.githubusercontent.com/ipta/pulsar-clock-corrections/main/T2runtime/clock/gps2utc_c0p.clk",
+            "https://raw.githubusercontent.com/ipta/pulsar-clock-corrections/main/T2runtime/clock/gps2utc_cc.clk",
+            "https://raw.githubusercontent.com/ipta/pulsar-clock-corrections/main/T2runtime/clock/gps2utc_tempo2.clk",
+            "https://raw.githubusercontent.com/ipta/pulsar-clock-corrections/main/T2runtime/clock/jb2gps.clk",
+            "https://raw.githubusercontent.com/ipta/pulsar-clock-corrections/main/T2runtime/clock/jbdfb2jb.clk",
+            "https://raw.githubusercontent.com/ipta/pulsar-clock-corrections/main/T2runtime/clock/jbroach2jb.clk",
+            "https://raw.githubusercontent.com/ipta/pulsar-clock-corrections/main/T2runtime/clock/mk2utc.clk",
+            "https://raw.githubusercontent.com/ipta/pulsar-clock-corrections/main/T2runtime/clock/mk2utc_observatory.clk",
+            "https://raw.githubusercontent.com/ipta/pulsar-clock-corrections/main/T2runtime/clock/mo2gps.clk",
+            "https://raw.githubusercontent.com/ipta/pulsar-clock-corrections/main/T2runtime/clock/ncyobs2obspm.clk",
+            "https://raw.githubusercontent.com/ipta/pulsar-clock-corrections/main/T2runtime/clock/obspm2gps.clk",
+            "https://raw.githubusercontent.com/ipta/pulsar-clock-corrections/main/T2runtime/clock/pks2gps.clk",
+            "https://raw.githubusercontent.com/ipta/pulsar-clock-corrections/main/T2runtime/clock/srt2gps.clk",
+            "https://raw.githubusercontent.com/ipta/pulsar-clock-corrections/main/T2runtime/clock/tai2tt_bipm01.clk",
+            "https://raw.githubusercontent.com/ipta/pulsar-clock-corrections/main/T2runtime/clock/tai2tt_bipm05.clk",
+            "https://raw.githubusercontent.com/ipta/pulsar-clock-corrections/main/T2runtime/clock/tai2tt_bipm06.clk",
+            "https://raw.githubusercontent.com/ipta/pulsar-clock-corrections/main/T2runtime/clock/tai2tt_bipm2003.clk",
+            "https://raw.githubusercontent.com/ipta/pulsar-clock-corrections/main/T2runtime/clock/tai2tt_bipm2004.clk",
+            "https://raw.githubusercontent.com/ipta/pulsar-clock-corrections/main/T2runtime/clock/tai2tt_bipm2010.clk",
+            "https://raw.githubusercontent.com/ipta/pulsar-clock-corrections/main/T2runtime/clock/tai2tt_bipm2012.clk",
+            "https://raw.githubusercontent.com/ipta/pulsar-clock-corrections/main/T2runtime/clock/tai2tt_bipm2013.clk",
+            "https://raw.githubusercontent.com/ipta/pulsar-clock-corrections/main/T2runtime/clock/tai2tt_bipm2014.clk",
+            "https://raw.githubusercontent.com/ipta/pulsar-clock-corrections/main/T2runtime/clock/tai2tt_bipm2015.clk",
+            "https://raw.githubusercontent.com/ipta/pulsar-clock-corrections/main/T2runtime/clock/tai2tt_bipm2016.clk",
+            "https://raw.githubusercontent.com/ipta/pulsar-clock-corrections/main/T2runtime/clock/tai2tt_bipm2017.clk",
+            "https://raw.githubusercontent.com/ipta/pulsar-clock-corrections/main/T2runtime/clock/tai2tt_bipm2018.clk",
+            "https://raw.githubusercontent.com/ipta/pulsar-clock-corrections/main/T2runtime/clock/tai2tt_bipm2019.clk",
+            "https://raw.githubusercontent.com/ipta/pulsar-clock-corrections/main/T2runtime/clock/tai2tt_bipm2020.clk",
+            "https://raw.githubusercontent.com/ipta/pulsar-clock-corrections/main/T2runtime/clock/tai2tt_bipm2021.clk",
+            "https://raw.githubusercontent.com/ipta/pulsar-clock-corrections/main/T2runtime/clock/tai2tt_bipm92.clk",
+            "https://raw.githubusercontent.com/ipta/pulsar-clock-corrections/main/T2runtime/clock/vla2gps.clk",
+            "https://raw.githubusercontent.com/ipta/pulsar-clock-corrections/main/T2runtime/clock/vla2nist.clk",
+            "https://raw.githubusercontent.com/ipta/pulsar-clock-corrections/main/T2runtime/clock/wsrt2gps.clk",
+            "https://raw.githubusercontent.com/ipta/pulsar-clock-corrections/main/index.txt",
+            "https://raw.githubusercontent.com/ipta/pulsar-clock-corrections/main/tempo/clock/leap.sec",
+            "https://raw.githubusercontent.com/ipta/pulsar-clock-corrections/main/tempo/clock/time_ao.dat",
+            "https://raw.githubusercontent.com/ipta/pulsar-clock-corrections/main/tempo/clock/time_fast.dat",
+            "https://raw.githubusercontent.com/ipta/pulsar-clock-corrections/main/tempo/clock/time_gb140.dat",
+            "https://raw.githubusercontent.com/ipta/pulsar-clock-corrections/main/tempo/clock/time_gb853.dat",
+            "https://raw.githubusercontent.com/ipta/pulsar-clock-corrections/main/tempo/clock/time_gbt.dat",
+            "https://raw.githubusercontent.com/ipta/pulsar-clock-corrections/main/tempo/clock/time_jb.dat",
+            "https://raw.githubusercontent.com/ipta/pulsar-clock-corrections/main/tempo/clock/time_nuppi.dat",
+            "https://raw.githubusercontent.com/ipta/pulsar-clock-corrections/main/tempo/clock/time_pks.dat",
+            "https://raw.githubusercontent.com/ipta/pulsar-clock-corrections/main/tempo/clock/time_vla.dat",
+            "https://raw.githubusercontent.com/ipta/pulsar-clock-corrections/main/tempo/clock/time_wsrt.dat",
+            "https://raw.githubusercontent.com/ipta/pulsar-clock-corrections/main/tempo/clock/ut1.dat",
+        ]
+        for k in expected:
+            assert k in astropy.utils.data.cache_contents()
+
+
+def test_no_internet_forbids_internet(tmp_path, monkeypatch):
+    monkeypatch.delattr("astropy.utils.data.download_file")
+    with set_temp_cache(tmp_path):
+        with set_temp_config():
+            set_no_internet()
+            (Time.now() + 1 * u.year).utc.tai.ut1
+            assert not astropy.utils.data.cache_contents()

tox.ini

@@ -66,6 +66,58 @@ deps =
     numdifftools
 commands = {posargs:pytest}
 
+[testenv:astropy40]
+description =
+    Run known-tricky tests with Astropy 4.0 specifically
+basepython = python3.8
+deps =
+    numpy==1.17.*
+    astropy==4.0
+    pytest
+    coverage
+    hypothesis
+    numdifftools
+commands = {posargs:pytest tests/test_utils.py}
+
+[testenv:astropy43]
+description =
+    Run known-tricky tests with Astropy 4.3 specifically
+basepython = python3.8
+deps =
+    numpy==1.17.*
+    astropy==4.3
+    pytest
+    coverage
+    hypothesis
+    numdifftools
+commands = {posargs:pytest tests/test_utils.py}
+
+[testenv:astropy50]
+description =
+    Run known-tricky tests with Astropy 5.0 specifically
+basepython = python3.8
+deps =
+    numpy==1.18.*
+    astropy==5.0
+    pytest
+    coverage
+    hypothesis
+    numdifftools
+commands = {posargs:pytest tests/test_utils.py}
+
+[testenv:astropy51]
+description =
+    Run known-tricky tests with Astropy 5.1 specifically
+basepython = python3.8
+deps =
+    numpy
+    astropy==5.1
+    pytest
+    coverage
+    hypothesis
+    numdifftools
+commands = {posargs:pytest tests/test_utils.py}
+
 [testenv:report]
 skip_install = true
 deps = coverage