Skip to content

Commit

Permalink
Merge pull request #914 from aarchiba/fix-908
Browse files Browse the repository at this point in the history 
Fix 908
  • Loading branch information
@paulray
paulray authored Dec 20, 2020
2 parents 4695b85 + 08c4959 commit 8854638
Show file tree
Hide file tree
Showing 3 changed files with 46 additions and 33 deletions.
60 changes: 37 additions & 23 deletions src/pint/toa.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,12 +3,12 @@
In particular, single TOAs are represented by :class:`pint.toa.TOA` objects, and if you
want to manage a collection of these we recommend you use a :class:`pint.toa.TOAs` object
as this makes certain operations much more convenient.
"""
from __future__ import absolute_import, division, print_function, unicode_literals

import copy
import gzip
import hashlib
import os
import re
import warnings
Expand Down Expand Up @@ -78,6 +78,12 @@
JD_MJD = 2400000.5


def _compute_hash(filename):
h = hashlib.sha256()
h.update(open(filename, "rb").read())
return h.digest()


def get_TOAs(
timfile,
ephem=None,
Expand Down Expand Up @@ -149,7 +155,6 @@ def get_TOAs(
-------
TOAs
Completed TOAs object representing the data.
"""
if model:
# If the keyword args are set, override what is in the model
Expand All @@ -173,11 +178,13 @@ def get_TOAs(
)
else:
log.warning(
f'CLOCK = {model["CLOCK"].value} is not implemented. Using TT({bipm_default}) instead.'
f'CLOCK = {model["CLOCK"].value} is not implemented. '
f"Using TT({bipm_default}) instead."
)
else:
log.warning(
f'CLOCK = {model["CLOCK"].value} is not implemented. Using TT({bipm_default}) instead.'
f'CLOCK = {model["CLOCK"].value} is not implemented. '
f"Using TT({bipm_default}) instead."
)

if planets is None and model["PLANET_SHAPIRO"].value:
Expand All @@ -190,6 +197,14 @@ def get_TOAs(
picklefile = _check_pickle(timfile)
if picklefile:
t = TOAs(picklefile)
if hasattr(t, "hashes"):
for f, v in t.hashes.items():
if _compute_hash(f) != v:
updatepickle = True
log.info("Pickle based on files that have changed")
break
else:
updatepickle = True
if (
include_gps is not None
and t.clock_corr_info.get("include_gps", None) != include_gps
Expand Down Expand Up @@ -252,6 +267,11 @@ def get_TOAs(

if usepickle and updatepickle:
log.info("Pickling TOAs.")
if isinstance(t.filename, str):
files = [t.filename]
else:
files = t.filename
t.hashes = {f: _compute_hash(f) for f in files}
t.pickle()
return t

Expand All @@ -278,14 +298,6 @@ def _check_pickle(toafilename, picklefilename=None):
if picklefilename is None:
return ""

# Check if TOA is newer than pickle
if os.path.getmtime(picklefilename) < os.path.getmtime(toafilename):
return ""

# TODO add more tests. Some things to consider:
# 1. Check file contents via md5sum (will require storing this in pickle).
# 2. Check INCLUDEd TOA files (will require some TOA file parsing).

# All checks passed, return name of pickle.
return picklefilename

Expand Down Expand Up @@ -353,7 +365,6 @@ def _parse_TOA_line(line, fmt="Unknown"):
Return an MJD tuple and a dictionary of other TOA information.
The format can be one of: Comment, Command, Blank, Tempo2,
Princeton, ITOA, Parkes, or Unknown.
"""
MJD = None
fmt = _toa_format(line, fmt)
Expand Down Expand Up @@ -565,7 +576,7 @@ def format_toa_line(
def make_fake_toas(
startMJD, endMJD, ntoas, model, freq=999999, obs="GBT", error=1 * u.us
):
"""Make evenly spaced toas with residuals = 0 and without errors
"""Make evenly spaced toas with residuals = 0 and without errors.
Might be able to do different frequencies if fed an array of frequencies,
only works with one observatory at a time
Expand Down Expand Up @@ -960,7 +971,7 @@ def __init__(self, toafile=None, toalist=None):
inc_fns = [
x[0][1] for x in self.commands if x[0][0].upper() == "INCLUDE"
]
self.filename = [toafile] + inc_fns if inc_fns else toafile
self.filename = [toafile] + inc_fns if inc_fns else toafile
elif toafile is not None:
self.read_toa_file(toafile)
self.filename = ""
Expand Down Expand Up @@ -1268,7 +1279,7 @@ def unselect(self):
)
try:
self.table = self.table_selects.pop()
except (AttributeError, IndexError) as e:
except (AttributeError, IndexError):
log.error("No previous TOA table found. No changes made.")

def pickle(self, filename=None):
Expand All @@ -1277,8 +1288,12 @@ def pickle(self, filename=None):
self.pintversion = pint.__version__
if filename is not None:
pickle.dump(self, open(filename, "wb"))
elif (self.filename is not None) and (type(self.filename) is not list):
pickle.dump(self, gzip.open(self.filename + ".pickle.gz", "wb"))
elif self.filename is not None:
if isinstance(self.filename, str):
filename = self.filename
else:
filename = self.filename[0]
pickle.dump(self, gzip.open(filename + ".pickle.gz", "wb"))
else:
raise ValueError("TOA pickle method needs a (single) filename.")

Expand Down Expand Up @@ -1781,9 +1796,8 @@ def read_pickle_file(self, filename):
tmp = pickle.load(infile)
if not hasattr(tmp, "pintversion") or tmp.pintversion != pint.__version__:
log.error(
"PINT version in pickle file is different than current version!\n*** Suggest deleting {}".format(
filename
)
f"PINT version in pickle file is different than current version! "
f"*** Suggest deleting {filename}"
)
self.filename = tmp.filename
if hasattr(tmp, "toas"):
Expand All @@ -1794,6 +1808,7 @@ def read_pickle_file(self, filename):
self.clock_corr_info = tmp.clock_corr_info
self.ephem = tmp.ephem
self.planets = tmp.planets
self.hashes = tmp.hashes

def read_toa_file(self, filename, process_includes=True, top=True):
"""Read TOAs from the given filename.
Expand Down Expand Up @@ -1927,7 +1942,7 @@ def merge_TOAs(TOAs_list):
"""Merge a list of TOAs instances and return a new combined TOAs instance
In order for a merge to work, each TOAs instance needs to have
been created using the same Solar System Ephemeris (EPHEM),
been created using the same Solar System Ephemeris (EPHEM),
the same reference timescale (i.e. CLOCK), and the same value of
.planets (i.e. whether planetary PosVel columns are in the tables
or not).
Expand Down Expand Up @@ -1990,4 +2005,3 @@ def merge_TOAs(TOAs_list):
# Now we need to re-arrange and group the tables
nt.table = nt.table.group_by("obs")
return nt

16 changes: 8 additions & 8 deletions tests/datafile/parkes.toa
View file
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
# These are Parkes format and have been changed to also be Parkes observatory
PUPPI_J2044+28_58852_652 432.3420 58852.7590686063892 0.00 120.75 7
PUPPI_J2044+28_58852_679 1433.237 58852.7921241426833 0.00 248.48 7
PUPPI_J2044+28_58882_586 432.3420 58882.6825930247209 0.00 219.67 7
PUPPI_J2044+28_58968_366 432.3430 58968.4254960153599 0.00 167.32 7
PUPPI_J2044+28_58970_343 431.9220 58970.4001441717446 0.00 136.16 7
PUPPI_J2044+28_58974_334 431.9220 58974.3901323879601 0.00 132.61 7
PUPPI_J2044+28_58981_310 431.9220 58981.3623757223578 0.00 154.07 7
PUPPI_J2044+28_59000_285 431.9220 59000.3318112360150 0.00 180.83 7
PUPPI_J2044+28_58852_652 432.3420 58852.7590686063892 0.00 120.75 @
PUPPI_J2044+28_58852_679 1433.237 58852.7921241426833 0.00 248.48 @
PUPPI_J2044+28_58882_586 432.3420 58882.6825930247209 0.00 219.67 @
PUPPI_J2044+28_58968_366 432.3430 58968.4254960153599 0.00 167.32 @
PUPPI_J2044+28_58970_343 431.9220 58970.4001441717446 0.00 136.16 @
PUPPI_J2044+28_58974_334 431.9220 58974.3901323879601 0.00 132.61 @
PUPPI_J2044+28_58981_310 431.9220 58981.3623757223578 0.00 154.07 @
PUPPI_J2044+28_59000_285 431.9220 59000.3318112360150 0.00 180.83 @
3 changes: 1 addition & 2 deletions tests/test_toa_reader.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ def setUp(self):

def test_read_parkes(self):
ts = toa.get_TOAs("parkes.toa")
assert "parkes" in ts.observatories
assert "barycenter" in ts.observatories
assert ts.ntoas == 8

def test_commands(self):
Expand Down Expand Up @@ -158,4 +158,3 @@ def test_toa_merge():
toas[0].ephem = "DE436"
with pytest.raises(TypeError):
nt = toa.merge_TOAs(toas)

0 comments on commit 8854638

Please sign in to comment.