Merge pull request #60 from datamade/bugfix/59-time-limit

Add time limit to nightly scrape

Makefile

@@ -1,3 +1,5 @@
+year=$(shell date +%Y)
+
 .PHONY: all
 all: upload
 
@@ -53,11 +55,11 @@ civil-%.jl: cases.db
  START=$$(sqlite-utils query --csv --no-headers cases.db \
  "$(CIVIL_SCRAPE_START_QUERY)" -p subdivision $*); \
  echo $$START; \
- scrapy crawl civil -a division=$* -a start=$$START -O $@;
+ scrapy crawl civil -s CLOSESPIDER_TIMEOUT=3600 -a year=$(year) -a division=$* -a start=$$START -O $@;
 
 chancery.jl: cases.db
  START=$$(sqlite3 cases.db < scripts/nightly_chancery_start.sql); \
- scrapy crawl chancery -a start=$$START -O $@;
+ scrapy crawl chancery -a year=$(year) -a start=$$START -O $@;
 
 cases.db :
  sqlite3 $@ < scripts/initialize_db.sql