From 7cdcfcd5938c569f04eccf9cc3ed95755644934a Mon Sep 17 00:00:00 2001
From: msj <monkruman.st.jules@datamde.us>
Date: Wed, 8 May 2024 10:21:02 -0400
Subject: [PATCH] Add time limit to nightly scrape + year option

---
 Makefile | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/Makefile b/Makefile
index 877f962..4719af9 100644
--- a/Makefile
+++ b/Makefile
@@ -1,3 +1,5 @@
+year=$(shell date +%Y)
+
 .PHONY: all
 all: upload
 
@@ -53,11 +55,11 @@ civil-%.jl: cases.db
 	START=$$(sqlite-utils query --csv --no-headers cases.db \
 	      "$(CIVIL_SCRAPE_START_QUERY)" -p subdivision $*); \
 				echo $$START; \
-	      scrapy crawl civil -a division=$* -a start=$$START -O $@;
+	      scrapy crawl civil -s CLOSESPIDER_TIMEOUT=3600 -a year=$(year) -a division=$* -a start=$$START -O $@;
 
 chancery.jl: cases.db
 	START=$$(sqlite3 cases.db < scripts/nightly_chancery_start.sql); \
-	      scrapy crawl chancery -a start=$$START -O $@;
+	      scrapy crawl chancery -a year=$(year) -a start=$$START -O $@;
 
 cases.db :
 	sqlite3 $@ < scripts/initialize_db.sql