-
Notifications
You must be signed in to change notification settings - Fork 13
/
Makefile
139 lines (117 loc) · 4.98 KB
/
Makefile
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
# set defaults (see: https://tech.davis-hansson.com/p/make/)
SHELL := bash
.ONESHELL:
.SHELLFLAGS := -eu -o pipefail -c
.DELETE_ON_ERROR:
MAKEFLAGS += --warn-undefined-variables
MAKEFLAGS += --no-builtin-rules
# tell Make to use `>` instead of tabs to specify indentation
# see: https://tech.davis-hansson.com/p/make/
ifeq ($(origin .RECIPEPREFIX), undefined)
$(error This Make does not support .RECIPEPREFIX. Please use GNU Make 4.0 or later)
endif
.RECIPEPREFIX = >
# MapPLUTO data
# NOTE: "archived" versions of the data follow a slightly different naming convention then current releases
# e.g. nyc_mappluto_<version>_arc_shp.zip
# so if the mappluto data download fails, then check to make sure the version hasn't been archived.
MAPPLUTO_DATA_VERSION := "20v8"
MAPPLUTO_DATA_FILE := "nyc_mappluto_$(MAPPLUTO_DATA_VERSION)_shp.zip"
MAPPLUTO_DATA_URI := "https://www1.nyc.gov/assets/planning/download/zip/data-maps/open-data/$(MAPPLUTO_DATA_FILE)"
# J51 exemption data
J51_DATA_URI := "https://data.cityofnewyork.us/api/views/y7az-s7wc/rows.csv?accessType=DOWNLOAD"
# Postgres
POSTGRES_PASSWORD := "postgres"
POSTGRES_USER := "postgres"
POSTGRES_DB := "airs"
# db-service/.build.sentinel has a lot of dependencies, listing them here for readability
DB_SERVICE_BUILD_DEPS := db-service/.volume.sentinel db-service/data/MapPLUTO.shp
DB_SERVICE_BUILD_DEPS += 421a-service/data/421a.csv db-service/data/j51_exemptions.csv
.PHONY: all
all: db-service/.run.sentinel
.PHONY: clean
clean: clean-421a clean-db
.PHONY: clean-421a
clean-421a:
> @echo "Cleaning 421a service..."
> rm -rf ./421a-service/data/*
> rm -rf ./421a-service/__pycache__
> rm -f ./421a-service/.build.sentinel
ifneq ($(strip $(shell docker ps -a | grep airs-data-421a)),)
> docker stop airs-data-421a && docker rm airs-data-421a
endif
ifneq ($(strip $(shell docker images -a | grep clhenrick/python)),)
> docker image rm clhenrick/python:3.7-slim
endif
.PHONY: clean-db
clean-db:
> @echo "Cleaning db service..."
> rm -rf ./db-service/data/*
> rm -f ./db-service/.*.sentinel
ifneq ($(strip $(shell docker ps -a | grep airs-data-db)),)
> docker stop airs-data-db && docker rm airs-data-db
endif
ifneq ($(strip $(shell docker images -a | grep clhenrick/airs-data-db)),)
> docker image rm clhenrick/airs-data-db
endif
ifneq ($(strip $(shell docker volume ls | grep airs-data-db)),)
> docker volume rm airs-data-db
endif
##########################################################################################
# Database Initialization & likely_rs table creation
##########################################################################################
db-service/.run.sentinel: db-service/.build.sentinel
> @echo "Running Docker container for db-service..."
> docker run \
> --name airs-data-db \
> --publish 5433:5432 \
> --env POSTGRES_PASSWORD=$(POSTGRES_PASSWORD) \
> --env POSTGRES_USER=$(POSTGRES_USER) \
> --env POSTGRES_DB=$(POSTGRES_DB) \
> --volume airs-data-db:/var/lib/postgresql/data \
> --restart=unless-stopped \
> clhenrick/airs-data-db
> touch $@
db-service/.build.sentinel: $(DB_SERVICE_BUILD_DEPS)
> @echo "Building Docker container for db-service..."
> docker build -t clhenrick/airs-data-db -f ./db-service/Dockerfile .
> touch $@
db-service/.volume.sentinel:
> @echo "Creating Docker volume for db-service..."
> docker volume create airs-data-db
> touch $@
##########################################################################################
# MapPLUTO data sourcing
##########################################################################################
.INTERMEDIATE: db-service/data/MapPLUTO.shp
db-service/data/MapPLUTO.shp: db-service/data/$(MAPPLUTO_DATA_FILE)
> @echo "unzipping $< to $@..."
> cd ./db-service/data
> unzip $(MAPPLUTO_DATA_FILE)
> cd ../..
.INTERMEDIATE: db-service/data/$(MAPPLUTO_DATA_FILE)
db-service/data/$(MAPPLUTO_DATA_FILE): db-service/data/.data.sentinel
> @echo "downloading $@..."
> wget --no-use-server-timestamps $(MAPPLUTO_DATA_URI) -O $@
db-service/data/.data.sentinel:
> mkdir -p db-service/data
> touch db-service/data/.data.sentinel
##########################################################################################
# J51 data sourcing and processing
##########################################################################################
db-service/data/j51_exemptions.csv:
> @echo "downloading $(notdir $@)..."
> wget --no-use-server-timestamps $(J51_DATA_URI) -O $@
##########################################################################################
# 421a data sourcing and processing
##########################################################################################
421a-service/data/421a.csv: 421a-service/.build.sentinel
> @echo "Running Docker container for 421a-service to create $@..."
> cd ./421a-service
> chmod +x init.sh
> docker run --name airs-data-421a -it --rm -v $$(pwd):/home/data clhenrick/python:3.7-slim
> cd ..
421a-service/.build.sentinel:
> @echo "Buiding Docker image for 421a-service..."
> docker build -t clhenrick/python:3.7-slim -f ./421a-service/Dockerfile ./421a-service
> touch $@