-
Notifications
You must be signed in to change notification settings - Fork 0
/
getAttendance.py
executable file
·316 lines (269 loc) · 10.6 KB
/
getAttendance.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
#!/usr/bin/env python3
#
# Copyright 2024 Mike Jones, <[email protected]>
# AKA Grey Wolf <[email protected]>
# AKA Akela <[email protected]>
# [23rd Manchester (Birch with Fallowfield)]
# Scout Membership number: 12114313
#
# This program is free software: you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License,
# or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
# See the GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program. If not, see <https://www.gnu.org/licenses/>.
#
######################
# Before you go any further, get an API key here: https://www.onlinescoutmanager.co.uk/main.php
# Expand "Settings" (bottom of left pane)
# Select "My Account Details"
# From the new menu on the left in the main pane select "Develpoer Tools"
# Click Create Application button (top right)
# This will give you a client_id and client_secret: keep these secret; keep these safe
# Put this in a separate file perhaps called credentials.py <<EOF >.env
# OSM_API_KEY=<put value here>
# OSM_API_SECRET=<put secret here>
# OSM_API_RETURL=<put OAuth2 Callback URI here>
#EOF
#import credentials.py
import os
from dotenv import load_dotenv
import requests
from oauthlib.oauth2 import WebApplicationClient
import uuid
import json
from urllib.parse import urlparse,parse_qs
import itertools
import csv
from datetime import date, datetime, timedelta
import time
import webbrowser
from tkinter import filedialog, Tk
# Get secrets from .env
load_dotenv()
client_id = os.getenv("OSM_API_KEY")
client_secret = os.getenv("OSM_API_SECRET")
redirectURL = os.getenv("OSM_API_RETURL")
# These variables and decorator to stop exceeding rate limited http requests (no limit to start with)
HTTPWaitUntil = datetime.now()
RateLimit = 99999
RateLimitLeft = 99999
RateLimitTTL = 0
RateLimitBlockedState = False
from functools import wraps
def decoraterequest(f):
@wraps(f)
def wrapper(*args, **kwargs):
# Load global rate variables
global HTTPWaitUntil, RateLimit, RateLimitLeft, RateLimitTTL, RateLimitBlockedState
# Do something before
while datetime.now() < HTTPWaitUntil:
print("Waiting for rate limit to clear")
time.sleep(5)
r = f(*args, **kwargs)
if 'X-RateLimit-Limit' in r.headers:
RateLimit=int(r.headers['X-RateLimit-Limit'])
if 'X-RateLimit-Remaining' in r.headers:
RateLimitLeft=int(r.headers['X-RateLimit-Remaining'])
if RateLimitLeft < 20:
print ("Nearing rate limit for API, proceed with caution.")
if 'X-RateLimit-Reset' in r.headers:
RateLimitTTL=int(r.headers['X-RateLimit-Reset'])
if 'X-Blocked' in r.headers:
RateLimitBlockedState=r.headers['X-Blocked']
if r.status_code == 429:
if "Retry-After" in r.headers:
HTTPWaitUntil = datetime.now() + timedelta(seconds = int(r.headers['Retry-After']))
print("The API says retry after {}.".format(HTTPWaitUntil))
print("The API is blocked. Cannot continue")
quit()
elif r.status_code != 200:
print("Something went wrong accessing URL", args[0])
quit()
return r
return wrapper
requests.post=decoraterequest(requests.post)
requests.get=decoraterequest(requests.get)
#######################
# Prep OAuth2 initiator
client = WebApplicationClient(client_id)
authorization_url = 'https://www.onlinescoutmanager.co.uk/oauth/authorize'
token_url = 'https://www.onlinescoutmanager.co.uk/oauth/token'
state=str(uuid.uuid4())
url = client.prepare_request_uri(
authorization_url,
redirect_uri = redirectURL,
scope = ['section:attendance:read'],
state = state
)
#########################################
# Open System Browser and initiate OAuth2
print("Opening Browser for login")
webbrowser.open(url, new=1, autoraise=True)
returl = input("Enter full redirect URL that yout browser was sent to: ")
while not ("code=" in returl and redirectURL+"?" in returl and "state="+state in returl):
print("No, it should look something like: "+redirectURL+"?code=0123456789abcdef...&state="+state)
returl = input("Enter full redirect URL here:")
###############################
# Parse URL to get access token
purl=urlparse(returl)
query=parse_qs(purl.query)
code=query["code"][0]
###############################
# prep the OAuth2 token request
data = client.prepare_request_body(
code = code,
redirect_uri = redirectURL,
client_id = client_id,
client_secret = client_secret
)
# This header is required
headers = {'Content-type': 'application/x-www-form-urlencoded'}
##################
# Fire request off
response = requests.post(token_url, data=data, headers=headers)
#################################
# Pull access token from response
jtok=json.loads("{}")
if response.status_code==200:
jtok=json.loads(response.text)
access=jtok['access_token']
refresh=jtok['refresh_token']
############################################
# Set bearer header for subsequent API calls
header = {'Authorization': 'Bearer {}'.format(access)}
#############################################################################
# Get OSM startup infos (contains info about what sections/groups you can see
response = requests.get('https://www.onlinescoutmanager.co.uk/ext/generic/startup/?action=getData', headers=header)
#####################again status
# Gets the lump of json in the startup var setting exercise.
b=json.loads(response.text.split("var data_holder = ")[1])
# Build a section dict
sectionid=dict()
sectiontype=dict()
for a in b["globals"]["roles"]:
fullname=a['groupname']+": "+a['sectionname']
sectionid[fullname]=a['sectionid']
sectiontype[fullname]=a['section']
####################################
# Simple Menu for sections and dates
sectiondict = {str(index): element for index, element in enumerate(sorted(sectionid.keys()))}
menuOK=""
while menuOK != "Y" and menuOK != "y":
for i in sorted(sectiondict.keys()):
print (i,sectiondict[i])
while True:
get=input("Space separated list of sections to include: ")
got=get.split()
test=True
for i in got:
if i in sectiondict:
print(i, "selected:", sectiondict[i])
else:
print(i, "Invalid choice")
test=False
if test:
break
while True:
notbeforein=input("First Date (not before YYYY-MM-DD): ").strip()
try:
notbefore=date.fromisoformat(notbeforein)
break
except ValueError:
print("Date Not Valid; Try again!")
continue
while True:
notafterin=input("Last Date (not after YYYY-MM-DD [default: today]): ").strip()
if notafterin == "":
notafter=date.today()
print("Using Today's date",notafter)
break
try:
notafter=date.fromisoformat(notafterin)
break
except ValueError:
print("Date Not Valid; Try again!")
continue
##########
# Todo: Show user and get them to check menu choices before continuing
# Check choices are good via loop
print("Data to fetch: ")
for i in got:
print (" Section:",sectiondict[i])
print("Between {} and {} (inclusive)".format(notbefore,notafter))
menuOK=input("Happy with your choices? Yes/No/Abort: [y/n/A]: ").strip()
if menuOK == "A":
quit()
##################
# Get the relevant URLs for terms in the date range and sections chosen
terms={}
for i in got:
mysection=sectiondict[i]
for a in b["globals"]["terms"]:
for termchunk in b["globals"]["terms"][a]:
if 'sectionid' in termchunk:
if termchunk['sectionid'] == sectionid[mysection]:
term=termchunk['name']
termid=termchunk['termid']
start=date.fromisoformat(termchunk['startdate'])
end=date.fromisoformat(termchunk['enddate'])
if end < notbefore: #cant
# print("term out of (range older)", termchunk['startdate'], termchunk['enddate'])
pass
elif start > notafter:
# print("term out of (range newer)", termchunk['startdate'], termchunk['enddate'])
pass
else:
terms[i+": "+sectionid[mysection]+"; "+term]="https://www.onlinescoutmanager.co.uk/ext/members/attendance/?action=get§ionid="+sectionid[mysection]+"&termid="+termid+"§ion="+sectiontype[mysection]+"¬otal=true"
#####################################
# Go and get attendance data from OSM
meetings={} # should probably use a set here
names={}
for termurl in terms: # Loop over section specific terms we know about
response = requests.get(terms[termurl], headers=header) # Request term data
att=json.loads(response.text)
for item in att['items']: # dict()s per individual in this term containing name, meeting attendance, and some other data
namekey=item['firstname']+" "+item['lastname']+" ("+str(item['scoutid'])+")"
sectionstart=item['startdate']
ss=date.fromisoformat(sectionstart)
# sectionend=item['enddate'] # Can be a data string or None #!!!! can be weird depending on moving on choices; probably to do with unclear last term to show in values
# try:
# se=date.fromisoformat(sectionend)
# except TypeError:
# se=date.today()
se=date.today()
if not ( namekey in names ):
names[namekey]=dict() # Add name with dict to index so it can be filled if not already defined
for meeting in att["meetings"]: # Loop over meetings in this term
md=date.fromisoformat(meeting)
if md >= notbefore and md <= notafter: # Process meetings which fall within user's requested range (from menu)
meetings[meeting]=1
if md >= ss and md <= se: # Process meetings for this member which fall within their membership of this section
if meeting in item:
if item[meeting] == 'Yes':
names[namekey][meeting]=1
else:
names[namekey][meeting]=0
else:
names[namekey][meeting]=0 # sometimes value is not set (depending on who did register set to 0 if within date & section membership params)
################## Consider making names section specific or adding col for each section entry per person?
# Write to CSV
##############
# Use tkinter to popup a save dialogue
root = Tk()
root.withdraw()
file = filedialog.asksaveasfilename(defaultextension='.csv')
# Write the data
with open(file, "w") as f:
w = csv.DictWriter(f, fieldnames=["name"]+sorted(meetings.keys()) )
w.writeheader()
for person in sorted(names.keys()):
row = {'name': person}
row.update(names[person])
w.writerow(row)