Uploaded the latest scraper version & backend API powered by flask

backend/scraper/.gitignore

@@ -0,0 +1,3 @@
+scraper_old.py
+config.ini
+env/

backend/scraper/README.md

@@ -0,0 +1,19 @@
+To use, first create a virtual environment: `python3 -m venv <myenvname>`
+
+Then activate it:
+
+On windows: `.\env\Scripts\activate.bat`
+On mac: `source venv/bin/activate`
+
+Then install needed packages: `pip3 install -r requirements.txt`
+
+Then create a file `config.ini` that contains the link to the MONGO database in the format seen in `config.ini.example`.
+
+Then launch the scraper: `python3 scraper.py`
+
+--------
+
+Updates for the future:
+- potentially put each scraper in its own file for better modularity
+- also improve the Handshake/TC mechanisms to automatically update login cookie information cause it is static at the moment
+- other stuff tba

backend/scraper/config.ini.example

@@ -0,0 +1,2 @@
+[PROD]
+DB_URI_SHORT = mongodb+srv://USER:[email protected]_blah_blah.mongodb.net/?retryWrites=true&w=majority

backend/scraper/drop_in.json

@@ -0,0 +1 @@
+[{"resource_type": "DIT", "course_id": "03-121", "course_name": "Modern Biology", "professor": null, "instructor": "Clara D.", "events": [{"weekday": 3, "date": null, "start_time": "08:00PM", "end_time": "10:00PM", "location": "Posner 282"}]}, {"resource_type": "DIT", "course_id": "06-361", "course_name": "Unit Operations", "professor": null, "instructor": "Alex P. & Chetan C.", "events": [{"weekday": 2, "date": null, "start_time": "06:00PM", "end_time": "09:00PM", "location": "Posner 282"}]}, {"resource_type": "DIT", "course_id": "06-364", "course_name": "Chemical Reaction Engineering", "professor": null, "instructor": "Alex P. & Chetan C.", "events": [{"weekday": 2, "date": null, "start_time": "06:00PM", "end_time": "09:00PM", "location": "Posner 282"}]}, {"resource_type": "DIT", "course_id": "09-105", "course_name": "Modern Chemistry I", "professor": null, "instructor": "Mars H. & Jisoo Y.", "events": [{"weekday": 1, "date": null, "start_time": "08:00PM", "end_time": "10:00PM", "location": "Posner 282"}]}, {"resource_type": "DIT", "course_id": "09-106", "course_name": "Modern Chemistry II", "professor": null, "instructor": "Mars H. & Jisoo Y.", "events": [{"weekday": 1, "date": null, "start_time": "08:00PM", "end_time": "10:00PM", "location": "Posner 282"}]}, {"resource_type": "DIT", "course_id": "09-218", "course_name": "Organic Chemistry II", "professor": null, "instructor": "Mackenzie R.", "events": [{"weekday": 2, "date": null, "start_time": "08:00PM", "end_time": "10:00PM", "location": "Posner A62"}]}, {"resource_type": "DIT", "course_id": "09-220", "course_name": "Modern Organic Chemistry II", "professor": null, "instructor": "Mackenzie R.", "events": [{"weekday": 2, "date": null, "start_time": "08:00PM", "end_time": "10:00PM", "location": "Posner A62"}]}, {"resource_type": "DIT", "course_id": "09-348", "course_name": "Inorganic Chemistry", "professor": null, "instructor": "Rafael G.", "events": [{"weekday": 2, "date": null, "start_time": "08:00PM", "end_time": "10:00PM", "location": "Posner A63"}]}, {"resource_type": "DIT", "course_id": "15-110", "course_name": "Principles of Computing", "professor": null, "instructor": "Sachi A.", "events": [{"weekday": 7, "date": null, "start_time": "08:00PM", "end_time": "10:00PM", "location": "Fifth & Clyde Neighborhood Space"}]}, {"resource_type": "DIT", "course_id": "15-112", "course_name": "Fundamentals of Programming", "professor": null, "instructor": "Wesley T.", "events": [{"weekday": 1, "date": null, "start_time": "08:00PM", "end_time": "10:00PM", "location": "Posner A63"}]}, {"resource_type": "DIT", "course_id": "15-122", "course_name": "Principles of Imperative Computing", "professor": null, "instructor": "Rozay C.", "events": [{"weekday": 4, "date": null, "start_time": "08:00PM", "end_time": "10:00PM", "location": "Posner 282"}]}, {"resource_type": "DIT", "course_id": "15-251", "course_name": "Great Ideas in Theoretical Computer Science", "professor": null, "instructor": "Rozay C.", "events": [{"weekday": 4, "date": null, "start_time": "08:00PM", "end_time": "10:00PM", "location": "Posner 282"}]}, {"resource_type": "DIT", "course_id": "15-150", "course_name": "Principles of Functional Programming", "professor": null, "instructor": "Helena Y.", "events": [{"weekday": 7, "date": null, "start_time": "08:00PM", "end_time": "10:00PM", "location": "Posner 282"}]}, {"resource_type": "DIT", "course_id": "18-220", "course_name": "Electronic Devices and Analog Circuits", "professor": null, "instructor": "Sid S.", "events": [{"weekday": 1, "date": null, "start_time": "08:00PM", "end_time": "10:00PM", "location": "Posner 280"}]}, {"resource_type": "DIT", "course_id": "18-213", "course_name": "Intro to Computer Systems", "professor": null, "instructor": "Sid S.", "events": [{"weekday": 1, "date": null, "start_time": "08:00PM", "end_time": "10:00PM", "location": "Posner 280"}]}, {"resource_type": "DIT", "course_id": "21-122", "course_name": "Integration & Approximation", "professor": null, "instructor": "Lucas Y.", "events": [{"weekday": 1, "date": null, "start_time": "08:00PM", "end_time": "10:00PM", "location": "Posner A62"}]}, {"resource_type": "DIT", "course_id": "21-127", "course_name": "Concepts", "professor": null, "instructor": "Michael P.", "events": [{"weekday": 7, "date": null, "start_time": "08:00PM", "end_time": "10:00PM", "location": "Posner A63"}]}, {"resource_type": "DIT", "course_id": "21-256", "course_name": "Multivariate Analysis", "professor": null, "instructor": "Annie Y.", "events": [{"weekday": 7, "date": null, "start_time": "04:00PM", "end_time": "06:00PM", "location": "Posner A62"}]}, {"resource_type": "DIT", "course_id": "21-259", "course_name": "Calculus in 3D", "professor": null, "instructor": "Aidan C.", "events": [{"weekday": 7, "date": null, "start_time": "08:00PM", "end_time": "10:00PM", "location": "Posner 280"}]}, {"resource_type": "DIT", "course_id": "21-254", "course_name": "Linear Algebra", "professor": null, "instructor": "Claire S.", "events": [{"weekday": 4, "date": null, "start_time": "08:00PM", "end_time": "10:00PM", "location": "Fifth & Clyde Neighborhood Space"}]}, {"resource_type": "DIT", "course_id": "24-231", "course_name": "Fluid Mechanics", "professor": null, "instructor": "Marianne B.", "events": [{"weekday": 3, "date": null, "start_time": "08:00PM", "end_time": "10:00PM", "location": "Posner A62"}]}, {"resource_type": "DIT", "course_id": "24-262", "course_name": "Mechanics II", "professor": null, "instructor": "Marianne B.", "events": [{"weekday": 3, "date": null, "start_time": "08:00PM", "end_time": "10:00PM", "location": "Posner A62"}]}, {"resource_type": "DIT", "course_id": "33-122", "course_name": "Physics II for Science Students", "professor": null, "instructor": "Yi Lin Q.", "events": [{"weekday": 2, "date": null, "start_time": "08:00PM", "end_time": "10:00PM", "location": "Posner 280"}]}, {"resource_type": "DIT", "course_id": "36-200", "course_name": "Reasoning with Data", "professor": null, "instructor": "Pranav S.", "events": [{"weekday": 4, "date": null, "start_time": "08:00PM", "end_time": "10:00PM", "location": "Posner 280"}]}, {"resource_type": "DIT", "course_id": "36-202", "course_name": "Methods for Statistics & Data Science", "professor": null, "instructor": "Pranav S.", "events": [{"weekday": 4, "date": null, "start_time": "08:00PM", "end_time": "10:00PM", "location": "Posner 280"}]}]

backend/scraper/handshake.json

@@ -0,0 +1 @@
+[{"resource_type": "Career", "resource_source": "Handshake", "event_name": "5G University Program for CMU - Qualcomm Wireless Academy", "event_host": "Qualcomm", "events": [{"weekday": 2, "date": "03-28-2023", "start_time": "03:00AM", "end_time": "03:00AM", "location": "Virtual"}], "categories": ["Hiring", "Guidance"]}, {"resource_type": "Career", "resource_source": "Handshake", "event_name": "Virtual FBI STEM Speaker Series", "event_host": "Federal Bureau of Investigation", "events": [{"weekday": 4, "date": "05-16-2024", "start_time": "01:00PM", "end_time": "02:00PM", "location": "Virtual"}], "categories": ["Hiring", "Employers"]}, {"resource_type": "Career", "resource_source": "Handshake", "event_name": "SMBC Meet and Greet (Leasing & Finance, Real Estate, Latin America Project Groups)", "event_host": "SMBC", "events": [{"weekday": 4, "date": "05-30-2024", "start_time": "01:00PM", "end_time": "02:00PM", "location": "Virtual"}], "categories": ["Hiring", "Employers"]}, {"resource_type": "Career", "resource_source": "Handshake", "event_name": "AT&T Technology Academy", "event_host": "AT&T", "events": [{"weekday": 1, "date": "06-10-2024", "start_time": "08:00AM", "end_time": "06:00PM", "location": "Virtual"}], "categories": ["Employers", "Guidance"]}, {"resource_type": "Career", "resource_source": "Handshake", "event_name": "*VIRTUAL: Summer Hires @ Heinz Career Fair", "event_host": "Carnegie Mellon University - CMU", "events": [{"weekday": 3, "date": "06-26-2024", "start_time": "12:00PM", "end_time": "02:00PM", "location": "Virtual"}], "categories": ["Career fairs"]}, {"resource_type": "Career", "resource_source": "Handshake", "event_name": "*Learn to Earn: Negotiating an Offer* CMU-SV", "event_host": "Carnegie Mellon University - CMU", "events": [{"weekday": 5, "date": "07-19-2024", "start_time": "03:00PM", "end_time": "04:00PM", "location": "Virtual"}], "categories": ["Guidance"]}, {"resource_type": "Career", "resource_source": "Handshake", "event_name": "*Starting Strong! Prepare for Your Full-Time Job Search* CMU-SV ", "event_host": "Carnegie Mellon University - CMU", "events": [{"weekday": 5, "date": "08-09-2024", "start_time": "03:00PM", "end_time": "04:00PM", "location": "Virtual"}], "categories": ["Guidance"]}, {"resource_type": "Career", "resource_source": "Handshake", "event_name": "Virtual Learning Event: Systems Engineering & Project Management", "event_host": "AI Futures", "events": [{"weekday": 4, "date": "06-20-2024", "start_time": "12:00PM", "end_time": "01:00PM", "location": "Virtual"}], "categories": ["Networking"]}, {"resource_type": "Career", "resource_source": "Handshake", "event_name": "Women in Wealth: College Leadership Academy", "event_host": "Morgan Stanley", "events": [{"weekday": 4, "date": "06-20-2024", "start_time": "12:00PM", "end_time": "01:00PM", "location": "Virtual"}], "categories": ["Hiring", "Employers"]}, {"resource_type": "Career", "resource_source": "Handshake", "event_name": "Pride Month", "event_host": "Bank of America", "events": [{"weekday": 4, "date": "06-20-2024", "start_time": "03:00PM", "end_time": "04:30PM", "location": "Virtual"}], "categories": ["Networking"]}, {"resource_type": "Career", "resource_source": "Handshake", "event_name": "Handshake 101: how to make the most of your summer using Handshake", "event_host": "Handshake", "events": [{"weekday": 4, "date": "06-20-2024", "start_time": "03:00PM", "end_time": "04:00PM", "location": "Virtual"}], "categories": ["Guidance"]}, {"resource_type": "Career", "resource_source": "Handshake", "event_name": "AmeriCorps NCCC: Application and Selection Webinar", "event_host": "AmeriCorps", "events": [{"weekday": 4, "date": "06-20-2024", "start_time": "04:00PM", "end_time": "05:00PM", "location": "Virtual"}], "categories": ["Hiring", "Guidance"]}, {"resource_type": "Career", "resource_source": "Handshake", "event_name": "Students & Grads Virtual Event: Business Case Workshop", "event_host": "Capital One", "events": [{"weekday": 4, "date": "06-20-2024", "start_time": "04:00PM", "end_time": "05:30PM", "location": "Virtual"}], "categories": ["Employers", "Networking"]}, {"resource_type": "Career", "resource_source": "Handshake", "event_name": "Mayo Clinic Administrative Fellowship Webinar June Session 2024", "event_host": "Mayo Clinic", "events": [{"weekday": 4, "date": "06-20-2024", "start_time": "06:00PM", "end_time": "07:15PM", "location": "Virtual"}], "categories": ["Hiring", "Employers"]}, {"resource_type": "Career", "resource_source": "Handshake", "event_name": "Veterans - Get into Energy with NiSource", "event_host": "NiSource, Inc.", "events": [{"weekday": 4, "date": "06-20-2024", "start_time": "06:30PM", "end_time": "07:30PM", "location": "Virtual"}], "categories": ["Employers"]}, {"resource_type": "Career", "resource_source": "Handshake", "event_name": "Learn About Capital One's 2025 Analyst Program & Recruiter Q & A ", "event_host": "Capital One", "events": [{"weekday": 5, "date": "06-21-2024", "start_time": "12:00PM", "end_time": "01:30PM", "location": "Virtual"}], "categories": ["Employers", "Networking"]}, {"resource_type": "Career", "resource_source": "Handshake", "event_name": "Learn About Capital One's 2025 Tech Program & Recruiter Q&A", "event_host": "Capital One", "events": [{"weekday": 1, "date": "06-24-2024", "start_time": "12:00PM", "end_time": "01:00PM", "location": "Virtual"}], "categories": ["Employers"]}, {"resource_type": "Career", "resource_source": "Handshake", "event_name": "BCG Launch ", "event_host": "Boston Consulting Group (BCG)", "events": [{"weekday": 1, "date": "06-24-2024", "start_time": "07:00PM", "end_time": "07:00PM", "location": "Virtual"}], "categories": ["Hiring", "Employers"]}, {"resource_type": "Career", "resource_source": "Handshake", "event_name": "SMBC Reality Series: Being Black in Banking", "event_host": "SMBC", "events": [{"weekday": 2, "date": "06-25-2024", "start_time": "12:00PM", "end_time": "01:00PM", "location": "Virtual"}], "categories": ["Employers", "Guidance"]}, {"resource_type": "Career", "resource_source": "Handshake", "event_name": "Students & Grads Virtual Event: What is a Business Analyst in Dallas?", "event_host": "Capital One", "events": [{"weekday": 2, "date": "06-25-2024", "start_time": "01:00PM", "end_time": "02:00PM", "location": "Virtual"}], "categories": ["Employers", "Guidance"]}, {"resource_type": "Career", "resource_source": "Handshake", "event_name": "SMBC Reality Series: Being LatinX in Banking", "event_host": "SMBC", "events": [{"weekday": 2, "date": "06-25-2024", "start_time": "02:00PM", "end_time": "03:00PM", "location": "Virtual"}], "categories": ["Employers", "Guidance"]}, {"resource_type": "Career", "resource_source": "Handshake", "event_name": "Amazon Culture \u2014How Glamazon fosters community & promotes Diversity and Inclusion", "event_host": "Amazon", "events": [{"weekday": 2, "date": "06-25-2024", "start_time": "02:00PM", "end_time": "05:00PM", "location": "Virtual"}], "categories": ["Employers"]}, {"resource_type": "Career", "resource_source": "Handshake", "event_name": "Pride & Progress: Spectrum Employee Voices", "event_host": "Tennessee Valley Authority (TVA)", "events": [{"weekday": 2, "date": "06-25-2024", "start_time": "03:00PM", "end_time": "04:00PM", "location": "Virtual"}], "categories": ["Hiring", "Employers"]}, {"resource_type": "Career", "resource_source": "Handshake", "event_name": "FBI Special Agent Info/Q&A Session (Hosted by FBI San Francisco)", "event_host": "Federal Bureau of Investigation", "events": [{"weekday": 2, "date": "06-25-2024", "start_time": "03:30PM", "end_time": "04:30PM", "location": "Virtual"}], "categories": ["Employers"]}, {"resource_type": "Career", "resource_source": "Handshake", "event_name": "Joining AmeriCorps: Pathways to Employment", "event_host": "AmeriCorps", "events": [{"weekday": 2, "date": "06-25-2024", "start_time": "04:00PM", "end_time": "05:00PM", "location": "Virtual"}], "categories": ["Hiring", "Employers"]}, {"resource_type": "Career", "resource_source": "Handshake", "event_name": "SMBC Reality Series: Being Asian in Banking", "event_host": "SMBC", "events": [{"weekday": 2, "date": "06-25-2024", "start_time": "04:00PM", "end_time": "05:00PM", "location": "Virtual"}], "categories": ["Employers", "Guidance"]}, {"resource_type": "Career", "resource_source": "Handshake", "event_name": "Successful Interviewing: Tips from Recruiters and Hiring Managers ", "event_host": "PACCAR", "events": [{"weekday": 2, "date": "06-25-2024", "start_time": "04:00PM", "end_time": "05:00PM", "location": "Virtual"}], "categories": ["Guidance"]}, {"resource_type": "Career", "resource_source": "Handshake", "event_name": " Learn About Capital One's 2025 Analyst Program & Recruiter Q&A", "event_host": "Capital One", "events": [{"weekday": 2, "date": "06-25-2024", "start_time": "05:00PM", "end_time": "06:30PM", "location": "Virtual"}], "categories": ["Employers", "Networking"]}, {"resource_type": "Career", "resource_source": "Handshake", "event_name": "Calling All Changemakers:  Career (& Life) Lessons You Can Use Now - Ep. 1", "event_host": "Teach For America", "events": [{"weekday": 2, "date": "06-25-2024", "start_time": "06:00PM", "end_time": "07:00PM", "location": "Virtual"}], "categories": ["Guidance", "Conference"]}, {"resource_type": "Career", "resource_source": "Handshake", "event_name": "Veterans - Get into Energy with NiSource", "event_host": "NiSource, Inc.", "events": [{"weekday": 2, "date": "06-25-2024", "start_time": "06:30PM", "end_time": "07:30PM", "location": "Virtual"}], "categories": ["Employers"]}]