diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 94663b5c..0f84605d 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -50,6 +50,9 @@ jobs: - name: Install Requirements run: python3 -m pip install --upgrade pip && pip3 install wheel && pip3 install -r requirements.txt && pip3 uninstall -y typing + + - name: Write secrets to config file + run: echo "[analytics]\napi_key=${{ secrets.GA4_SECRET_CLI}}" > config.ini - name: Build Wheel run: python3 setup.py bdist_wheel diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 00000000..4451cd0e --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1 @@ +include config.ini \ No newline at end of file diff --git a/config.ini b/config.ini new file mode 100644 index 00000000..8e1acda0 --- /dev/null +++ b/config.ini @@ -0,0 +1,2 @@ +[analytics] +api_key={{ secrets.GA4_API_KEY}} diff --git a/pros/cli/build.py b/pros/cli/build.py index bd9fdcb8..581cacd1 100644 --- a/pros/cli/build.py +++ b/pros/cli/build.py @@ -21,7 +21,7 @@ def make(project: c.Project, build_args): """ Build current PROS project or cwd """ - analytics.send("make") + analytics.send("make", dict.fromkeys(build_args)) exit_code = project.compile(build_args) if exit_code != 0: logger(__name__).error(f'Failed to make project: Exit Code {exit_code}', extra={'sentry': False}) @@ -35,7 +35,11 @@ def make(project: c.Project, build_args): @project_option() @click.pass_context def make_upload(ctx, project: c.Project, build_args: List[str], **upload_args): - analytics.send("make-upload") + options = dict.fromkeys(build_args) + # add target, after, icon, and slot from upload_args to options. dont include other keys from upload_args + options.update({k: v for k, v in upload_args.items() if k in ['target', 'after', 'icon', 'slot']}) + analytics.send("make_upload", options) + ctx.invoke(make, project=project, build_args=build_args) ctx.invoke(upload, project=project, **upload_args) @@ -46,7 +50,11 @@ def make_upload(ctx, project: c.Project, build_args: List[str], **upload_args): @project_option() @click.pass_context def make_upload_terminal(ctx, project: c.Project, build_args, **upload_args): - analytics.send("make-upload-terminal") + options = dict.fromkeys(build_args) + # add target, after, icon, and slot from upload_args to options. dont include other keys from upload_args + options.update({k: v for k, v in upload_args.items() if k in ['target', 'after', 'icon', 'slot']}) + analytics.send("make_upload_terminal", options) + from .terminal import terminal ctx.invoke(make, project=project, build_args=build_args) ctx.invoke(upload, project=project, **upload_args) @@ -67,7 +75,7 @@ def build_compile_commands(project: c.Project, suppress_output: bool, compile_co Build a compile_commands.json compatible with cquery :return: """ - analytics.send("build-compile-commands") + analytics.send("build_compile_commands", dict.fromkeys(build_args).update({'suppress_output': suppress_output, 'sandbox': sandbox, 'compile_commands': 1 if compile_commands else 0})) exit_code = project.make_scan_build(build_args, cdb_file=compile_commands, suppress_output=suppress_output, sandbox=sandbox) if exit_code != 0: diff --git a/pros/cli/conductor.py b/pros/cli/conductor.py index 06b26e86..faf20040 100644 --- a/pros/cli/conductor.py +++ b/pros/cli/conductor.py @@ -42,7 +42,16 @@ def fetch(query: c.BaseTemplate): Visit https://pros.cs.purdue.edu/v5/cli/conductor.html to learn more """ - analytics.send("fetch-template") + + + # query identifier can be a path to a local template, or just the template name@version. we want to check if it's a path, and if it is, extract the name@version from the full path + # we do this so that we don't send user's full paths to analytics, as that's a bit of a privacy concern (could contain username, etc.) + name_version = query.identifier + if os.path.exists(query.identifier): + name_version = os.path.basename(query.identifier) + analytics.send("fetch_template", {'template': name_version.replace("@", "_")}) + + template_file = None if os.path.exists(query.identifier): template_file = query.identifier @@ -102,7 +111,12 @@ def apply(project: c.Project, query: c.BaseTemplate, **kwargs): Visit https://pros.cs.purdue.edu/v5/cli/conductor.html to learn more """ - analytics.send("apply-template") + # query identifier can be a path to a local template, or just the template name@version. we want to check if it's a path, and if it is, extract the name@version from the full path + # we do this so that we don't send user's full paths to analytics, as that's a bit of a privacy concern (could contain username, etc.) + name_version = query.identifier + if os.path.exists(query.identifier): + name_version = os.path.basename(query.identifier) + analytics.send("apply_template", {'template': name_version.replace("@", "_"), "beta": kwargs.get('beta'), "upgrade_ok": kwargs.get('upgrade_ok'), "install_ok": kwargs.get('install_ok')}) return c.Conductor().apply_template(project, identifier=query, **kwargs) @@ -127,7 +141,9 @@ def install(ctx: click.Context, **kwargs): Visit https://pros.cs.purdue.edu/v5/cli/conductor.html to learn more """ - analytics.send("install-template") + # im gonna have analytics on this for now, just to prove that this command is not used. all it is is a wrapper for apply with install_ok=True, + # however, install_ok is True by default, so this command is literally useless + analytics.send("install_template") return ctx.invoke(apply, install_ok=True, **kwargs) @@ -154,7 +170,9 @@ def upgrade(ctx: click.Context, project: c.Project, query: c.BaseTemplate, **kwa Visit https://pros.cs.purdue.edu/v5/cli/conductor.html to learn more """ - analytics.send("upgrade-project") + templatename = query.name if query.name else "All" + print(kwargs) + analytics.send("upgrade_template", {"template_name": templatename, "beta": kwargs.get('beta'), "download_ok": kwargs.get('download_ok'), "install_ok": kwargs.get('install_ok')}) if not query.name: for template in project.templates.keys(): click.secho(f'Upgrading {template}', color='yellow') @@ -180,7 +198,9 @@ def uninstall_template(project: c.Project, query: c.BaseTemplate, remove_user: b Visit https://pros.cs.purdue.edu/v5/cli/conductor.html to learn more """ - analytics.send("uninstall-template") + templatename = query.name if query.name else "All" + analytics.send("uninstall_template", {"template_name": templatename, "remove_user": remove_user, "remove_empty_dirs": remove_empty_directories, "no_make_clean": no_make_clean}) + c.Conductor().remove_template(project, query, remove_user=remove_user, remove_empty_directories=remove_empty_directories) if no_make_clean: @@ -210,14 +230,16 @@ def uninstall_template(project: c.Project, query: c.BaseTemplate, remove_user: b @default_options def new_project(ctx: click.Context, path: str, target: str, version: str, force_user: bool = False, force_system: bool = False, - no_default_libs: bool = False, compile_after: bool = True, build_cache: bool = None, **kwargs): + no_default_libs: bool = False, compile_after: bool = True, build_cache: bool = None, beta: bool = False, **kwargs): """ Create a new PROS project Visit https://pros.cs.purdue.edu/v5/cli/conductor.html to learn more """ - analytics.send("new-project") + analytics.send("new_project", {"version": version, "no_default_libs": no_default_libs, "compile_after": compile_after, "build_cache": build_cache, "beta": beta}) + version_source = version.lower() == 'latest' + if version.lower() == 'latest' or not version: version = '>0' if not force_system and c.Project.find_project(path) is not None: @@ -270,7 +292,7 @@ def query_templates(ctx, query: c.BaseTemplate, allow_offline: bool, allow_onlin Visit https://pros.cs.purdue.edu/v5/cli/conductor.html to learn more """ - analytics.send("query-templates") + analytics.send("query_templates", {"allow_offline": allow_offline, "allow_online": allow_online, "force_refresh": force_refresh, "limit": limit, "beta": beta}) if limit < 0: limit = 15 templates = c.Conductor().resolve_templates(query, allow_offline=allow_offline, allow_online=allow_online, @@ -313,7 +335,7 @@ def info_project(project: c.Project, ls_upgrades): Visit https://pros.cs.purdue.edu/v5/cli/conductor.html to learn more """ - analytics.send("info-project") + analytics.send("info_project") from pros.conductor.project import ProjectReport report = ProjectReport(project) _conductor = c.Conductor() diff --git a/pros/cli/conductor_utils.py b/pros/cli/conductor_utils.py index cb22cffc..3894752f 100644 --- a/pros/cli/conductor_utils.py +++ b/pros/cli/conductor_utils.py @@ -55,7 +55,7 @@ def create_template(ctx, path: str, destination: str, do_zip: bool, **kwargs): pros conduct create-template . libblrs 2.0.1 --system "firmware/*.a" --system "include/*.h" """ - analytics.send("create-template") + analytics.send("create_template", {"kernels": kwargs['supported_kernels'], "target": kwargs['target'], "zip": do_zip, "name": kwargs['name']}) project = c.Project.find_project(path, recurse_times=1) if project: project = c.Project(project) @@ -152,7 +152,7 @@ def filename_remap(file_path: str) -> str: @template_query(required=False) @default_options def purge_template(query: c.BaseTemplate, force): - analytics.send("purge-template") + analytics.send("purge_template", {"template": query.identifier if query else None, "force": force}) if not query: force = click.confirm('Are you sure you want to remove all cached templates? This action is non-reversable!', abort=True) diff --git a/pros/cli/main.py b/pros/cli/main.py index 679d9897..5ab63d39 100644 --- a/pros/cli/main.py +++ b/pros/cli/main.py @@ -102,6 +102,17 @@ def use_analytics(ctx: click.Context, param, value): ui.echo('Analytics set to : {}'.format(analytics.useAnalytics)) ctx.exit(0) +def show_analytics(ctx: click.Context, param, value): + ctx.ensure_object(dict) + # if analytics are currently enabled + if(analytics.useAnalytics): + ui.echo('\nAnalytics are currently ENABLED.\n') + ui.echo('DATA currently being collected is as follows:\n 1) Commands being run\n 2) Non identifying command arguments') + else: + ui.echo('\nAnalytics are currently DISABLED.') + + ctx.exit(0) + @click.command('pros', cls=PROSCommandCollection, @@ -112,6 +123,8 @@ def use_analytics(ctx: click.Context, param, value): callback=version) @click.option('--use-analytics', help='Set analytics usage (True/False).', type=str, expose_value=False, is_eager=True, default=None, callback=use_analytics) +@click.option('--show-analytics', help='Show current analytics usage.', is_flag=True, expose_value=False, + is_eager=True, default=False, callback=show_analytics) def cli(ctx): pros.common.sentry.register() ctx.call_on_close(after_command) diff --git a/pros/cli/misc_commands.py b/pros/cli/misc_commands.py index 8566456a..8127c27b 100644 --- a/pros/cli/misc_commands.py +++ b/pros/cli/misc_commands.py @@ -17,12 +17,15 @@ def upgrade(force_check, no_install): """ Check for updates to the PROS CLI """ + + analytics.send("upgrade", {"force_check": force_check, "no_install": no_install}) + with ui.Notification(): ui.echo('The "pros upgrade" command is currently non-functioning. Did you mean to run "pros c upgrade"?', color='yellow') return # Dead code below - analytics.send("upgrade") + from pros.upgrade import UpgradeManager manager = UpgradeManager() manifest = manager.get_manifest(force_check) diff --git a/pros/cli/terminal.py b/pros/cli/terminal.py index 2f05f2fe..419c1b8f 100644 --- a/pros/cli/terminal.py +++ b/pros/cli/terminal.py @@ -43,7 +43,7 @@ def terminal(port: str, backend: str, **kwargs): Note: share backend is not yet implemented. """ - analytics.send("terminal") + analytics.send("terminal", {"raw": kwargs.get('raw', False), "hex": kwargs.get('hex', False), "banner": kwargs.get('request_banner', True), "output": True if kwargs.get('output', None) else False}) from pros.serial.devices.vex.v5_user_device import V5UserDevice from pros.serial.terminal import Terminal is_v5_user_joystick = False diff --git a/pros/cli/upload.py b/pros/cli/upload.py index 545609a4..85773545 100644 --- a/pros/cli/upload.py +++ b/pros/cli/upload.py @@ -53,7 +53,9 @@ def upload(path: Optional[str], project: Optional[c.Project], port: str, **kwarg [PORT] may be any valid communication port file, such as COM1 or /dev/ttyACM0. If left blank, then a port is automatically detected based on the target (or as supplied by the PROS project) """ - analytics.send("upload") + # send analytics with target, after, slot, and icon + analytics.send('upload', {"target": kwargs.get('target', None), "after": kwargs.get('after', None), "slot": kwargs.get('slot', None), "icon": kwargs.get('icon', None)}) + import pros.serial.devices.vex as vex from pros.serial.ports import DirectPort kwargs['ide_version'] = project.kernel if not project==None else "None" @@ -164,7 +166,7 @@ def ls_usb(target): """ List plugged in VEX Devices """ - analytics.send("ls-usb") + analytics.send("ls_usb") from pros.serial.devices.vex import find_v5_ports, find_cortex_ports class PortReport(object): @@ -204,7 +206,9 @@ def __str__(self): @shadow_command(upload) @click.pass_context def make_upload_terminal(ctx, **upload_kwargs): - analytics.send("upload-terminal") + # send analytics with target, after, slot, and icon + analytics.send_analytics('upload_terminal', {"target": upload_kwargs.get('target', None), "after": upload_kwargs.get('after', None), "slot": upload_kwargs.get('slot', None), "icon": upload_kwargs.get('icon', None)}) + from .terminal import terminal ctx.invoke(upload, **upload_kwargs) ctx.invoke(terminal, request_banner=False) diff --git a/pros/cli/user_script.py b/pros/cli/user_script.py index be0f8259..bf4f4357 100644 --- a/pros/cli/user_script.py +++ b/pros/cli/user_script.py @@ -16,7 +16,7 @@ def user_script(script_file): """ Run a script file with the PROS CLI package """ - analytics.send("user-script") + analytics.send("user_script") import os.path import importlib.util package_name = os.path.splitext(os.path.split(script_file)[0])[0] diff --git a/pros/cli/v5_utils.py b/pros/cli/v5_utils.py index 7c0809bd..5dae6c45 100644 --- a/pros/cli/v5_utils.py +++ b/pros/cli/v5_utils.py @@ -47,7 +47,7 @@ def ls_files(port: str, vid: int, options: int): """ List files on the flash filesystem """ - analytics.send("ls-files") + analytics.send("ls_files") from pros.serial.devices.vex import V5Device from pros.serial.ports import DirectPort port = resolve_v5_port(port, 'system')[0] @@ -72,7 +72,7 @@ def read_file(file_name: str, port: str, vid: int, source: str): """ Read file on the flash filesystem to stdout """ - analytics.send("read-file") + analytics.send("read_file") from pros.serial.devices.vex import V5Device from pros.serial.ports import DirectPort port = resolve_v5_port(port, 'system')[0] @@ -98,7 +98,7 @@ def write_file(file, port: str, remote_file: str, **kwargs): """ Write a file to the V5. """ - analytics.send("write-file") + analytics.send("write_file") from pros.serial.ports import DirectPort from pros.serial.devices.vex import V5Device port = resolve_v5_port(port, 'system')[0] @@ -121,7 +121,7 @@ def rm_file(file_name: str, port: str, vid: int, erase_all: bool): """ Remove a file from the flash filesystem """ - analytics.send("rm-file") + analytics.send("rm_file") from pros.serial.devices.vex import V5Device from pros.serial.ports import DirectPort port = resolve_v5_port(port, 'system')[0] @@ -142,7 +142,7 @@ def cat_metadata(file_name: str, port: str, vid: int): """ Print metadata for a file """ - analytics.send("cat-metadata") + analytics.send("cat_metadata") from pros.serial.devices.vex import V5Device from pros.serial.ports import DirectPort port = resolve_v5_port(port, 'system')[0] @@ -162,6 +162,7 @@ def rm_program(slot: int, port: str, vid: int): """ Remove a program from the flash filesystem """ + analytics.send("rm_program") from pros.serial.devices.vex import V5Device from pros.serial.ports import DirectPort port = resolve_v5_port(port, 'system')[0] @@ -182,7 +183,7 @@ def rm_all(port: str, vid: int): """ Remove all user programs from the V5 """ - analytics.send("rm-all") + analytics.send("rm_all") from pros.serial.devices.vex import V5Device from pros.serial.ports import DirectPort port = resolve_v5_port(port, 'system')[0] @@ -207,7 +208,7 @@ def run(slot: str, port: str): """ Run a V5 program """ - analytics.send("run") + analytics.send("run", {"slot": slot}) from pros.serial.devices.vex import V5Device from pros.serial.ports import DirectPort file = f'slot_{slot}.bin' @@ -232,6 +233,7 @@ def stop(port: str): If FILE is unspecified or is a directory, then attempts to find the correct filename based on the PROS project """ + analytics.send("stop") from pros.serial.devices.vex import V5Device from pros.serial.ports import DirectPort port = resolve_v5_port(port, 'system')[0] @@ -251,6 +253,7 @@ def capture(file_name: str, port: str, force: bool = False): """ Take a screen capture of the display """ + analytics.send("capture_screen", {"force": force}) from pros.serial.devices.vex import V5Device from pros.serial.ports import DirectPort import png @@ -297,6 +300,7 @@ def capture(file_name: str, port: str, force: bool = False): @click.argument('port', type=str, default=None, required=False) @default_options def set_variable(variable, value, port): + analytics.send("set_variable", {"variable": value}) import pros.serial.devices.vex as vex from pros.serial.ports import DirectPort @@ -313,6 +317,7 @@ def set_variable(variable, value, port): @click.argument('port', type=str, default=None, required=False) @default_options def read_variable(variable, port): + analytics.send("read_variable", {"variable": variable}) import pros.serial.devices.vex as vex from pros.serial.ports import DirectPort diff --git a/pros/ga/analytics.py b/pros/ga/analytics.py index 6f786105..1b8a9c2b 100644 --- a/pros/ga/analytics.py +++ b/pros/ga/analytics.py @@ -1,14 +1,17 @@ -import json -from os import path +from configparser import ConfigParser +import sys import uuid -import requests +import time from requests_futures.sessions import FuturesSession -import random from concurrent.futures import as_completed +from pros.common.utils import get_version -url = 'https://www.google-analytics.com/collect' agent = 'pros-cli' +config = ConfigParser() +config.read('config.ini') + + """ PROS ANALYTICS CLASS """ @@ -18,48 +21,87 @@ def __init__(self): from pros.config.cli_config import cli_config as get_cli_config self.cli_config = get_cli_config() #If GA hasn't been setup yet (first time install/update) - if not self.cli_config.ga: - #Default values for GA + if not self.cli_config.ga or not self.cli_config.ga.get("unix_timestamp", None): + + ''' + We need to ask the user if they want to opt in to analytics. If they do, we generate a UUID for them and a unix timestamp of when they opted in. + We also need to tell them what we collect, and how they can opt out. This *should* meet some level of GDPR compliance, but I'm not an expert so not positive. + https://gdpr.eu/gdpr-consent-requirements/ this official page seems to agree. "One easy way to avoid large GDPR fines is to always get permission from your users before using their personal data" + + ''' + + print("PROS CLI collects analytics while in use in order to better understand how people use PROS and improve this software. The data collected is as follows:\n 1) Commands being run\n 2) Non identifying command arguments.") + print("The scope of collected data may be expanded in the future. To see what data is actively being collected, run the command `pros --show-analytics`") + print("We do not collect any personal information, or specifics of your projects, file paths, etc.") + print("You may opt out of analytics at any time by running `pros --use-analytics False`, or may opt out for a single command by adding the `--no-analytics` flag.") + print("For questions or concerns, please contact us at pros_development@cs.purdue.edu\n") + response = input("Do you choose to opt in to analytics? (y/N): ").lower() + while response not in ["y", "n"]: + print("Invalid response. Please enter y or n.") + response = input("Do you choose to opt in to analytics? (y/N): ").lower() + + if response == "y": + response = "True" + print("Thank you for opting in to analytics! You may opt out at any time by running the command `pros --use-analytics False`, or for a specific command by adding the `--no-analytics` flag.") + else: + response = "False" + print("You have opted out of analytics. You may opt back in at any time by running the command `pros --use-analytics True`.") + # Default values for GA + # generate a unix timestamp self.cli_config.ga = { - "enabled": "True", - "ga_id": "UA-84548828-8", + "enabled": response, + "unix_timestamp": int(time.time()), "u_id": str(uuid.uuid4()) } self.cli_config.save() self.sent = False #Variables that the class will use - self.gaID = self.cli_config.ga['ga_id'] + self.user_timestamp = self.cli_config.ga['unix_timestamp'] self.useAnalytics = self.cli_config.ga['enabled'] self.uID = self.cli_config.ga['u_id'] self.pendingRequests = [] - def send(self,action): + def send(self, action, kw={}): + # Send analytics to GA if not self.useAnalytics or self.sent: return - self.sent=True # Prevent Send from being called multiple times + + self.sent = True # Prevent Send from being called multiple times try: - #Payload to be sent to GA, idk what some of them are but it works + # Copy kw to prevent modifying the original + kwargs = kw.copy() + kwargs["engagement_time_msec"] = 1 + kwargs["cli_version"] = get_version() + kwargs["platform"] = sys.platform + + for key, val in kwargs.items(): + # checking for required value + if val is None: + kwargs[key] = "Unspecified_Default" + + # Hardcoding api secret for now + # key = config['analytics']['api_key'] + + url = f'https://www.google-analytics.com/mp/collect?measurement_id=G-PXK9EBVY1Y&api_secret=DRDnqpaeTSebT7BsuGk_oA' payload = { - 'v': 1, - 'tid': self.gaID, - 'aip': 1, - 'z': random.random(), - 'cid': self.uID, - 't': 'event', - 'ec': 'action', - 'ea': action, - 'el': 'CLI', - 'ev': '1', - 'ni': 0 + "client_id": f'CLI.{self.user_timestamp}', + "user_id": self.uID, + "non_personalized_ads": True, + "events": [ + { + "name": action, + "params": kwargs + } + ] } session = FuturesSession() #Send payload to GA servers future = session.post(url=url, - data=payload, - headers={'User-Agent': agent}, - timeout=5.0) + data=str(payload).replace("False","false").replace("True","true").replace("Null","null"), + timeout=5.0, + verify = True) self.pendingRequests.append(future) except Exception as e: @@ -77,11 +119,12 @@ def process_requests(self): for future in as_completed(self.pendingRequests): try: response = future.result() - - if not response.status_code==200: + #print(response) + #print(vars(response)) + if not response.status_code==204: print("Something went wrong while sending analytics!") print(response) - + print(vars(response)) responses.append(response) except Exception: diff --git a/setup.py b/setup.py index f26a9741..19c5dc7f 100644 --- a/setup.py +++ b/setup.py @@ -18,5 +18,6 @@ 'pros=pros.cli.main:main', 'prosv5=pros.cli.main:main' ] - } + }, + include_package_data=True, )