Working version of permissions and ad networks, with only a few netwo…

…rks and apps so far

.pre-commit-config.yaml

@@ -6,7 +6,7 @@ repos:
  - id: black
  # Ruff
  - repo: https://github.com/charliermarsh/ruff-pre-commit
- rev: v0.1.15
+ rev: v0.2.1
  hooks:
  - id: ruff
  args: [--fix]

backend/api_app/controllers/apps.py

@@ -163,6 +163,7 @@ def get_app_overview_dict(collection: str) -> Collection:
 
 
 class AppController(Controller):
+
  """Controller holding all API endpoints for an app."""
 
  path = "/api/apps"
@@ -223,8 +224,6 @@ async def get_package_info(self: Self, store_id: str) -> PackageDetails:
  """
  logger.info(f"{self.path} start")
 
- # store_id='com.zhiliaoapp.musically'
-
  df = get_app_package_details(store_id)
 
  if df.empty:
@@ -242,13 +241,16 @@ async def get_package_info(self: Self, store_id: str) -> PackageDetails:
  is_android_activity = df["android_name"].str.contains(
  r"^(com.android)|(android)",
  )
-
+ trackers = ")|(".join(TRACKER_PACKAGE_IDS)
+ trackers = f"^({trackers})"
  is_tracker = df["android_name"].str.contains(
- f"^({")|(".join(TRACKER_PACKAGE_IDS)})",
+ trackers,
  )
 
+ ads = ")|(".join(AD_NETWORK_PACKAGE_IDS)
+ ads = f"({ads})"
  is_ads = df["android_name"].str.contains(
- f"({")|(".join(AD_NETWORK_PACKAGE_IDS)})",
+ ads,
  )
 
  permissions_df = df[is_permission]
@@ -263,15 +265,17 @@ async def get_package_info(self: Self, store_id: str) -> PackageDetails:
  & ~is_tracker
  & ~is_ads
  ]
-
- trackers_dict = {
- "trackers": tracker_df.android_name.tolist(),
- "permissions": permissions_df.android_name.tolist(),
- "ads": ads_df.android_name.tolist(),
- "android": android_services_df.android_name.tolist(),
- "leftovers": left_overs_df.android_name.tolist(),
- }
-
+ permissions_list = permissions_df.android_name.tolist()
+ permissions_list = [
+ x.replace("android.permission.", "") for x in permissions_list
+ ]
+ trackers_dict = PackageDetails(
+ trackers=tracker_df.android_name.tolist(),
+ permissions=permissions_list,
+ ads=ads_df.android_name.tolist(),
+ android=android_services_df.android_name.tolist(),
+ leftovers=left_overs_df.android_name.tolist(),
+ )
  return trackers_dict
 
  @get(path="/{store_id:str}/ranks", cache=3600)

backend/api_app/models.py

@@ -3,6 +3,7 @@
 
 @dataclass
 class AppDetail:
+
  """TODO: Fill out all details."""
 
  store_id: str
@@ -11,6 +12,7 @@ class AppDetail:
 
 @dataclass
 class PackageDetails:
+
  """Lists of android_name from Manifest."""
 
  permissions: list[str]

backend/dbcon/connections.py

@@ -47,6 +47,7 @@ def get_postgres_server_ips(server_name: str) -> tuple[str, str]:
 
 
 class PostgresCon:
+
  """Class for managing the connection to postgres
 
  Parameters

backend/dbcon/queries.py

@@ -308,7 +308,7 @@ def get_appstore_categories() -> pd.DataFrame:
  return df
 
 
-def query_ranks_for_app(store_id: str, days=30) -> pd.DataFrame:
+def query_ranks_for_app(store_id: str, days: int = 30) -> pd.DataFrame:
  start_date = (
  datetime.datetime.now(datetime.UTC) - datetime.timedelta(days=days)
  ).strftime("%Y-%m-%d")
@@ -529,6 +529,7 @@ def get_app_package_details(store_id: str) -> pd.DataFrame:
 
 def clean_app_df(df: pd.DataFrame) -> pd.DataFrame:
  """Apply generic cleaning for a DF with app data from store_apps table."""
+ df["store"] = df["store"].replace({1: "Google Play", 2: "Apple App Store"})
  string_nums = ["installs", "review_count", "rating_count"]
  for col in string_nums:
  df[f"{col}_num"] = df[col]

backend/pyproject.toml

@@ -15,22 +15,24 @@ requires = ["setuptools", "wheel"]
 
 [tool.ruff]
 #select = ["E", "W", "F", "B", "I", "N", "UP"]
-select = ["ALL"]
+lint.select = ["ALL"]
 
 target-version = 'py312'
 
-ignore = [
+lint.ignore = [
  "E501", # line length
  "W291", # trailing space at end of line
  "G", # Ignore pyflake logging as it does not like f-strings in log messages.
  "RET504", # Unnecessary assignment to variable before return, but I prefer for readability.
  "PD901", # avoid 'df' for dataframe, but I prefer this as the main df in each function
+ "D211", # Suppress ruff internal conflict
+ "D213", # Suppress ruff internal conflict
 ]
 
 # Avoid trying to fix flake8-bugbear (`B`) violations.
-unfixable = ["B"]
+lint.unfixable = ["B"]
 
-fixable = ["ALL"]
+lint.fixable = ["ALL"]
 
 exclude = ["tests/*"]
 

frontend/.prettierrc

@@ -3,7 +3,6 @@
  "singleQuote": true,
  "trailingComma": "none",
  "printWidth": 100,
- "plugins": ["prettier-plugin-svelte"],
- "pluginSearchDirs": ["."],
+ "plugins": ["./node_modules/prettier-plugin-svelte/plugin.js"],
  "overrides": [{ "files": "*.svelte", "options": { "parser": "svelte" } }]
 }

frontend/package.json

@@ -18,7 +18,7 @@
  "eslint": "^8.52.0",
  "eslint-config-prettier": "^9.1.0",
  "eslint-plugin-svelte": "^2.4.0",
- "prettier": "^3.2.4",
+ "prettier": "^3.2.5",
  "prettier-plugin-svelte": "^3.1.2",
  "svelte-check": "^3.5.2"
  },

frontend/src/routes/apps/[id]/+page.server.ts

@@ -14,6 +14,9 @@ export const load: PageServerLoad = async ({ params, setHeaders, locals }) => {
  const res = fetch(`http://localhost:8000/api/apps/${id}`);
 
  const ranks = fetch(`http://localhost:8000/api/apps/${id}/ranks`);
+
+ const packageInfo = fetch(`http://localhost:8000/api/apps/${id}/packageinfo`);
+
  try {
  return {
  myapp: {
@@ -57,6 +60,27 @@ export const load: PageServerLoad = async ({ params, setHeaders, locals }) => {
  return 'Uncaught Error';
  }
  )
+ },
+ myPackageInfo: {
+ streamed: packageInfo
+ .then((resp) => {
+ if (resp.status === 200) {
+ return resp.json();
+ } else if (resp.status === 404) {
+ console.log('App Ranks Not found');
+ return 'App Not Found';
+ } else if (resp.status === 500) {
+ console.log('Ranks API Server error');
+ return 'Backend Error';
+ }
+ })
+ .then(
+ (json) => json,
+ (error) => {
+ console.log('Uncaught error', error);
+ return 'Uncaught Error';
+ }
+ )
  }
  };
  } catch (error) {

frontend/src/routes/apps/[id]/+page.svelte

@@ -225,12 +225,46 @@
  <p>First Crawled: {appdata.created_at}</p>
  </div>
  </div>
+
+ {#await data.myPackageInfo.streamed}
+ Loading ...
+ {:then packageInfo}
+ {#if typeof packageInfo == 'string'}
+ <p>Permissions info not yet available for this app.</p>
+ {:else}
+ {#if packageInfo.permissions && packageInfo.permissions.length > 0}
+ <h4 class="h4 md:h3 p-2">Permissions</h4>
+ <div class="px-4 md:px-8">
+ {#each packageInfo.permissions as permission}
+ <p>{permission}</p>
+ {/each}
+ </div>
+ {/if}
+ {#if packageInfo.trackers && packageInfo.trackers.length > 0}
+ <h4 class="h4 md:h3 p-2">Trackers</h4>
+ <div class="px-4 md:px-8">
+ {#each packageInfo.trackers as tracker}
+ <p>{tracker}</p>
+ {/each}
+ </div>
+ {/if}
+ {#if packageInfo.ads && packageInfo.ads.length > 0}
+ <h4 class="h4 md:h3 p-2">Ad Networks</h4>
+ <div class="px-4 md:px-8">
+ {#each packageInfo.ads as ad}
+ <p>{ad}</p>
+ {/each}
+ </div>
+ {/if}
+ {/if}
+ {/await}
  </div>
  </section>
  {/if}
 {:catch}
  <p>The server caught an error. Please try again or try another app.</p>
 {/await}
+
 <a href="/"><p>Back to Home</p></a>
 
 <style>

frontend/src/types.ts

@@ -118,6 +118,15 @@ export interface AppFullDetails {
  myranks: {
  streamed: Promise<{ latest: AppRankDetail[]; history: AppRankDetail[] }>;
  };
+ myPackageInfo: {
+ streamed: Promise<{
+ permissions: string[];
+ trackers: string[];
+ ads: string[];
+ android: string[];
+ leftover: string[];
+ }>;
+ };
 }
 
 export interface AppHistoryInfo {