From e4967d37423d9b2a5de27dbde2a401822a58343a Mon Sep 17 00:00:00 2001
From: james
Date: Sun, 18 Feb 2024 17:14:38 +0800
Subject: [PATCH 1/5] WIP: Switching to companies
---
backend/api_app/controllers/networks.py | 6 ++--
backend/api_app/controllers/trackers.py | 19 ++---------
backend/api_app/models.py | 31 +++--------------
backend/app.py | 6 ++--
backend/dbcon/queries.py | 35 ++++++++++++--------
frontend/src/routes/networks/+page.server.ts | 2 +-
frontend/src/routes/networks/+page.svelte | 12 +++----
frontend/src/routes/trackers/+page.server.ts | 2 +-
frontend/src/routes/trackers/+page.svelte | 12 +++----
frontend/src/types.ts | 12 +++----
10 files changed, 53 insertions(+), 84 deletions(-)
diff --git a/backend/api_app/controllers/networks.py b/backend/api_app/controllers/networks.py
index e822f6a..be0c649 100644
--- a/backend/api_app/controllers/networks.py
+++ b/backend/api_app/controllers/networks.py
@@ -10,15 +10,15 @@
from api_app.models import NetworkApps, TopNetworks
from config import get_logger
-from dbcon.queries import get_apps_for_network, get_top_networks
+from dbcon.queries import get_apps_for_network, get_top_companies
logger = get_logger(__name__)
def networks_overview() -> TopNetworks:
"""Process networks and return TopNetworks class."""
- df = get_top_networks()
- df = df[~df["network_name"].isna()]
+ df = get_top_companies(categories=[1])
+ df = df[~df["name"].isna()]
df = df.sort_values("app_count", ascending=False)
networks = TopNetworks(networks=df.to_dict(orient="records"))
return networks
diff --git a/backend/api_app/controllers/trackers.py b/backend/api_app/controllers/trackers.py
index 385bdd6..81c008d 100644
--- a/backend/api_app/controllers/trackers.py
+++ b/backend/api_app/controllers/trackers.py
@@ -10,14 +10,14 @@
from api_app.models import TopTrackers, TrackerApps
from config import get_logger
-from dbcon.queries import get_apps_for_tracker, get_top_trackers
+from dbcon.queries import get_apps_for_tracker, get_top_companies
logger = get_logger(__name__)
def trackers_overview() -> TopTrackers:
"""Process trackers and return TopTrackers class."""
- df = get_top_trackers()
+ df = get_top_companies(categories=[2, 3])
df = df[~df["tracker_name"].isna()]
df = df.sort_values("app_count", ascending=False)
trackers = TopTrackers(trackers=df.to_dict(orient="records"))
@@ -30,21 +30,6 @@ class TrackersController(Controller):
path = "/api/trackers/"
- @get(path="/", cache=True)
- async def top_trackers(self: Self) -> TopTrackers:
- """Handle GET request for a list of top trackers.
-
- Returns
- -------
- A dictionary representation of the list of trackers
- each with an id, name, type and total of apps.
-
- """
- logger.info(f"{self.path} start")
- overview = trackers_overview()
-
- return overview
-
@get(path="/{tracker_name:str}", cache=3600)
async def get_tracker_apps(self: Self, tracker_name: str) -> TrackerApps:
"""Handle GET request for a specific tracker.
diff --git a/backend/api_app/models.py b/backend/api_app/models.py
index bbb2ff3..34c026d 100644
--- a/backend/api_app/models.py
+++ b/backend/api_app/models.py
@@ -121,45 +121,24 @@ class CategoriesOverview:
@dataclass
-class TrackerDetail:
+class CompanyDetail:
"""Describes details of a tracker.
Includes its db identifier, name, and the count of its occurrences.
"""
- tracker: int
- tracker_name: str
- count: int
-
-
-@dataclass
-class NetworkDetail:
-
- """Describes details of a network.
-
- Includes its db identifier, name, and the count of its occurrences.
- """
-
- network: int
- network_name: str
+ company_id: int
+ name: str
count: int
@dataclass
-class TopTrackers:
-
- """Contains a list of TrackerDetail objects representing the top trackers identified."""
-
- trackers: list[TrackerDetail]
-
-
-@dataclass
-class TopNetworks:
+class TopCompanies:
"""Contains a list of NetworkDetail objects representing the top networks identified."""
- networks: list[NetworkDetail]
+ companies: list[CompanyDetail]
@dataclass
diff --git a/backend/app.py b/backend/app.py
index b03c5a7..efaaf7f 100644
--- a/backend/app.py
+++ b/backend/app.py
@@ -8,9 +8,8 @@
from api_app.controllers.apps import AppController
from api_app.controllers.categories import CategoryController
-from api_app.controllers.networks import NetworksController
+from api_app.controllers.companies import CompaniesController
from api_app.controllers.rankings import RankingsController
-from api_app.controllers.trackers import TrackersController
cors_config = CORSConfig(
allow_origins=[
@@ -39,8 +38,7 @@ class MyOpenAPIController(OpenAPIController):
AppController,
CategoryController,
RankingsController,
- TrackersController,
- NetworksController,
+ CompaniesController,
],
cors_config=cors_config,
openapi_config=OpenAPIConfig(
diff --git a/backend/dbcon/queries.py b/backend/dbcon/queries.py
index 6527861..c92bede 100644
--- a/backend/dbcon/queries.py
+++ b/backend/dbcon/queries.py
@@ -38,9 +38,10 @@ def load_sql_file(file_name: str) -> str:
QUERY_STORE_COLLECTION_CATEGORY_MAP = load_sql_file(
"query_store_collection_category_map.sql",
)
-QUERY_TOP_TRACKERS = load_sql_file("query_top_trackers.sql")
QUERY_TRACKER_APPS = load_sql_file("query_tracker_apps.sql")
-QUERY_TOP_NETWORKS = load_sql_file("query_top_networks.sql")
+QUERY_TOP_COMPANIES = load_sql_file("query_top_companies.sql")
+QUERY_TOP_PARENT_COMPANIES = load_sql_file("query_top_parent_companies.sql")
+QUERY_NETWORK_APPS = load_sql_file("query_network_apps.sql")
QUERY_NETWORK_APPS = load_sql_file("query_network_apps.sql")
@@ -312,23 +313,29 @@ def get_manifest_names() -> pd.DataFrame:
return df
-def get_top_trackers() -> pd.DataFrame:
- """Get top trackers.
+def get_top_companies(
+ categories: list[int], group_by_parent: bool = False,
+) -> pd.DataFrame:
+ """Get top networks, mmps or other companies.
Data is pre-processed by materialized views.
- """
- df = pd.read_sql(QUERY_TOP_TRACKERS, DBCON.engine)
- return df
-
-
-def get_top_networks() -> pd.DataFrame:
- """Get top networks.
-
- Data is pre-processed by materialized views.
+ Args:
+ ----
+ categories (list[int]): list of 1=adnetworks, 2=MMP/Attribution, 3=Analytics.
+ group_by_parent (bool): Group by a parent entity if such entity exists.
"""
- df = pd.read_sql(QUERY_TOP_NETWORKS, DBCON.engine)
+ if group_by_parent:
+ df = pd.read_sql(
+ QUERY_TOP_COMPANIES, DBCON.engine, params={"categories": tuple(categories)},
+ )
+ else:
+ df = pd.read_sql(
+ QUERY_TOP_PARENT_COMPANIES,
+ DBCON.engine,
+ params={"categories": tuple(categories)},
+ )
return df
diff --git a/frontend/src/routes/networks/+page.server.ts b/frontend/src/routes/networks/+page.server.ts
index 54850aa..3733fb8 100644
--- a/frontend/src/routes/networks/+page.server.ts
+++ b/frontend/src/routes/networks/+page.server.ts
@@ -14,7 +14,7 @@ export const load: PageServerLoad = async ({ setHeaders }) => {
const res = fetch(`http://localhost:8000/api/networks`);
return {
- networks: {
+ companies: {
streamed: res.then((resp) => resp.json())
}
};
diff --git a/frontend/src/routes/networks/+page.svelte b/frontend/src/routes/networks/+page.svelte
index 4f19e05..d22cfa7 100644
--- a/frontend/src/routes/networks/+page.svelte
+++ b/frontend/src/routes/networks/+page.svelte
@@ -2,8 +2,8 @@
import type { TopNetworksInfo } from '../../types';
export let data: TopNetworksInfo;
- function navigate(tracker_name: string) {
- window.location.href = `/networks/${tracker_name}`;
+ function navigate(name: string) {
+ window.location.href = `/networks/${name}`;
}
@@ -25,7 +25,7 @@
networks. Currently this list is for Android APKs only.
- {#await data.networks.streamed}
+ {#await data.companies.streamed}
Loading Ad Networks...
{:then networks}
@@ -39,12 +39,12 @@
- {#each Object.entries(networks.networks) as [_prop, values]}
- navigate(values.network_name)} style="cursor: pointer;">
+ {#each Object.entries(networks.companies) as [_prop, values]}
+
navigate(values.name)} style="cursor: pointer;">
- {values.network_name}
+ {values.name}
|
diff --git a/frontend/src/routes/trackers/+page.server.ts b/frontend/src/routes/trackers/+page.server.ts
index d488fb4..f96808d 100644
--- a/frontend/src/routes/trackers/+page.server.ts
+++ b/frontend/src/routes/trackers/+page.server.ts
@@ -12,7 +12,7 @@ export const load: PageServerLoad = async ({ setHeaders }) => {
const res = fetch(`http://localhost:8000/api/trackers`);
return {
- trackers: {
+ companies: {
streamed: res.then((resp) => resp.json())
}
};
diff --git a/frontend/src/routes/trackers/+page.svelte b/frontend/src/routes/trackers/+page.svelte
index a8e0bbd..afb71d3 100644
--- a/frontend/src/routes/trackers/+page.svelte
+++ b/frontend/src/routes/trackers/+page.svelte
@@ -2,8 +2,8 @@
import type { TopTrackersInfo } from '../../types';
export let data: TopTrackersInfo;
- function navigate(tracker_name: string) {
- window.location.href = `/trackers/${tracker_name}`;
+ function navigate(name: string) {
+ window.location.href = `/trackers/${name}`;
}
@@ -25,7 +25,7 @@
Android APKs only.
- {#await data.trackers.streamed}
+ {#await data.companies.streamed}
Loading App Trackers...
{:then trackers}
@@ -39,12 +39,12 @@
- {#each Object.entries(trackers.trackers) as [_prop, values]}
- navigate(values.tracker_name)} style="cursor: pointer;">
+ {#each Object.entries(trackers.companies) as [_prop, values]}
+
navigate(values.name)} style="cursor: pointer;">
- {values.tracker_name}
+ {values.name}
|
diff --git a/frontend/src/types.ts b/frontend/src/types.ts
index 79a415b..2008467 100644
--- a/frontend/src/types.ts
+++ b/frontend/src/types.ts
@@ -149,20 +149,20 @@ export interface AppFullDetails {
}
export interface TrackerDetail {
- tracker_name: string;
+ name: string;
app_count: number;
percent: number;
}
export interface TopTrackersInfo {
status?: number;
error?: string;
- trackers: {
- streamed: Promise<{ trackers: TrackerDetail[] }>;
+ companies: {
+ streamed: Promise<{ companies: TrackerDetail[] }>;
};
}
export interface NetworkDetail {
- network_name: string;
+ name: string;
app_count: number;
percent: number;
}
@@ -170,8 +170,8 @@ export interface NetworkDetail {
export interface TopNetworksInfo {
status?: number;
error?: string;
- networks: {
- streamed: Promise<{ networks: NetworkDetail[] }>;
+ companies: {
+ streamed: Promise<{ companies: NetworkDetail[] }>;
};
}
export interface AppHistoryInfo {
From 3221d59db86e737715d90c8001f67436d1df2cf2 Mon Sep 17 00:00:00 2001
From: james
Date: Sun, 18 Feb 2024 19:59:30 +0800
Subject: [PATCH 2/5] Adde companies
---
backend/api_app/controllers/companies.py | 92 ++++++++++++++++++++++++
backend/api_app/models.py | 13 +---
2 files changed, 94 insertions(+), 11 deletions(-)
create mode 100644 backend/api_app/controllers/companies.py
diff --git a/backend/api_app/controllers/companies.py b/backend/api_app/controllers/companies.py
new file mode 100644
index 0000000..b46b514
--- /dev/null
+++ b/backend/api_app/controllers/companies.py
@@ -0,0 +1,92 @@
+"""API endoipoints for companies.
+
+/networks/ returns list of top networks.
+/trackers/ returns list of top trackers.
+"""
+
+from typing import Self
+
+from litestar import Controller, get
+from litestar.exceptions import NotFoundException
+
+from api_app.models import CompanyApps, TopCompanies
+from config import get_logger
+from dbcon.queries import get_apps_for_network, get_top_companies
+
+logger = get_logger(__name__)
+
+
+def companies_overview(categories: list[int]) -> TopCompanies:
+ """Process networks and return TopCompanies class."""
+ df = get_top_companies(categories=categories)
+ df = df[~df["name"].isna()]
+ df = df.sort_values("app_count", ascending=False)
+ networks = TopCompanies(companies=df.to_dict(orient="records"))
+ return networks
+
+
+class CompaniesController(Controller):
+
+ """API EndPoint return for all ad tech companies."""
+
+ path = "/api/"
+
+ @get(path="/networks", cache=3600)
+ async def top_networks(self: Self) -> TopCompanies:
+ """Handle GET request for a list of top networks.
+
+ Returns
+ -------
+ A dictionary representation of the list of networks
+ each with an id, name, type and total of apps.
+
+ """
+ logger.info(f"{self.path} start")
+ overview = companies_overview([1])
+
+ return overview
+
+ @get(path="/trackers", cache=3600)
+ async def top_trackers(self: Self) -> TopCompanies:
+ """Handle GET request for a list of top trackers.
+
+ Returns
+ -------
+ A dictionary representation of the list of trackers
+ each with an id, name, type and total of apps.
+
+ """
+ logger.info(f"{self.path} start")
+ overview = companies_overview([2, 3])
+
+ return overview
+
+ @get(path="/companies/{copmany_name:str}", cache=3600)
+ async def get_company_apps(self: Self, company_name: str) -> CompanyApps:
+ """Handle GET request for a specific company.
+
+ Args:
+ ----
+ company_name: The name of the company to retrieve apps for.
+
+ Returns:
+ -------
+ json.
+
+ """
+ logger.info(f"{self.path} start")
+ apps_df = get_apps_for_network(company_name)
+
+ if apps_df.empty:
+ msg = f"Network Name not found: {company_name!r}"
+ raise NotFoundException(
+ msg,
+ status_code=404,
+ )
+ apps_dict = apps_df.to_dict(orient="records")
+
+ apps = CompanyApps(
+ title=company_name,
+ apps=apps_dict,
+ )
+ return apps
diff --git a/backend/api_app/models.py b/backend/api_app/models.py
index 34c026d..4b92c45 100644
--- a/backend/api_app/models.py
+++ b/backend/api_app/models.py
@@ -60,18 +60,9 @@ class DeveloperApps:
@dataclass
-class TrackerApps:
+class CompanyApps:
- """A tracker's list of apps."""
-
- title: str
- apps: list[AppDetail]
-
-
-@dataclass
-class NetworkApps:
-
- """A network's list of apps."""
+ """A company's list of apps."""
title: str
apps: list[AppDetail]
From 0980e3adc4405a166e8a6cf6b5a4f71b9973ed83 Mon Sep 17 00:00:00 2001
From: james
Date: Sun, 18 Feb 2024 20:07:51 +0800
Subject: [PATCH 3/5] WIP: Switching to companies
---
backend/api_app/controllers/categories.py | 14 +++-
backend/api_app/controllers/networks.py | 76 -------------------
backend/api_app/controllers/trackers.py | 61 ---------------
backend/dbcon/sql/query_top_companies.sql | 39 ++++++++++
.../dbcon/sql/query_top_parent_companies.sql | 46 +++++++++++
frontend/src/routes/+layout.server.ts | 4 +-
6 files changed, 97 insertions(+), 143 deletions(-)
delete mode 100644 backend/api_app/controllers/networks.py
delete mode 100644 backend/api_app/controllers/trackers.py
create mode 100644 backend/dbcon/sql/query_top_companies.sql
create mode 100644 backend/dbcon/sql/query_top_parent_companies.sql
diff --git a/backend/api_app/controllers/categories.py b/backend/api_app/controllers/categories.py
index 807dac3..a086adc 100644
--- a/backend/api_app/controllers/categories.py
+++ b/backend/api_app/controllers/categories.py
@@ -4,6 +4,8 @@
"""
+from typing import Self
+
import numpy as np
from litestar import Controller, get
@@ -15,8 +17,9 @@
def category_overview() -> CategoriesOverview:
+ """Categories for apps."""
cats = get_appstore_categories()
- cats = cats[cats["total_apps"] > 100]
+ cats = cats[cats["total_apps"] > 100] # noqa: PLR2004 magic number ok
cats["name"] = cats["category"]
cats["name"] = (
@@ -49,10 +52,13 @@ def category_overview() -> CategoriesOverview:
class CategoryController(Controller):
+
+ """App Store Categories API."""
+
path = "/api/categories"
@get(path="/", cache=True)
- async def get_categories_overview(self) -> CategoriesOverview:
+ async def get_categories_overview(self: Self) -> CategoriesOverview:
"""Handle GET request for a list of categories.
Returns
@@ -66,8 +72,8 @@ async def get_categories_overview(self) -> CategoriesOverview:
return overview
- @get(path="/{category_id:str}", cache=3600)
- async def get_category(self, category_id: str) -> Category:
+ @get(path="/{category_id:str}", cache=True)
+ async def get_category(self: Self, category_id: str) -> Category:
"""Handle GET request for a single category.
Returns
diff --git a/backend/api_app/controllers/networks.py b/backend/api_app/controllers/networks.py
deleted file mode 100644
index be0c649..0000000
--- a/backend/api_app/controllers/networks.py
+++ /dev/null
@@ -1,76 +0,0 @@
-"""API endoipoints for app networks.
-
-/networks/ returns list of top networks.
-"""
-
-from typing import Self
-
-from litestar import Controller, get
-from litestar.exceptions import NotFoundException
-
-from api_app.models import NetworkApps, TopNetworks
-from config import get_logger
-from dbcon.queries import get_apps_for_network, get_top_companies
-
-logger = get_logger(__name__)
-
-
-def networks_overview() -> TopNetworks:
- """Process networks and return TopNetworks class."""
- df = get_top_companies(categories=[1])
- df = df[~df["name"].isna()]
- df = df.sort_values("app_count", ascending=False)
- networks = TopNetworks(networks=df.to_dict(orient="records"))
- return networks
-
-
-class NetworksController(Controller):
-
- """API EndPoint return for app networks."""
-
- path = "/api/networks/"
-
- @get(path="/", cache=True)
- async def top_networks(self: Self) -> TopNetworks:
- """Handle GET request for a list of top networks.
-
- Returns
- -------
- A dictionary representation of the list of networks
- each with an id, name, type and total of apps.
-
- """
- logger.info(f"{self.path} start")
- overview = networks_overview()
-
- return overview
-
- @get(path="/{network_name:str}", cache=3600)
- async def get_network_apps(self: Self, network_name: str) -> NetworkApps:
- """Handle GET request for a specific network.
-
- Args:
- ----
- network_name (str): The name of the network to retrieve apps for.
-
- Returns:
- -------
- json
-
- """
- logger.info(f"{self.path} start")
- apps_df = get_apps_for_network(network_name)
-
- if apps_df.empty:
- msg = f"Network Name not found: {network_name!r}"
- raise NotFoundException(
- msg,
- status_code=404,
- )
- apps_dict = apps_df.to_dict(orient="records")
-
- apps = NetworkApps(
- title=network_name,
- apps=apps_dict,
- )
- return apps
diff --git a/backend/api_app/controllers/trackers.py b/backend/api_app/controllers/trackers.py
deleted file mode 100644
index 81c008d..0000000
--- a/backend/api_app/controllers/trackers.py
+++ /dev/null
@@ -1,61 +0,0 @@
-"""API endoipoints for app trackers.
-
-/trackers/ returns list of top trackers.
-"""
-
-from typing import Self
-
-from litestar import Controller, get
-from litestar.exceptions import NotFoundException
-
-from api_app.models import TopTrackers, TrackerApps
-from config import get_logger
-from dbcon.queries import get_apps_for_tracker, get_top_companies
-
-logger = get_logger(__name__)
-
-
-def trackers_overview() -> TopTrackers:
- """Process trackers and return TopTrackers class."""
- df = get_top_companies(categories=[2, 3])
- df = df[~df["tracker_name"].isna()]
- df = df.sort_values("app_count", ascending=False)
- trackers = TopTrackers(trackers=df.to_dict(orient="records"))
- return trackers
-
-
-class TrackersController(Controller):
-
- """API EndPoint return for app trackers."""
-
- path = "/api/trackers/"
-
- @get(path="/{tracker_name:str}", cache=3600)
- async def get_tracker_apps(self: Self, tracker_name: str) -> TrackerApps:
- """Handle GET request for a specific tracker.
-
- Args:
- ----
- tracker_name (str): The name of the tracker to retrieve apps for.
-
- Returns:
- -------
- json
-
- """
- logger.info(f"{self.path} start")
- apps_df = get_apps_for_tracker(tracker_name)
-
- if apps_df.empty:
- msg = f"Store ID not found: {tracker_name!r}"
- raise NotFoundException(
- msg,
- status_code=404,
- )
- apps_dict = apps_df.to_dict(orient="records")
-
- apps = TrackerApps(
- title=tracker_name,
- apps=apps_dict,
- )
- return apps
diff --git a/backend/dbcon/sql/query_top_companies.sql b/backend/dbcon/sql/query_top_companies.sql
new file mode 100644
index 0000000..4036974
--- /dev/null
+++ b/backend/dbcon/sql/query_top_companies.sql
@@ -0,0 +1,39 @@
+WITH
+counts AS (
+ SELECT
+ sac.company_id,
+ COUNT(DISTINCT sac.store_app) AS app_count
+ FROM
+ adtech.store_apps_companies sac
+ LEFT JOIN adtech.company_categories cc
+ON
+ sac.company_id = cc.company_id
+ LEFT JOIN adtech.categories cat ON
+ cc.category_id = cat.id
+ WHERE
+ cat.id IN :categories
+ GROUP BY
+ sac.company_id
+),
+total_app_count AS (
+ SELECT
+ COUNT(DISTINCT store_app)
+ FROM
+ adtech.store_apps_companies
+)
+SELECT
+ c.name AS name,
+ tc.app_count,
+ total_app_count.count AS total_app_count,
+ (
+ tc.app_count / total_app_count.count::decimal
+ ) AS PERCENT
+ FROM
+ counts AS tc
+ LEFT JOIN adtech.companies AS c
+ ON
+ tc.company_id = c.id
+ INNER JOIN total_app_count
+ ON TRUE
+ ORDER BY
+ tc.app_count DESC;
diff --git a/backend/dbcon/sql/query_top_parent_companies.sql b/backend/dbcon/sql/query_top_parent_companies.sql
new file mode 100644
index 0000000..42602a8
--- /dev/null
+++ b/backend/dbcon/sql/query_top_parent_companies.sql
@@ -0,0 +1,46 @@
+WITH
+parent_counts AS (
+ SELECT
+ COALESCE(
+ ccom.parent_company_id,
+ sac.company_id
+ ) AS parent_or_self_id,
+ COUNT(DISTINCT sac.store_app) AS app_count
+ FROM
+ adtech.store_apps_companies sac
+ LEFT JOIN adtech.company_categories cc
+ON
+ sac.company_id = cc.company_id
+ LEFT JOIN adtech.categories cat ON
+ cc.category_id = cat.id
+ LEFT JOIN adtech.companies ccom
+ ON
+ sac.company_id = ccom.id
+ WHERE
+ cat.id IN :categories
+ GROUP BY
+ parent_or_self_id
+),
+total_app_count AS (
+ SELECT
+ COUNT(DISTINCT store_app)
+ FROM
+ adtech.store_apps_companies
+)
+SELECT
+ c.name AS name,
+ tc.app_count,
+ total_app_count.count AS total_app_count,
+ (
+ tc.app_count / total_app_count.count::decimal
+ ) AS PERCENT
+ FROM
+ parent_counts AS tc
+ LEFT JOIN adtech.companies AS c
+ ON
+ tc.parent_or_self_id = c.id
+ INNER JOIN total_app_count
+ ON
+ TRUE
+ ORDER BY
+ tc.app_count DESC;
diff --git a/frontend/src/routes/+layout.server.ts b/frontend/src/routes/+layout.server.ts
index 141579a..2483ed0 100644
--- a/frontend/src/routes/+layout.server.ts
+++ b/frontend/src/routes/+layout.server.ts
@@ -4,7 +4,7 @@ export const ssr = true;
export const csr = true;
export const load: PageServerLoad = async () => {
- console.log(`load categories start`);
+ // console.log(`load categories start`);
const res = fetch(`http://localhost:8000/api/categories`);
// setHeaders({
@@ -21,7 +21,7 @@ export const load: PageServerLoad = async () => {
console.log('App Not found');
return 'App Not Found';
} else if (resp.status === 500) {
- console.log('App API Server error');
+ console.log('Categories API Server error');
return 'Backend Error';
}
})
From bc75550413e14df50642d03b95d6073f05433dbb Mon Sep 17 00:00:00 2001
From: james
Date: Sun, 18 Feb 2024 20:08:38 +0800
Subject: [PATCH 4/5] Delete unused sql files
---
backend/dbcon/sql/query_top_networks.sql | 31 ------------------------
backend/dbcon/sql/query_top_trackers.sql | 31 ------------------------
2 files changed, 62 deletions(-)
delete mode 100644 backend/dbcon/sql/query_top_networks.sql
delete mode 100644 backend/dbcon/sql/query_top_trackers.sql
diff --git a/backend/dbcon/sql/query_top_networks.sql b/backend/dbcon/sql/query_top_networks.sql
deleted file mode 100644
index 5b4fbca..0000000
--- a/backend/dbcon/sql/query_top_networks.sql
+++ /dev/null
@@ -1,31 +0,0 @@
-WITH
-network_counts AS (
- SELECT
- network,
- count(DISTINCT store_app) AS app_count
- FROM
- store_apps_networks
- GROUP BY
- network
-),
-
-total_app_count AS (
- SELECT count(DISTINCT store_app)
- FROM
- store_apps_networks
-)
-
-SELECT
- t.name AS network_name,
- tc.app_count,
- total_app_count.count AS total_app_count,
- (
- tc.app_count / total_app_count.count::decimal
- ) AS percent
-FROM
- network_counts AS tc
-LEFT JOIN networks AS t
- ON
- tc.network = t.id
-INNER JOIN total_app_count ON
- TRUE;
diff --git a/backend/dbcon/sql/query_top_trackers.sql b/backend/dbcon/sql/query_top_trackers.sql
deleted file mode 100644
index bc3d9c0..0000000
--- a/backend/dbcon/sql/query_top_trackers.sql
+++ /dev/null
@@ -1,31 +0,0 @@
-WITH
-tracker_counts AS (
- SELECT
- tracker,
- count(DISTINCT store_app) AS app_count
- FROM
- store_apps_trackers
- GROUP BY
- tracker
-),
-
-total_app_count AS (
- SELECT count(DISTINCT store_app)
- FROM
- store_apps_trackers
-)
-
-SELECT
- t.name AS tracker_name,
- tc.app_count,
- total_app_count.count AS total_app_count,
- (
- tc.app_count / total_app_count.count::decimal
- ) AS percent
-FROM
- tracker_counts AS tc
-LEFT JOIN trackers AS t
- ON
- tc.tracker = t.id
-INNER JOIN total_app_count ON
- TRUE;
From ac694ac9b54357b0043e5a5b9b5c1dd57071d3df Mon Sep 17 00:00:00 2001
From: james
Date: Sun, 18 Feb 2024 20:39:58 +0800
Subject: [PATCH 5/5] Companies and trackers use new api
---
backend/api_app/controllers/companies.py | 6 +--
backend/dbcon/queries.py | 51 ++++++++++---------
backend/dbcon/sql/query_company_apps.sql | 17 +++++++
backend/dbcon/sql/query_network_apps.sql | 17 -------
.../dbcon/sql/query_parent_company_apps.sql | 20 ++++++++
backend/dbcon/sql/query_tracker_apps.sql | 17 -------
.../routes/networks/[name]/+page.server.ts | 2 +-
.../src/routes/networks/[name]/+page.svelte | 2 +-
.../routes/trackers/[name]/+page.server.ts | 2 +-
9 files changed, 69 insertions(+), 65 deletions(-)
create mode 100644 backend/dbcon/sql/query_company_apps.sql
delete mode 100644 backend/dbcon/sql/query_network_apps.sql
create mode 100644 backend/dbcon/sql/query_parent_company_apps.sql
delete mode 100644 backend/dbcon/sql/query_tracker_apps.sql
diff --git a/backend/api_app/controllers/companies.py b/backend/api_app/controllers/companies.py
index b46b514..1e92935 100644
--- a/backend/api_app/controllers/companies.py
+++ b/backend/api_app/controllers/companies.py
@@ -11,7 +11,7 @@
from api_app.models import CompanyApps, TopCompanies
from config import get_logger
-from dbcon.queries import get_apps_for_network, get_top_companies
+from dbcon.queries import get_apps_for_company, get_top_companies
logger = get_logger(__name__)
@@ -61,7 +61,7 @@ async def top_trackers(self: Self) -> TopCompanies:
return overview
- @get(path="/companies/{copmany_name:str}", cache=3600)
+ @get(path="/companies/{company_name:str}", cache=3600)
async def get_company_apps(self: Self, company_name: str) -> CompanyApps:
"""Handle GET request for a specific company.
@@ -75,7 +75,7 @@ async def get_company_apps(self: Self, company_name: str) -> CompanyApps:
"""
logger.info(f"{self.path} start")
- apps_df = get_apps_for_network(company_name)
+ apps_df = get_apps_for_company(company_name, include_parents=True)
if apps_df.empty:
msg = f"Network Name not found: {company_name!r}"
diff --git a/backend/dbcon/queries.py b/backend/dbcon/queries.py
index c92bede..3241f44 100644
--- a/backend/dbcon/queries.py
+++ b/backend/dbcon/queries.py
@@ -38,11 +38,10 @@ def load_sql_file(file_name: str) -> str:
QUERY_STORE_COLLECTION_CATEGORY_MAP = load_sql_file(
"query_store_collection_category_map.sql",
)
-QUERY_TRACKER_APPS = load_sql_file("query_tracker_apps.sql")
QUERY_TOP_COMPANIES = load_sql_file("query_top_companies.sql")
QUERY_TOP_PARENT_COMPANIES = load_sql_file("query_top_parent_companies.sql")
-QUERY_NETWORK_APPS = load_sql_file("query_network_apps.sql")
-QUERY_NETWORK_APPS = load_sql_file("query_network_apps.sql")
+QUERY_COMPANY_APPS = load_sql_file("query_company_apps.sql")
+QUERY_PARENT_COMPANY_APPS = load_sql_file("query_parent_company_apps.sql")
def get_recent_apps(collection: str, limit: int = 20) -> pd.DataFrame:
@@ -265,27 +264,25 @@ def get_single_developer(developer_id: str) -> pd.DataFrame:
return df
-def get_apps_for_tracker(tracker_name: str) -> pd.DataFrame:
- """Get apps for for a tracker."""
- logger.info(f"Tracker: {tracker_name=}")
- df = pd.read_sql(
- QUERY_TRACKER_APPS,
- con=DBCON.engine,
- params={"tracker_name": tracker_name, "mylimit": 20},
- )
- if not df.empty:
- df = clean_app_df(df)
- return df
-
-
-def get_apps_for_network(network_name: str) -> pd.DataFrame:
+def get_apps_for_company(
+ network_name: str,
+ *,
+ include_parents: bool = False,
+) -> pd.DataFrame:
"""Get apps for for a network."""
- logger.info(f"Tracker: {network_name=}")
- df = pd.read_sql(
- QUERY_NETWORK_APPS,
- con=DBCON.engine,
- params={"network_name": network_name, "mylimit": 20},
- )
+ logger.info(f"Tracker: {network_name=} & {include_parents=}")
+ if include_parents:
+ df = pd.read_sql(
+ QUERY_PARENT_COMPANY_APPS,
+ con=DBCON.engine,
+ params={"network_name": network_name, "mylimit": 20},
+ )
+ else:
+ df = pd.read_sql(
+ QUERY_COMPANY_APPS,
+ con=DBCON.engine,
+ params={"network_name": network_name, "mylimit": 20},
+ )
if not df.empty:
df = clean_app_df(df)
return df
@@ -314,7 +311,9 @@ def get_manifest_names() -> pd.DataFrame:
def get_top_companies(
- categories: list[int], group_by_parent: bool = False,
+ categories: list[int],
+ *,
+ group_by_parent: bool = False,
) -> pd.DataFrame:
"""Get top networks, mmps or other companies.
@@ -328,7 +327,9 @@ def get_top_companies(
"""
if group_by_parent:
df = pd.read_sql(
- QUERY_TOP_COMPANIES, DBCON.engine, params={"categories": tuple(categories)},
+ QUERY_TOP_COMPANIES,
+ DBCON.engine,
+ params={"categories": tuple(categories)},
)
else:
df = pd.read_sql(
diff --git a/backend/dbcon/sql/query_company_apps.sql b/backend/dbcon/sql/query_company_apps.sql
new file mode 100644
index 0000000..75a8d51
--- /dev/null
+++ b/backend/dbcon/sql/query_company_apps.sql
@@ -0,0 +1,17 @@
+SELECT *
+FROM
+ public.store_apps
+WHERE
+ id IN
+ (
+ SELECT sac.store_app
+ FROM
+ adtech.store_apps_companies AS sac
+ LEFT JOIN adtech.companies AS c
+ ON
+ sac.company_id = c.id
+ WHERE
+ c.name = :network_name
+ )
+ORDER BY installs DESC
+LIMIT :mylimit;
diff --git a/backend/dbcon/sql/query_network_apps.sql b/backend/dbcon/sql/query_network_apps.sql
deleted file mode 100644
index c31116f..0000000
--- a/backend/dbcon/sql/query_network_apps.sql
+++ /dev/null
@@ -1,17 +0,0 @@
-SELECT *
-FROM
- store_apps
-WHERE
- id IN
- (
- SELECT sat.store_app
- FROM
- store_apps_networks AS sat
- LEFT JOIN networks AS t
- ON
- sat.network = t.id
- WHERE
- t.name = :network_name
- )
-ORDER BY installs DESC
-LIMIT :mylimit;
diff --git a/backend/dbcon/sql/query_parent_company_apps.sql b/backend/dbcon/sql/query_parent_company_apps.sql
new file mode 100644
index 0000000..8de9b49
--- /dev/null
+++ b/backend/dbcon/sql/query_parent_company_apps.sql
@@ -0,0 +1,20 @@
+SELECT *
+FROM
+ public.store_apps
+WHERE
+ id IN
+ (
+ SELECT sac.store_app
+ FROM
+ adtech.store_apps_companies AS sac
+ LEFT JOIN adtech.companies AS c
+ ON
+ sac.company_id = c.id
+ LEFT JOIN adtech.companies AS pc
+ ON
+ sac.company_id = pc.parent_company_id
+ WHERE
+ c.name = :network_name OR pc.name = :network_name
+ )
+ORDER BY installs DESC
+LIMIT :mylimit;
diff --git a/backend/dbcon/sql/query_tracker_apps.sql b/backend/dbcon/sql/query_tracker_apps.sql
deleted file mode 100644
index 8cb4727..0000000
--- a/backend/dbcon/sql/query_tracker_apps.sql
+++ /dev/null
@@ -1,17 +0,0 @@
-SELECT *
-FROM
- store_apps
-WHERE
- id IN
- (
- SELECT sat.store_app
- FROM
- store_apps_trackers AS sat
- LEFT JOIN trackers AS t
- ON
- sat.tracker = t.id
- WHERE
- t.name = :tracker_name
- )
-ORDER BY installs DESC
-LIMIT :mylimit;
diff --git a/frontend/src/routes/networks/[name]/+page.server.ts b/frontend/src/routes/networks/[name]/+page.server.ts
index cb2c508..9d775db 100644
--- a/frontend/src/routes/networks/[name]/+page.server.ts
+++ b/frontend/src/routes/networks/[name]/+page.server.ts
@@ -7,7 +7,7 @@ export const csr: boolean = true;
console.log('Script executed');
export const load: PageServerLoad = async ({ params, locals }) => {
const networkName = params.name;
- const res = fetch(`http://localhost:8000/api/networks/${networkName}`);
+ const res = fetch(`http://localhost:8000/api/companies/${networkName}`);
console.log(`start load apps for tracker=${networkName}`);
try {
return {
diff --git a/frontend/src/routes/networks/[name]/+page.svelte b/frontend/src/routes/networks/[name]/+page.svelte
index 4d878f2..db594ff 100644
--- a/frontend/src/routes/networks/[name]/+page.svelte
+++ b/frontend/src/routes/networks/[name]/+page.svelte
@@ -11,7 +11,7 @@
{:then apps}
{#if typeof apps == 'string'}
- Failed to load tracker
+ Failed to load network's apps.
{:else}
Apps using {apps.title}
diff --git a/frontend/src/routes/trackers/[name]/+page.server.ts b/frontend/src/routes/trackers/[name]/+page.server.ts
index ac6c14a..498cb56 100644
--- a/frontend/src/routes/trackers/[name]/+page.server.ts
+++ b/frontend/src/routes/trackers/[name]/+page.server.ts
@@ -6,7 +6,7 @@ export const csr: boolean = true;
console.log('Script executed');
export const load: PageServerLoad = async ({ params, locals }) => {
const trackerName = params.name;
- const res = fetch(`http://localhost:8000/api/trackers/${trackerName}`);
+ const res = fetch(`http://localhost:8000/api/companies/${trackerName}`);
console.log(`start load apps for tracker=${trackerName}`);
try {
return {