diff --git a/bloom/.gitignore b/.gitignore similarity index 100% rename from bloom/.gitignore rename to .gitignore diff --git a/bloom/.pre-commit-config.yaml b/.pre-commit-config.yaml similarity index 100% rename from bloom/.pre-commit-config.yaml rename to .pre-commit-config.yaml diff --git a/bloom/Trawlwatcher.py b/bloom/Trawlwatcher.py index e3b251d5..ca26e8bf 100644 --- a/bloom/Trawlwatcher.py +++ b/bloom/Trawlwatcher.py @@ -10,10 +10,14 @@ layout="wide", ) +class EnvFileNotFoundError(FileNotFoundError): + def __init__(self, env_path:str) -> None: + super().__init__(f"Couldn't find .env file at {env_path}") + # FILL IN YOUR CREDENTIALS .env file HERE !! -env_path = Path('.') / '.env.template' +env_path = Path('.') / '.env.template2' if not env_path.is_file(): - raise FileNotFoundError(f"Couldn't find .env file at {env_path.absolute()}") + raise EnvFileNotFoundError(env_path.absolute()) load_dotenv(env_path) def local_css(file_name: str) -> None: diff --git a/bloom/alembic/init_script/load_amp_data.py b/bloom/alembic/init_script/load_amp_data.py index 86326de1..af65396d 100644 --- a/bloom/alembic/init_script/load_amp_data.py +++ b/bloom/alembic/init_script/load_amp_data.py @@ -34,7 +34,7 @@ engine = create_engine(db_url, echo=False) df = pd.read_csv( - Path(os.path.dirname(__file__)).joinpath("../../data/zones_subset_02022024.csv"), + Path(__file__).parent.joinpath("../../data/zones_subset_02022024.csv"), sep=",", ) diff --git a/bloom/alembic/init_script/load_positions_data.py b/bloom/alembic/init_script/load_positions_data.py index 6db0e4e3..9e06b42a 100644 --- a/bloom/alembic/init_script/load_positions_data.py +++ b/bloom/alembic/init_script/load_positions_data.py @@ -31,7 +31,7 @@ engine = create_engine(db_url) df = pd.read_csv( - Path(os.path.dirname(__file__)).joinpath("../../data/spire_positions_subset_02022024.csv"), + Path(__file__).parent.joinpath("../../data/spire_positions_subset_02022024.csv"), sep="," ) diff --git a/bloom/alembic/init_script/load_vessels_data.py b/bloom/alembic/init_script/load_vessels_data.py index 03190585..16e9ef4e 100644 --- a/bloom/alembic/init_script/load_vessels_data.py +++ b/bloom/alembic/init_script/load_vessels_data.py @@ -29,7 +29,7 @@ ) engine = create_engine(db_url) df = pd.read_csv( - Path(os.path.dirname(__file__)).joinpath("../../data/chalutiers_pelagiques.csv"), + Path(__file__).parent.joinpath("../../data/chalutiers_pelagiques.csv"), sep=";", dtype={"loa": float, "IMO": str}, ) diff --git a/bloom/alembic/versions/961cee5426d6_create_amp_table.py b/bloom/alembic/versions/961cee5426d6_create_amp_table.py index 0cfe991b..4ce684fc 100644 --- a/bloom/alembic/versions/961cee5426d6_create_amp_table.py +++ b/bloom/alembic/versions/961cee5426d6_create_amp_table.py @@ -7,6 +7,7 @@ """ from alembic import op import sqlalchemy as sa +from sqlalchemy import Inspector import geoalchemy2 # revision identifiers, used by Alembic. @@ -26,7 +27,8 @@ def upgrade() -> None: sa.Column("iucn_cat", sa.String), sa.Column("parent_iso", sa.String), sa.Column("iso3", sa.String), - sa.Column("geometry", geoalchemy2.types.Geometry(geometry_type="GEOMETRY", srid=4326)), + sa.Column("geometry", geoalchemy2.types.Geometry(geometry_type="GEOMETRY", + srid=4326)), sa.Column("benificiaries", sa.String) ) diff --git a/bloom/app.py b/bloom/app.py index 5ca43378..6ba497b5 100644 --- a/bloom/app.py +++ b/bloom/app.py @@ -34,17 +34,19 @@ def main() -> None: if args.mode == ExecutionMode.LOCAL: logger.info("Starting scraping with internal scheduler") - scheduler = PeriodicScheduler( - function=marine_traffic_usecase.scrap_vessels, - interval=SCRAP_INTERVAL, - ) + # Commented as marine_traffic_usecase was commented before + # error F821 Undefined name `marine_traffic_usecase` + #scheduler = PeriodicScheduler( + # function=marine_traffic_usecase.scrap_vessels, + # interval=SCRAP_INTERVAL, + #) spire_traffic_usecase.save_vessels( spire_traffic_usecase.get_all_vessels(timestamp), ) #marine_traffic_usecase.scrap_vessels(timestamp) alert_usecase.generate_alerts(timestamp) - while True: - scheduler.start() + #while True: + # scheduler.start() else: logger.info("Starting scraping with external scheduler") spire_traffic_usecase.save_vessels( diff --git a/bloom/bloom/config.py b/bloom/bloom/config.py index c0f0e1ea..0464462f 100644 --- a/bloom/bloom/config.py +++ b/bloom/bloom/config.py @@ -9,19 +9,20 @@ class Settings(BaseSettings): postgres_port = os.environ.get("POSTGRES_PORT") postgres_db = os.environ.get("POSTGRES_DB") - print("db_url: ", "postgresql://"+postgres_user+":"+postgres_password+"@"+postgres_hostname+":"+postgres_port+"/"+postgres_db) + #print("db_url: ", f"postgresql://{postgres_user}:{postgres_password}@{postgres_hostname}:" + # f"{postgres_port}/{postgres_db}") db_url = ( "postgresql://" - + postgres_user - + ":" - + postgres_password - + "@" - + postgres_hostname - + ":" - + postgres_port - + "/" - + postgres_db + f"{postgres_user}" + ":" + f"{postgres_password}" + "@" + f"{postgres_hostname}" + ":" + f"{postgres_port}" + "/" + f"{postgres_db}" ) srid: int = 4326 diff --git a/bloom/bloom/domain/vessels/vessel_trajectory.py b/bloom/bloom/domain/vessels/vessel_trajectory.py index 054bd8a8..18135285 100644 --- a/bloom/bloom/domain/vessels/vessel_trajectory.py +++ b/bloom/bloom/domain/vessels/vessel_trajectory.py @@ -84,6 +84,10 @@ def mpas(self) -> []: if hasattr(self, "_mpas"): return self._mpas return None + + @mpas.setter + def mpas(self, value:[])->None: + self._mpas = value def get_closest_marine_protected_areas(self, radius: int = 100) -> None: self._mpas, self._mpas_gdf = get_closest_marine_protected_areas( @@ -121,8 +125,7 @@ def query( assert len(filtered_data) > 0 filtered_vessel = VesselTrajectory(self.metadata, filtered_data.copy()) - if hasattr(self, "_mpas"): - filtered_vessel._mpas = self._mpas + filtered_vessel.mpas(self.mpas()) filtered_vessel.positions.index = filtered_data.index diff --git a/bloom/bloom/infra/repositories/repository_alert.py b/bloom/bloom/infra/repositories/repository_alert.py index 24aaac47..653fb4c6 100644 --- a/bloom/bloom/infra/repositories/repository_alert.py +++ b/bloom/bloom/infra/repositories/repository_alert.py @@ -23,13 +23,13 @@ def save_alerts(self, timestamp: datetime) -> None: ( SELECT timestamp, vessel_id, (CAST(ST_Contains(mpa_fr_with_mn.geometry,current_position) AS INT) - CAST(ST_Contains(mpa_fr_with_mn.geometry,previous_position) AS INT)) as cross_mpa, ARRAY_AGG(mpa_fr_with_mn.index ORDER BY mpa_fr_with_mn.index DESC) AS mpa_ids FROM (SELECT spire_vessel_positions.vessel_id AS vessel_id, - spire_vessel_positions.position AS current_position, - spire_vessel_positions.timestamp AS timestamp, + spire_vessel_positions.position AS current_position, + spire_vessel_positions.timestamp AS timestamp, LAG(spire_vessel_positions.position) OVER (PARTITION BY spire_vessel_positions.vessel_id ORDER BY spire_vessel_positions.timestamp) AS previous_position FROM spire_vessel_positions WHERE spire_vessel_positions.timestamp >= TIMESTAMP '{timestamp}' - INTERVAL '15 minutes' AND spire_vessel_positions.timestamp < TIMESTAMP '{timestamp}' + INTERVAL '15 minutes' ) AS foo CROSS JOIN mpa_fr_with_mn WHERE previous_position IS NOT NULL and ST_Contains(mpa_fr_with_mn.geometry,current_position) != ST_Contains(mpa_fr_with_mn.geometry,previous_position) GROUP BY vessel_id, timestamp,cross_mpa ); - """, # nosec: B608 + """, # nosec: B608 # noqa: E501 ) session.execute(sql) session.commit() @@ -37,22 +37,22 @@ def save_alerts(self, timestamp: datetime) -> None: # an other query with the same result : # WITH cte_query1 AS ( - # SELECT spire_vessel_positions.vessel_id AS vessel_id, ARRAY_AGG(mpa_fr_with_mn.index ORDER BY mpa_fr_with_mn.index DESC) AS mpa_ids + # SELECT spire_vessel_positions.vessel_id AS vessel_id, ARRAY_AGG(mpa_fr_with_mn.index ORDER BY mpa_fr_with_mn.index DESC) AS mpa_ids # noqa: E501 # FROM spire_vessel_positions - # JOIN mpa_fr_with_mn ON ST_Contains(mpa_fr_with_mn.geometry, spire_vessel_positions.position) - # WHERE spire_vessel_positions.timestamp = TO_TIMESTAMP('2023-11-17 12:00', 'YYYY-MM-DD HH24:MI') + # JOIN mpa_fr_with_mn ON ST_Contains(mpa_fr_with_mn.geometry, spire_vessel_positions.position) # noqa: E501 + # WHERE spire_vessel_positions.timestamp = TO_TIMESTAMP('2023-11-17 12:00', 'YYYY-MM-DD HH24:MI') # noqa: E501 # GROUP BY vessel_id # ), # cte_query2 AS ( - # SELECT DISTINCT spire_vessel_positions.vessel_id AS vessel_id, ARRAY_AGG(mpa_fr_with_mn.index ORDER BY mpa_fr_with_mn.index DESC) AS mpa_ids + # SELECT DISTINCT spire_vessel_positions.vessel_id AS vessel_id, ARRAY_AGG(mpa_fr_with_mn.index ORDER BY mpa_fr_with_mn.index DESC) AS mpa_ids # noqa: E501 # FROM spire_vessel_positions - # JOIN mpa_fr_with_mn ON ST_Contains(mpa_fr_with_mn.geometry, spire_vessel_positions.position) - # WHERE spire_vessel_positions.timestamp = TO_TIMESTAMP('2023-11-17 12:15', 'YYYY-MM-DD HH24:MI') + # JOIN mpa_fr_with_mn ON ST_Contains(mpa_fr_with_mn.geometry, spire_vessel_positions.position) # noqa: E501 + # WHERE spire_vessel_positions.timestamp = TO_TIMESTAMP('2023-11-17 12:15', 'YYYY-MM-DD HH24:MI') # noqa: E501 # GROUP BY vessel_id # ) - # SELECT vessel_id, mpa_ids, -1 AS value FROM cte_query1 EXCEPT SELECT vessel_id, mpa_ids, -1 AS value FROM cte_query2 + # SELECT vessel_id, mpa_ids, -1 AS value FROM cte_query1 EXCEPT SELECT vessel_id, mpa_ids, -1 AS value FROM cte_query2 # noqa: E501 # UNION ALL - # SELECT vessel_id, mpa_ids, 1 AS value FROM cte_query2 EXCEPT SELECT vessel_id, mpa_ids, 1 AS value FROM cte_query1 + # SELECT vessel_id, mpa_ids, 1 AS value FROM cte_query2 EXCEPT SELECT vessel_id, mpa_ids, 1 AS value FROM cte_query1 # noqa: E501 def load_alert(self, timestamp: datetime) -> list[Alert]: with self.session_factory() as session: diff --git a/bloom/requirements.txt b/bloom/requirements.txt deleted file mode 100644 index 79dc9d1b..00000000 --- a/bloom/requirements.txt +++ /dev/null @@ -1,24 +0,0 @@ -pandas==2.2.1 -selenium==4.18.1 -undetected-chromedriver==3.4.6 -openpyxl==3.1.0 -pydantic==1.10.14 -pyyaml==6.0 -shapely==2.0.1 -geopandas==0.14.3 -sqlalchemy==2.0 -alembic==1.10.2 -geoalchemy2==0.13.1 -psycopg2-binary==2.9.6 -dependency-injector==4.41.0 -dependency_injection==1.2.0 -requests==2.31 -gql==3.4.0 -requests-toolbelt==1.0 -slack-sdk==3.21.3 -folium==0.14.0 -matplotlib==3.8.0 -geopy==2.4.0 -streamlit==1.27.2 -streamlit-folium==0.15.0 -python-dotenv==1.0.0 \ No newline at end of file diff --git a/bloom/poetry.lock b/poetry.lock similarity index 100% rename from bloom/poetry.lock rename to poetry.lock diff --git a/bloom/pyproject.toml b/pyproject.toml similarity index 100% rename from bloom/pyproject.toml rename to pyproject.toml diff --git a/bloom/tox.ini b/tox.ini similarity index 100% rename from bloom/tox.ini rename to tox.ini