Assembly.download module and Events.dump logging updated

pipelines/nextflow/modules/download/download_asm_data.nf

@@ -36,6 +36,6 @@ process DOWNLOAD_ASM_DATA {
 
     shell:
         '''
-        assembly_download --accession !{meta.accession} --asm_download_dir ./
+        assembly_download --accession !{meta.accession} --asm_download_dir ./ -d
         '''
 }

pipelines/nextflow/modules/events/dump_events.nf

@@ -33,6 +33,7 @@ process DUMP_EVENTS {
             --user '${server.user}' \
             --password '${server.password}' \
             --database '${db.database}' \
-            --output_file "events.txt"
+            --output_file "events.txt" \
+            -d
         """
 }

src/python/ensembl/brc4/runnable/core_server.py

@@ -17,10 +17,10 @@
 import argparse
 import re
 from typing import Dict, List, Any
+import logging
 
 import mysql.connector
 
-
 class CoreServer:
     """Basic interface to a MySQL server with core databases.
 
@@ -55,7 +55,10 @@ def connect(self) -> None:
         )
 
     def set_database(self, db_name: str) -> None:
-        self._connector.database = db_name
+        try:
+            self._connector.database = db_name
+        except:
+            logging.critical(f"Unknown database: '{db_name}' - Not located on host !")
 
     def get_cursor(self):
         return self._connector.cursor()
@@ -90,6 +93,10 @@ def get_cores(
 
         dbs = self.get_all_cores()
 
+        # Check if there are databases returned from query to host
+        if not dbs:
+            logging.warning("No databases returned from query")
+
         if prefix:
             dbs = [db for db in dbs if db.startswith(f"{prefix}")]
         if dbname_re:

src/python/ensembl/io/genomio/assembly/download.py

@@ -239,13 +239,6 @@ def retrieve_assembly_data(
     download_path = Path(download_dir)
     download_dir = download_path / accession
 
-    # Configure logging
-    log_file = f"{accession}_download.log"
-    reload(logging)
-    logging.basicConfig(
-        filename=log_file, format="%(levelname)s:%(message)s", filemode="w", level=logging.DEBUG
-    )
-
     # Set and create dedicated dir for download
     if not download_dir.is_dir():
         download_dir.mkdir(parents=True)
@@ -281,6 +274,25 @@ def main() -> None:
     parser.add_argument_dst_path(
         "--download_dir", default=Path.cwd(), help="Folder where the data will be downloaded"
     )
+    parser.add_argument("-v", "--verbose", action="store_true", required=False,
+        help="Verbose level logging")
+    parser.add_argument("-d", "--debug", action="store_true", required=False,
+        help="Debug level logging")
     args = parser.parse_args()
 
-    retrieve_assembly_data(**vars(args))
+    # Configure logging
+    date_format='%Y/%m/%d_%I:%M:%S(%p)'
+    logging_format='%(asctime)s - %(levelname)s - %(message)s'
+    reload(logging)
+    log_level = None
+    if args.debug:
+        log_level = logging.DEBUG
+    elif args.verbose:
+        log_level = logging.INFO
+
+    logging.basicConfig(
+        format=logging_format, datefmt=date_format,
+        filemode="w", level=log_level
+        )
+
+    retrieve_assembly_data(args.accession, args.download_dir)

src/python/ensembl/io/genomio/events/dump.py

@@ -16,7 +16,9 @@
 
 from datetime import datetime
 from pathlib import Path
+from importlib import reload
 from typing import List, Dict, Optional, Set, Tuple
+import logging
 
 from ensembl.brc4.runnable.core_server import CoreServer
 from ensembl.utils.argparse import ArgumentParser
@@ -307,7 +309,7 @@ def get_history(self) -> List:
 
         events = []
         for session in sessions:
-            print(f"Mapping session {session['release']}")
+            logging.info(f"Mapping session {session['release']}")
             pairs = self.get_pairs(session["id"])
             session_events = self.make_events(pairs)
             for event in session_events:
@@ -327,7 +329,7 @@ def print_events(self, events: List[StableIdEvent], output_file: Path) -> None:
 
         """
         if not events:
-            print("No events to print")
+            logging.info("No events to print")
             return
         with output_file.open("w") as out_fh:
             for event in events:
@@ -379,7 +381,7 @@ def get_pairs(self, session_id: int) -> List[Pair]:
         for db in cursor:
             pair = Pair(old_id=db[0], new_id=db[1])
             pairs.append(pair)
-        print(f"{len(pairs)} stable id events")
+        logging.debug(f"{len(pairs)} stable id events")
         return pairs
 
     def make_events(self, pairs: List[Pair]) -> List:
@@ -423,7 +425,7 @@ def make_events(self, pairs: List[Pair]) -> List:
                 stats[name] += 1
 
         for stat, value in stats.items():
-            print(f"\t{stat} = {value}")
+            logging.info(f"\t{stat} = {value}")
 
         return events
 
@@ -525,8 +527,27 @@ def main() -> None:
     )
     parser.add_server_arguments(include_database=True)
     parser.add_argument_dst_path("--output_file", required=True, help="Output file")
+    parser.add_argument("-v", "--verbose", action="store_true", required=False,
+        help="Verbose level logging")
+    parser.add_argument("-d", "--debug", action="store_true", required=False,
+        help="Debug level logging")
     args = parser.parse_args()
 
+    # Configure logging
+    date_format='%Y/%m/%d_%I:%M:%S(%p)'
+    logging_format='%(asctime)s - %(levelname)s - %(message)s'
+    reload(logging)
+    log_level = None
+    if args.debug:
+        log_level = logging.DEBUG
+    elif args.verbose:
+        log_level = logging.INFO
+
+    logging.basicConfig(
+        format=logging_format, datefmt=date_format,
+        filemode="w", level=log_level
+        )
+
     # Start
     factory = CoreServer(host=args.host, port=args.port, user=args.user, password=args.password)
     factory.set_database(args.database)

src/python/ensembl/io/genomio/events/load.py

@@ -21,7 +21,9 @@
 from os import PathLike
 from pathlib import Path
 import re
+from importlib import reload
 from typing import Dict, Generator, List, Optional, Tuple
+import logging
 
 from sqlalchemy.orm import Session
 
@@ -64,7 +66,7 @@ def add_event(self, event: IdEvent) -> None:
 
 
 class EventCollection:
-    """ "Collection of events with loader/writer in various formats."""
+    """Collection of events with loader/writer in various formats."""
 
     def __init__(self) -> None:
         self.events: List[IdEvent] = []
@@ -143,7 +145,7 @@ def _parse_gene_diff_event(self, event_string: str) -> Generator[Tuple[str, str,
         splitter = f"({event_sep})"
         parts = re.split(splitter, event_string)
         if len(parts) != 3:
-            print(f"Wrong partition: from '{event_string}' to '{parts}'")
+            logging.warning(f"Wrong partition: from '{event_string}' to '{parts}'")
             return
         [from_ids, sep, to_ids] = parts
         event_name = event_symbol[sep]
@@ -165,7 +167,7 @@ def remap_to_ids(self, map_dict: Dict[str, str]):
             elif event.to_id in map_dict:
                 event.to_id = map_dict[event.to_id]
             else:
-                print(f"No map for to_id {event.to_id}")
+                logging.info(f"No map for to_id {event.to_id}")
                 no_map += 1
 
         if no_map:
@@ -174,7 +176,7 @@ def remap_to_ids(self, map_dict: Dict[str, str]):
     def write_events_to_file(self, output_file: PathLike) -> None:
         """Write the events to a file."""
         with Path(output_file).open("w") as out_fh:
-            print(f"Write {len(self.events)} events to {output_file}")
+            logging.info(f"Write {len(self.events)} events to {output_file}")
             for event in self.events:
                 out_fh.write(f"{event}\n")
 
@@ -194,7 +196,7 @@ def write_events_to_db(self, session: Session, update: bool = False) -> None:
         # Then, add the mapping, and the events for this mapping
         for release, mapping in mappings.items():
             if update:
-                print(f"Adding mapping for release {release} ({len(mapping.events)} events)")
+                logging.info(f"Adding mapping for release {release} ({len(mapping.events)} events)")
                 map_session = MappingSession(new_release=mapping.release, created=mapping.release_date)
                 session.add(map_session)
                 session.flush()
@@ -217,9 +219,9 @@ def write_events_to_db(self, session: Session, update: bool = False) -> None:
                     session.add(id_event)
                 session.commit()
             else:
-                print(f"Found mapping for release {release} ({len(mapping.events)} events)")
+                logging.info(f"Found mapping for release {release} ({len(mapping.events)} events)")
         if not update:
-            print("Run your command again with '--update' to add them")
+            logging.info("Run your command again with '--update' to add mapping events.")
 
 
 def main() -> None:
@@ -235,8 +237,27 @@ def main() -> None:
         ),
     )
     parser.add_argument("--update", action="store_true", help="Make changes to the database")
+    parser.add_argument("-v", "--verbose", action="store_true", required=False,
+        help="Verbose level logging")
+    parser.add_argument("-d", "--debug", action="store_true", required=False,
+        help="Debug level logging")
     args = parser.parse_args()
 
+    # Configure logging
+    date_format='%Y/%m/%d_%I:%M:%S(%p)'
+    logging_format='%(asctime)s - %(levelname)s - %(message)s'
+    reload(logging)
+    log_level = None
+    if args.debug:
+        log_level = logging.DEBUG
+    elif args.verbose:
+        log_level = logging.INFO
+
+    logging.basicConfig(
+        format=logging_format, datefmt=date_format,
+        filemode="w", level=log_level
+        )
+
     # Start
     dbc = DBConnection(args.url)