-
Notifications
You must be signed in to change notification settings - Fork 7
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Assembly.download module and Events.dump logging updated #203
Changes from all commits
ce9be8c
e9396b1
96de600
28db808
1d8de79
686815e
c6030f6
27f52a1
4fb1f2c
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -239,13 +239,6 @@ def retrieve_assembly_data( | |
download_path = Path(download_dir) | ||
download_dir = download_path / accession | ||
|
||
# Configure logging | ||
log_file = f"{accession}_download.log" | ||
reload(logging) | ||
logging.basicConfig( | ||
filename=log_file, format="%(levelname)s:%(message)s", filemode="w", level=logging.DEBUG | ||
) | ||
|
||
# Set and create dedicated dir for download | ||
if not download_dir.is_dir(): | ||
download_dir.mkdir(parents=True) | ||
|
@@ -281,6 +274,25 @@ def main() -> None: | |
parser.add_argument_dst_path( | ||
"--download_dir", default=Path.cwd(), help="Folder where the data will be downloaded" | ||
) | ||
parser.add_argument("-v", "--verbose", action="store_true", required=False, | ||
help="Verbose level logging") | ||
parser.add_argument("-d", "--debug", action="store_true", required=False, | ||
help="Debug level logging") | ||
args = parser.parse_args() | ||
|
||
retrieve_assembly_data(**vars(args)) | ||
# Configure logging | ||
date_format='%Y/%m/%d_%I:%M:%S(%p)' | ||
logging_format='%(asctime)s - %(levelname)s - %(message)s' | ||
reload(logging) | ||
log_level = None | ||
if args.debug: | ||
log_level = logging.DEBUG | ||
elif args.verbose: | ||
log_level = logging.INFO | ||
|
||
logging.basicConfig( | ||
format=logging_format, datefmt=date_format, | ||
filemode="w", level=log_level | ||
) | ||
|
||
retrieve_assembly_data(args.accession, args.download_dir) | ||
Comment on lines
+277
to
+298
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This should be rewritten with the new argparse and init_logging from ensembl-py |
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -16,7 +16,9 @@ | |
|
||
from datetime import datetime | ||
from pathlib import Path | ||
from importlib import reload | ||
from typing import List, Dict, Optional, Set, Tuple | ||
import logging | ||
|
||
from ensembl.brc4.runnable.core_server import CoreServer | ||
from ensembl.utils.argparse import ArgumentParser | ||
|
@@ -307,7 +309,7 @@ def get_history(self) -> List: | |
|
||
events = [] | ||
for session in sessions: | ||
print(f"Mapping session {session['release']}") | ||
logging.info(f"Mapping session {session['release']}") | ||
pairs = self.get_pairs(session["id"]) | ||
session_events = self.make_events(pairs) | ||
for event in session_events: | ||
|
@@ -327,7 +329,7 @@ def print_events(self, events: List[StableIdEvent], output_file: Path) -> None: | |
|
||
""" | ||
if not events: | ||
print("No events to print") | ||
logging.info("No events to print") | ||
return | ||
with output_file.open("w") as out_fh: | ||
for event in events: | ||
|
@@ -379,7 +381,7 @@ def get_pairs(self, session_id: int) -> List[Pair]: | |
for db in cursor: | ||
pair = Pair(old_id=db[0], new_id=db[1]) | ||
pairs.append(pair) | ||
print(f"{len(pairs)} stable id events") | ||
logging.debug(f"{len(pairs)} stable id events") | ||
return pairs | ||
|
||
def make_events(self, pairs: List[Pair]) -> List: | ||
|
@@ -423,7 +425,7 @@ def make_events(self, pairs: List[Pair]) -> List: | |
stats[name] += 1 | ||
|
||
for stat, value in stats.items(): | ||
print(f"\t{stat} = {value}") | ||
logging.info(f"\t{stat} = {value}") | ||
|
||
return events | ||
|
||
|
@@ -525,8 +527,27 @@ def main() -> None: | |
) | ||
parser.add_server_arguments(include_database=True) | ||
parser.add_argument_dst_path("--output_file", required=True, help="Output file") | ||
parser.add_argument("-v", "--verbose", action="store_true", required=False, | ||
help="Verbose level logging") | ||
parser.add_argument("-d", "--debug", action="store_true", required=False, | ||
help="Debug level logging") | ||
args = parser.parse_args() | ||
|
||
# Configure logging | ||
date_format='%Y/%m/%d_%I:%M:%S(%p)' | ||
logging_format='%(asctime)s - %(levelname)s - %(message)s' | ||
reload(logging) | ||
log_level = None | ||
if args.debug: | ||
log_level = logging.DEBUG | ||
elif args.verbose: | ||
log_level = logging.INFO | ||
|
||
logging.basicConfig( | ||
format=logging_format, datefmt=date_format, | ||
filemode="w", level=log_level | ||
) | ||
|
||
Comment on lines
+530
to
+550
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Likewise, rewrite with new system |
||
# Start | ||
factory = CoreServer(host=args.host, port=args.port, user=args.user, password=args.password) | ||
factory.set_database(args.database) | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -21,7 +21,9 @@ | |
from os import PathLike | ||
from pathlib import Path | ||
import re | ||
from importlib import reload | ||
from typing import Dict, Generator, List, Optional, Tuple | ||
import logging | ||
|
||
from sqlalchemy.orm import Session | ||
|
||
|
@@ -64,7 +66,7 @@ def add_event(self, event: IdEvent) -> None: | |
|
||
|
||
class EventCollection: | ||
""" "Collection of events with loader/writer in various formats.""" | ||
"""Collection of events with loader/writer in various formats.""" | ||
|
||
def __init__(self) -> None: | ||
self.events: List[IdEvent] = [] | ||
|
@@ -143,7 +145,7 @@ def _parse_gene_diff_event(self, event_string: str) -> Generator[Tuple[str, str, | |
splitter = f"({event_sep})" | ||
parts = re.split(splitter, event_string) | ||
if len(parts) != 3: | ||
print(f"Wrong partition: from '{event_string}' to '{parts}'") | ||
logging.warning(f"Wrong partition: from '{event_string}' to '{parts}'") | ||
return | ||
[from_ids, sep, to_ids] = parts | ||
event_name = event_symbol[sep] | ||
|
@@ -165,7 +167,7 @@ def remap_to_ids(self, map_dict: Dict[str, str]): | |
elif event.to_id in map_dict: | ||
event.to_id = map_dict[event.to_id] | ||
else: | ||
print(f"No map for to_id {event.to_id}") | ||
logging.info(f"No map for to_id {event.to_id}") | ||
no_map += 1 | ||
|
||
if no_map: | ||
|
@@ -174,7 +176,7 @@ def remap_to_ids(self, map_dict: Dict[str, str]): | |
def write_events_to_file(self, output_file: PathLike) -> None: | ||
"""Write the events to a file.""" | ||
with Path(output_file).open("w") as out_fh: | ||
print(f"Write {len(self.events)} events to {output_file}") | ||
logging.info(f"Write {len(self.events)} events to {output_file}") | ||
for event in self.events: | ||
out_fh.write(f"{event}\n") | ||
|
||
|
@@ -194,7 +196,7 @@ def write_events_to_db(self, session: Session, update: bool = False) -> None: | |
# Then, add the mapping, and the events for this mapping | ||
for release, mapping in mappings.items(): | ||
if update: | ||
print(f"Adding mapping for release {release} ({len(mapping.events)} events)") | ||
logging.info(f"Adding mapping for release {release} ({len(mapping.events)} events)") | ||
map_session = MappingSession(new_release=mapping.release, created=mapping.release_date) | ||
session.add(map_session) | ||
session.flush() | ||
|
@@ -217,9 +219,9 @@ def write_events_to_db(self, session: Session, update: bool = False) -> None: | |
session.add(id_event) | ||
session.commit() | ||
else: | ||
print(f"Found mapping for release {release} ({len(mapping.events)} events)") | ||
logging.info(f"Found mapping for release {release} ({len(mapping.events)} events)") | ||
if not update: | ||
print("Run your command again with '--update' to add them") | ||
logging.info("Run your command again with '--update' to add mapping events.") | ||
|
||
|
||
def main() -> None: | ||
|
@@ -235,8 +237,27 @@ def main() -> None: | |
), | ||
) | ||
parser.add_argument("--update", action="store_true", help="Make changes to the database") | ||
parser.add_argument("-v", "--verbose", action="store_true", required=False, | ||
help="Verbose level logging") | ||
parser.add_argument("-d", "--debug", action="store_true", required=False, | ||
help="Debug level logging") | ||
args = parser.parse_args() | ||
|
||
# Configure logging | ||
date_format='%Y/%m/%d_%I:%M:%S(%p)' | ||
logging_format='%(asctime)s - %(levelname)s - %(message)s' | ||
reload(logging) | ||
log_level = None | ||
if args.debug: | ||
log_level = logging.DEBUG | ||
elif args.verbose: | ||
log_level = logging.INFO | ||
|
||
logging.basicConfig( | ||
format=logging_format, datefmt=date_format, | ||
filemode="w", level=log_level | ||
) | ||
|
||
Comment on lines
+240
to
+260
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Again, rewrite |
||
# Start | ||
dbc = DBConnection(args.url) | ||
|
||
|
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I don't think this is needed: if there is an exception here, it should be allowed to be raised and the script should stop anyway