From 695e88ae3c536a9edd029e3c20ac591be2e7bc60 Mon Sep 17 00:00:00 2001 From: Benedikt Hilmes Date: Thu, 22 Sep 2022 23:57:19 +0200 Subject: [PATCH 01/22] first draft reports --- report/__init__.py | 0 report/report.py | 52 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 52 insertions(+) create mode 100644 report/__init__.py create mode 100644 report/report.py diff --git a/report/__init__.py b/report/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/report/report.py b/report/report.py new file mode 100644 index 00000000..c388c273 --- /dev/null +++ b/report/report.py @@ -0,0 +1,52 @@ +from sisyphus import * +from sisyphus import global_settings +import sys +import os +import getpass +import pprint +import gzip + + +class ReportFinishedOutput(Job): + + def __init__(self, + name: str, + output: tk.Path, + ): + self.name = name + self.output = output + self.out = self.output_path('report.gz') + + if hasattr(global_settings, "GLOBAL_SETTINGS_FILE"): + with open(global_settings.GLOBAL_SETTINGS_FILE) as f: + self.settings = f.read() + else: + self.settings = '' + + self.sis_command_line = sys.argv + self.cwd = os.path.abspath('.') + self.report_format = pprint.pformat + self.mail_address = getattr(global_settings, 'MAIL_ADDRESS', None) + + + def run(self): + user = getpass.getuser() + report = { + "output": self.output.get_path(), + "user": user, + "name": self.name, + "sis_command_line": self.sis_command_line, + "cwd": self.cwd, + } + + report = eval(repr(report)) + pprint.pprint(report) + + with gzip.open(str(self.out), 'w') as f: + f.write(self.report_format(report).encode() + b'\n') + + if self.mail_address: + self.sh('zcat {out} | mail -s \'Report finished: {name} \' {mail_address}') + + def tasks(self): + yield Task('run', mini_task=True) From 11eaacb0fb58dc7a3b52206dc8f4c26dbbac2b5f Mon Sep 17 00:00:00 2001 From: Benedikt Hilmes Date: Thu, 22 Sep 2022 23:58:04 +0200 Subject: [PATCH 02/22] import and black --- report/__init__.py | 1 + report/report.py | 83 +++++++++++++++++++++++----------------------- 2 files changed, 42 insertions(+), 42 deletions(-) diff --git a/report/__init__.py b/report/__init__.py index e69de29b..59a72895 100644 --- a/report/__init__.py +++ b/report/__init__.py @@ -0,0 +1 @@ +from .report import ReportFinishedOutput diff --git a/report/report.py b/report/report.py index c388c273..aff48d3f 100644 --- a/report/report.py +++ b/report/report.py @@ -8,45 +8,44 @@ class ReportFinishedOutput(Job): - - def __init__(self, - name: str, - output: tk.Path, - ): - self.name = name - self.output = output - self.out = self.output_path('report.gz') - - if hasattr(global_settings, "GLOBAL_SETTINGS_FILE"): - with open(global_settings.GLOBAL_SETTINGS_FILE) as f: - self.settings = f.read() - else: - self.settings = '' - - self.sis_command_line = sys.argv - self.cwd = os.path.abspath('.') - self.report_format = pprint.pformat - self.mail_address = getattr(global_settings, 'MAIL_ADDRESS', None) - - - def run(self): - user = getpass.getuser() - report = { - "output": self.output.get_path(), - "user": user, - "name": self.name, - "sis_command_line": self.sis_command_line, - "cwd": self.cwd, - } - - report = eval(repr(report)) - pprint.pprint(report) - - with gzip.open(str(self.out), 'w') as f: - f.write(self.report_format(report).encode() + b'\n') - - if self.mail_address: - self.sh('zcat {out} | mail -s \'Report finished: {name} \' {mail_address}') - - def tasks(self): - yield Task('run', mini_task=True) + def __init__( + self, + name: str, + output: tk.Path, + ): + self.name = name + self.output = output + self.out = self.output_path("report.gz") + + if hasattr(global_settings, "GLOBAL_SETTINGS_FILE"): + with open(global_settings.GLOBAL_SETTINGS_FILE) as f: + self.settings = f.read() + else: + self.settings = "" + + self.sis_command_line = sys.argv + self.cwd = os.path.abspath(".") + self.report_format = pprint.pformat + self.mail_address = getattr(global_settings, "MAIL_ADDRESS", None) + + def run(self): + user = getpass.getuser() + report = { + "output": self.output.get_path(), + "user": user, + "name": self.name, + "sis_command_line": self.sis_command_line, + "cwd": self.cwd, + } + + report = eval(repr(report)) + pprint.pprint(report) + + with gzip.open(str(self.out), "w") as f: + f.write(self.report_format(report).encode() + b"\n") + + if self.mail_address: + self.sh("zcat {out} | mail -s 'Report finished: {name} ' {mail_address}") + + def tasks(self): + yield Task("run", mini_task=True) From f7601035e96b819f697ee876b852b1f39bc3c0f2 Mon Sep 17 00:00:00 2001 From: Benedikt Hilmes Date: Fri, 23 Sep 2022 00:50:27 +0200 Subject: [PATCH 03/22] updates --- report/__init__.py | 2 +- report/report.py | 11 +++++++---- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/report/__init__.py b/report/__init__.py index 59a72895..f6554cba 100644 --- a/report/__init__.py +++ b/report/__init__.py @@ -1 +1 @@ -from .report import ReportFinishedOutput +from .report import SimpleReportJob diff --git a/report/report.py b/report/report.py index aff48d3f..d5539837 100644 --- a/report/report.py +++ b/report/report.py @@ -5,16 +5,17 @@ import getpass import pprint import gzip +from typing import Dict -class ReportFinishedOutput(Job): +class SimpleReportJob(Job): def __init__( self, name: str, - output: tk.Path, + results: Dict[str, tk.Variable], ): self.name = name - self.output = output + self.results = results self.out = self.output_path("report.gz") if hasattr(global_settings, "GLOBAL_SETTINGS_FILE"): @@ -30,8 +31,10 @@ def __init__( def run(self): user = getpass.getuser() + for var in self.results: + self.results[var] = self.results[var].get() report = { - "output": self.output.get_path(), + "output": self.results, "user": user, "name": self.name, "sis_command_line": self.sis_command_line, From 3a2353a67d1a59a91ef40705994bdaf881849a21 Mon Sep 17 00:00:00 2001 From: Benedikt Hilmes Date: Fri, 23 Sep 2022 23:40:29 +0200 Subject: [PATCH 04/22] now using report job from MT/Old ASR --- report/__init__.py | 2 +- report/report.py | 216 +++++++++++++++++++++++++++++++++++++++------ 2 files changed, 189 insertions(+), 29 deletions(-) diff --git a/report/__init__.py b/report/__init__.py index f6554cba..8700721a 100644 --- a/report/__init__.py +++ b/report/__init__.py @@ -1 +1 @@ -from .report import SimpleReportJob +from .report import * diff --git a/report/report.py b/report/report.py index d5539837..4d314fdd 100644 --- a/report/report.py +++ b/report/report.py @@ -1,54 +1,214 @@ from sisyphus import * -from sisyphus import global_settings import sys import os import getpass import pprint import gzip -from typing import Dict +from typing import Dict, Union, Callable +import time +import json +_Report_Type = Dict[str, Union[tk.Path, any]] + + +class ReportResultsJob(Job): + """ + Job to report results either via mail and/or just condensed in a file + """ -class SimpleReportJob(Job): def __init__( self, name: str, - results: Dict[str, tk.Variable], + report: _Report_Type, + report_dir: tk.Path = None, + report_format: Callable[[_Report_Type], str] = None, + recipe_path=Path("."), ): + """ + + :param name: + :param dict report: dictionary containing the report files + :param report_dir: + :param (report: dict) -> List[str] report_format: + :param recipe_path: + """ self.name = name - self.results = results - self.out = self.output_path("report.gz") + self.report = report + self.report_dir = report_dir + self.report_format = report_format + self.recipe_path = recipe_path - if hasattr(global_settings, "GLOBAL_SETTINGS_FILE"): - with open(global_settings.GLOBAL_SETTINGS_FILE) as f: - self.settings = f.read() - else: - self.settings = "" + self.out_report = self.output_path("report.gz") self.sis_command_line = sys.argv self.cwd = os.path.abspath(".") - self.report_format = pprint.pformat - self.mail_address = getattr(global_settings, "MAIL_ADDRESS", None) + self.config_path = tk.config_manager.current_config + assert self.config_path is not None, "Could not find config path" + try: + with open(self.config_path) as f: + self.config = f.read() + except IOError as e: + if e.errno != 2: + raise e + else: + self.config = self.config_path + + with open(gs.GLOBAL_SETTINGS_FILE) as f: + self.settings = f.read() + + for i in [ + "recipe_hash", + "recipe_date", + "date", + "user", + "name", + "config", + "config_path", + "config_line", + "settings", + "cwd", + "sis_command_line", + ]: + assert i not in report, "%s will be set automatically" def run(self): user = getpass.getuser() - for var in self.results: - self.results[var] = self.results[var].get() - report = { - "output": self.results, - "user": user, - "name": self.name, - "sis_command_line": self.sis_command_line, - "cwd": self.cwd, - } - - report = eval(repr(report)) - pprint.pprint(report) - - with gzip.open(str(self.out), "w") as f: + report = self.report.copy() + + report["date"] = time.time() + report["user"] = user + report["name"] = self.name + report["config_path"] = self.config_path + report["config"] = self.config + report["settings"] = self.settings + report["sis_command_line"] = self.sis_command_line + report["cwd"] = self.cwd + + # report = eval(repr(report)) + if self.report_format is None: + self.report_format = pprint.pformat + + with gzip.open(str(self.out_report), "w") as f: f.write(self.report_format(report).encode() + b"\n") + if self.report_dir: + report_file_name = "%s.%s.%s.gz" % ( + user, + self.config_path.replace("/", "_"), + self.name, + ) + report_path = os.path.join(self.report_dir, report_file_name) + print("Write report to %s" % report_path) + with gzip.open(report_path, "w") as f: + f.write(json.dumps(report).encode() + b"\n") + + self.mail_address = getattr(gs, "MAIL_ADDRESS", None) if self.mail_address: - self.sh("zcat {out} | mail -s 'Report finished: {name} ' {mail_address}") + self.sh( + "zcat {out_report} | mail -s 'Report finished: {name} {config_path}' {mail_address}" + ) def tasks(self): yield Task("run", mini_task=True) + + +def gmm_example_report_format(report: Dict[str, tk.Variable]) -> str: + """ + Example report format for a GMM evaluated on dev-clean and dev-other + :param report: + :return: + """ + results = { + "dev-clean": { + "Monophone": {}, + "Triphone": {}, + "SAT": {}, + "VTLN": {}, + "VTLN+SAT": {}, + }, + "dev-other": { + "Monophone": {}, + "Triphone": {}, + "SAT": {}, + "VTLN": {}, + "VTLN+SAT": {}, + }, + } + for step_name, score in report.items(): + if "dev-clean" in step_name: + set = "dev-clean" + else: + set = "dev-other" + if not step_name.startswith("scorer"): + continue + if "mono" in step_name: + step = "Monophone" + elif "tri" in step_name: + step = "Triphone" + elif "vtln+sat" in step_name: + step = "VTLN+SAT" + elif "sat" in step_name: + step = "SAT" + else: + step = "VTLN" + if "iter08" in step_name: + results[set][step]["08"] = score + elif "iter10" in step_name: + results[set][step]["10"] = score + + out = [] + out.append( + f"""Name: {report["name"]} + Path: {report["config_path"]} + Date: {report["date"]} + + Results:""" + ) + out.append("Step".ljust(20) + "dev-clean".ljust(10) + "dev-other") + out.append( + "Monophone".ljust(16) + + str(results["dev-clean"]["Monophone"]["10"]).ljust(14) + + str(results["dev-other"]["Monophone"]["10"]) + ) + out.append( + "Triphone 08".ljust(19) + + str(results["dev-clean"]["Triphone"]["08"]).ljust(14) + + str(results["dev-other"]["Triphone"]["08"]) + ) + out.append( + "Triphone 10".ljust(19) + + str(results["dev-clean"]["Triphone"]["10"]).ljust(14) + + str(results["dev-other"]["Triphone"]["10"]) + ) + out.append( + "VTLN 08".ljust(21) + + str(results["dev-clean"]["VTLN"]["08"]).ljust(14) + + str(results["dev-other"]["VTLN"]["08"]) + ) + out.append( + "VTLN 10".ljust(21) + + str(results["dev-clean"]["VTLN"]["10"]).ljust(14) + + str(results["dev-other"]["VTLN"]["10"]) + ) + out.append( + "SAT 08".ljust(23) + + str(results["dev-clean"]["SAT"]["08"]).ljust(14) + + str(results["dev-other"]["SAT"]["08"]) + ) + out.append( + "SAT 10".ljust(23) + + str(results["dev-clean"]["SAT"]["10"]).ljust(14) + + str(results["dev-other"]["SAT"]["10"]) + ) + out.append( + "VTLN+SAT 08".ljust(17) + + str(results["dev-clean"]["VTLN+SAT"]["08"]).ljust(14) + + str(results["dev-other"]["VTLN+SAT"]["08"]) + ) + out.append( + "VTLN+SAT 10".ljust(17) + + str(results["dev-clean"]["VTLN+SAT"]["10"]).ljust(14) + + str(results["dev-other"]["VTLN+SAT"]["10"]) + ) + + return "\n".join(out) From 353deaac689817d67832ad3ff038d5a932cec85d Mon Sep 17 00:00:00 2001 From: Benedikt Hilmes Date: Mon, 26 Sep 2022 09:27:31 +0200 Subject: [PATCH 05/22] cleanup --- report/report.py | 1 - 1 file changed, 1 deletion(-) diff --git a/report/report.py b/report/report.py index 4d314fdd..a961d22c 100644 --- a/report/report.py +++ b/report/report.py @@ -84,7 +84,6 @@ def run(self): report["sis_command_line"] = self.sis_command_line report["cwd"] = self.cwd - # report = eval(repr(report)) if self.report_format is None: self.report_format = pprint.pformat From a33158bbe39db9544f14dac3617fa4c6834d1766 Mon Sep 17 00:00:00 2001 From: Benedikt Hilmes Date: Tue, 27 Sep 2022 09:05:47 +0200 Subject: [PATCH 06/22] Update report/report.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Christoph M. Lüscher --- report/report.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/report/report.py b/report/report.py index a961d22c..89340776 100644 --- a/report/report.py +++ b/report/report.py @@ -113,7 +113,7 @@ def tasks(self): def gmm_example_report_format(report: Dict[str, tk.Variable]) -> str: """ - Example report format for a GMM evaluated on dev-clean and dev-other + Example report format for a GMM evaluated on dev-clean and dev-other of the LibrSpeech dataset :param report: :return: """ From b26e2a948fe8c3121afc4411aa1e4be9b76e7a62 Mon Sep 17 00:00:00 2001 From: Benedikt Hilmes Date: Tue, 27 Sep 2022 10:32:33 +0200 Subject: [PATCH 07/22] updates --- report/report.py | 99 ++++++++++++++++-------------------------------- 1 file changed, 32 insertions(+), 67 deletions(-) diff --git a/report/report.py b/report/report.py index 89340776..c9b30589 100644 --- a/report/report.py +++ b/report/report.py @@ -1,14 +1,12 @@ from sisyphus import * + import sys -import os import getpass -import pprint import gzip +from datetime import datetime from typing import Dict, Union, Callable -import time -import json -_Report_Type = Dict[str, Union[tk.Path, any]] +_Report_Type = Dict[str, Union[tk.AbstractPath, str]] class ReportResultsJob(Job): @@ -20,91 +18,57 @@ def __init__( self, name: str, report: _Report_Type, - report_dir: tk.Path = None, - report_format: Callable[[_Report_Type], str] = None, - recipe_path=Path("."), + report_format: Callable[[_Report_Type, any], str], + compress: bool = True, + **report_format_kwargs, ): """ - :param name: - :param dict report: dictionary containing the report files - :param report_dir: - :param (report: dict) -> List[str] report_format: - :param recipe_path: + :param name: Name of the report + :param report: Dictionary containing the report files + :param report_format: Function that converts the report dictionary to a string + :param report_format_kwargs: Addtional kwargs for the report format function + :param compress: Whether to compress the report or not """ self.name = name self.report = report - self.report_dir = report_dir self.report_format = report_format - self.recipe_path = recipe_path - - self.out_report = self.output_path("report.gz") + self.report_format_kwargs = report_format_kwargs + self.compress = compress self.sis_command_line = sys.argv - self.cwd = os.path.abspath(".") + self.mail_address = getattr(gs, "MAIL_ADDRESS", None) self.config_path = tk.config_manager.current_config assert self.config_path is not None, "Could not find config path" - try: - with open(self.config_path) as f: - self.config = f.read() - except IOError as e: - if e.errno != 2: - raise e - else: - self.config = self.config_path - - with open(gs.GLOBAL_SETTINGS_FILE) as f: - self.settings = f.read() - - for i in [ - "recipe_hash", - "recipe_date", - "date", - "user", - "name", - "config", - "config_path", - "config_line", - "settings", - "cwd", - "sis_command_line", - ]: + + for i in ["date", "user", "name", "config_path", "sis_command_line"]: assert i not in report, "%s will be set automatically" + self.out_report = self.output_path("report.gz") + def run(self): user = getpass.getuser() report = self.report.copy() - report["date"] = time.time() + report["date"] = datetime.now().strftime("%d/%m/%Y %H:%M:%S") report["user"] = user report["name"] = self.name report["config_path"] = self.config_path - report["config"] = self.config - report["settings"] = self.settings - report["sis_command_line"] = self.sis_command_line - report["cwd"] = self.cwd - - if self.report_format is None: - self.report_format = pprint.pformat - - with gzip.open(str(self.out_report), "w") as f: - f.write(self.report_format(report).encode() + b"\n") - - if self.report_dir: - report_file_name = "%s.%s.%s.gz" % ( - user, - self.config_path.replace("/", "_"), - self.name, - ) - report_path = os.path.join(self.report_dir, report_file_name) - print("Write report to %s" % report_path) - with gzip.open(report_path, "w") as f: - f.write(json.dumps(report).encode() + b"\n") + report["sis_command_line"] = str(self.sis_command_line) + + if self.compress: + with gzip.open(self.out_report.get_path(), "w") as f: + f.write( + self.report_format(report, **self.report_format_kwargs).encode() + + b"\n" + ) + else: + with open(self.out_report.get_path(), "w") as f: + f.write(self.report_format(report, **self.report_format_kwargs) + "\n") - self.mail_address = getattr(gs, "MAIL_ADDRESS", None) if self.mail_address: self.sh( - "zcat {out_report} | mail -s 'Report finished: {name} {config_path}' {mail_address}" + "zcat -f {out_report} | mail -s 'Report finished: {name} {config_path}' {mail_address}" ) def tasks(self): @@ -160,6 +124,7 @@ def gmm_example_report_format(report: Dict[str, tk.Variable]) -> str: f"""Name: {report["name"]} Path: {report["config_path"]} Date: {report["date"]} + Sis Command: {report["sis_command_line"]} Results:""" ) From 94221d078d9bb03d7d1b1867a7d5e937a93b4e8e Mon Sep 17 00:00:00 2001 From: Benedikt Hilmes Date: Tue, 27 Sep 2022 10:35:51 +0200 Subject: [PATCH 08/22] report type for example --- report/report.py | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/report/report.py b/report/report.py index c9b30589..69836a2b 100644 --- a/report/report.py +++ b/report/report.py @@ -5,7 +5,6 @@ import gzip from datetime import datetime from typing import Dict, Union, Callable - _Report_Type = Dict[str, Union[tk.AbstractPath, str]] @@ -20,7 +19,7 @@ def __init__( report: _Report_Type, report_format: Callable[[_Report_Type, any], str], compress: bool = True, - **report_format_kwargs, + **report_format_kwargs ): """ @@ -42,7 +41,7 @@ def __init__( assert self.config_path is not None, "Could not find config path" for i in ["date", "user", "name", "config_path", "sis_command_line"]: - assert i not in report, "%s will be set automatically" + assert i not in report, "%s will be set automatically" self.out_report = self.output_path("report.gz") @@ -57,14 +56,11 @@ def run(self): report["sis_command_line"] = str(self.sis_command_line) if self.compress: - with gzip.open(self.out_report.get_path(), "w") as f: - f.write( - self.report_format(report, **self.report_format_kwargs).encode() - + b"\n" - ) + with gzip.open(self.out_report.get_path(), "w") as f: + f.write(self.report_format(report, **self.report_format_kwargs).encode() + b"\n") else: - with open(self.out_report.get_path(), "w") as f: - f.write(self.report_format(report, **self.report_format_kwargs) + "\n") + with open(self.out_report.get_path(), "w") as f: + f.write(self.report_format(report, **self.report_format_kwargs) + "\n") if self.mail_address: self.sh( @@ -75,9 +71,9 @@ def tasks(self): yield Task("run", mini_task=True) -def gmm_example_report_format(report: Dict[str, tk.Variable]) -> str: +def gmm_example_report_format(report: _Report_Type) -> str: """ - Example report format for a GMM evaluated on dev-clean and dev-other of the LibrSpeech dataset + Example report format for a GMM evaluated on dev-clean and dev-other :param report: :return: """ From 0028a23fe6b1c439751c69c2b37e3d6f6b0d96b6 Mon Sep 17 00:00:00 2001 From: Benedikt Hilmes Date: Tue, 27 Sep 2022 10:37:12 +0200 Subject: [PATCH 09/22] black --- report/report.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/report/report.py b/report/report.py index 69836a2b..0027b8c9 100644 --- a/report/report.py +++ b/report/report.py @@ -5,6 +5,7 @@ import gzip from datetime import datetime from typing import Dict, Union, Callable + _Report_Type = Dict[str, Union[tk.AbstractPath, str]] @@ -19,7 +20,7 @@ def __init__( report: _Report_Type, report_format: Callable[[_Report_Type, any], str], compress: bool = True, - **report_format_kwargs + **report_format_kwargs, ): """ @@ -41,7 +42,7 @@ def __init__( assert self.config_path is not None, "Could not find config path" for i in ["date", "user", "name", "config_path", "sis_command_line"]: - assert i not in report, "%s will be set automatically" + assert i not in report, "%s will be set automatically" self.out_report = self.output_path("report.gz") @@ -56,11 +57,14 @@ def run(self): report["sis_command_line"] = str(self.sis_command_line) if self.compress: - with gzip.open(self.out_report.get_path(), "w") as f: - f.write(self.report_format(report, **self.report_format_kwargs).encode() + b"\n") + with gzip.open(self.out_report.get_path(), "w") as f: + f.write( + self.report_format(report, **self.report_format_kwargs).encode() + + b"\n" + ) else: - with open(self.out_report.get_path(), "w") as f: - f.write(self.report_format(report, **self.report_format_kwargs) + "\n") + with open(self.out_report.get_path(), "w") as f: + f.write(self.report_format(report, **self.report_format_kwargs) + "\n") if self.mail_address: self.sh( From dc93dd10c68d095be99a39b31227ce03a60d67f8 Mon Sep 17 00:00:00 2001 From: Benedikt Hilmes Date: Tue, 27 Sep 2022 10:38:25 +0200 Subject: [PATCH 10/22] order example --- report/report.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/report/report.py b/report/report.py index 0027b8c9..d3bb8b93 100644 --- a/report/report.py +++ b/report/report.py @@ -98,12 +98,12 @@ def gmm_example_report_format(report: _Report_Type) -> str: }, } for step_name, score in report.items(): + if not step_name.startswith("scorer"): + continue if "dev-clean" in step_name: set = "dev-clean" else: set = "dev-other" - if not step_name.startswith("scorer"): - continue if "mono" in step_name: step = "Monophone" elif "tri" in step_name: From 7c2b2d9a2aeffb4e41df53c3632d1e593559a5a5 Mon Sep 17 00:00:00 2001 From: Benedikt Hilmes Date: Tue, 27 Sep 2022 13:46:19 +0200 Subject: [PATCH 11/22] Mail Job --- report/report.py | 53 +++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 52 insertions(+), 1 deletion(-) diff --git a/report/report.py b/report/report.py index d3bb8b93..3ac4bec6 100644 --- a/report/report.py +++ b/report/report.py @@ -4,11 +4,62 @@ import getpass import gzip from datetime import datetime -from typing import Dict, Union, Callable +from typing import Dict, Union, Callable, Optional, Iterable _Report_Type = Dict[str, Union[tk.AbstractPath, str]] +class MailJob(Job): + """ + Job that sends a mail + """ + + def __init__( + self, + subject: str, + content: str, + mail_address: str = getpass.getuser(), + wait_on: Optional[Union[Iterable[tk.AbstractPath], tk.AbstractPath]] = None, + compress: bool = True, + ): + """ + + :param subject: Subject of the mail + :param mail_address: Mail address of the user + :param content: + :param wait_on: + """ + self.subject = subject + self.mail_address = mail_address + self.content = content + self.compress = compress + + if wait_on is not None: + if isinstance(wait_on, tk.AbstractPath): + self.add_input(wait_on) + elif isinstance(wait_on, Iterable): + for wait in wait_on: + self.add_input(wait) + else: + assert False, "Wrong wait_on input!" + + self.out_content = self.output_path("content") + + def tasks(self): + yield Task("run", mini_task=True) + + def run(self): + + if self.compress: + with gzip.open(self.out_content.get_path(), "w") as f: + f.write(self.content.encode() + b"\n") + else: + with open(self.out_content.get_path(), "w") as f: + f.write(self.content) + + self.sh("zcat -f {out_content} | mail -s '{subject}' {mail_address}") + + class ReportResultsJob(Job): """ Job to report results either via mail and/or just condensed in a file From 9bf63c6c8fe735541fc948bc549e1d57959d2b01 Mon Sep 17 00:00:00 2001 From: Benedikt Hilmes Date: Tue, 27 Sep 2022 18:23:13 +0200 Subject: [PATCH 12/22] more updates --- report/report.py | 128 +++++++++++++++++++---------------------------- 1 file changed, 51 insertions(+), 77 deletions(-) diff --git a/report/report.py b/report/report.py index 3ac4bec6..ffddc6d3 100644 --- a/report/report.py +++ b/report/report.py @@ -3,127 +3,101 @@ import sys import getpass import gzip -from datetime import datetime -from typing import Dict, Union, Callable, Optional, Iterable +from typing import Dict, Union, Callable, Optional +import pprint _Report_Type = Dict[str, Union[tk.AbstractPath, str]] -class MailJob(Job): +class GenerateReportStringJob(Job): """ - Job that sends a mail + Job to generate and output a report string """ def __init__( self, - subject: str, - content: str, - mail_address: str = getpass.getuser(), - wait_on: Optional[Union[Iterable[tk.AbstractPath], tk.AbstractPath]] = None, + report_values: _Report_Type, + report_template: Optional[Callable[[_Report_Type], str]] = None, compress: bool = True, ): """ - :param subject: Subject of the mail - :param mail_address: Mail address of the user - :param content: - :param wait_on: + :param report_values: + :param report_template: + :param compress: """ - self.subject = subject - self.mail_address = mail_address - self.content = content + self.report_values = report_values + self.report_template = report_template self.compress = compress - if wait_on is not None: - if isinstance(wait_on, tk.AbstractPath): - self.add_input(wait_on) - elif isinstance(wait_on, Iterable): - for wait in wait_on: - self.add_input(wait) - else: - assert False, "Wrong wait_on input!" - - self.out_content = self.output_path("content") + self.out_report = self.output_path("report.gz" if self.compress else "report") def tasks(self): yield Task("run", mini_task=True) def run(self): - if self.compress: - with gzip.open(self.out_content.get_path(), "w") as f: - f.write(self.content.encode() + b"\n") + if self.report_template: + report = self.report_template(self.report_values) + elif callable(self.report_values): + report = str(self.report_values()) else: - with open(self.out_content.get_path(), "w") as f: - f.write(self.content) + report = pprint.pformat(self.report_values, width=140) - self.sh("zcat -f {out_content} | mail -s '{subject}' {mail_address}") + if self.compress: + with gzip.open(self.out_report.get_path(), "wt") as f: + f.write(report) + else: + with open(self.out_report.get_path(), "wt") as f: + f.write(report) -class ReportResultsJob(Job): +class MailJob(Job): """ - Job to report results either via mail and/or just condensed in a file + Job that sends a mail upon completion of an output """ def __init__( self, - name: str, - report: _Report_Type, - report_format: Callable[[_Report_Type, any], str], - compress: bool = True, - **report_format_kwargs, + result: tk.AbstractPath, + subject: Optional[str] = None, + mail_address: str = getpass.getuser(), + send_contents: bool = False, ): """ - :param name: Name of the report - :param report: Dictionary containing the report files - :param report_format: Function that converts the report dictionary to a string - :param report_format_kwargs: Addtional kwargs for the report format function - :param compress: Whether to compress the report or not + :param result: graph output that triggers sending the mail + :param subject: Subject of the mail + :param mail_address: Mail address of recipient (default: user) + :param send_contents: send the contents of result in body of the mail """ - self.name = name - self.report = report - self.report_format = report_format - self.report_format_kwargs = report_format_kwargs - self.compress = compress - - self.sis_command_line = sys.argv - self.mail_address = getattr(gs, "MAIL_ADDRESS", None) - self.config_path = tk.config_manager.current_config - assert self.config_path is not None, "Could not find config path" + self.result = result + self.subject = subject + self.mail_address = mail_address + self.send_contents = send_contents - for i in ["date", "user", "name", "config_path", "sis_command_line"]: - assert i not in report, "%s will be set automatically" + self.out_status = self.output_var("out_status") - self.out_report = self.output_path("report.gz") + def tasks(self): + yield Task("run", mini_task=True) def run(self): - user = getpass.getuser() - report = self.report.copy() - report["date"] = datetime.now().strftime("%d/%m/%Y %H:%M:%S") - report["user"] = user - report["name"] = self.name - report["config_path"] = self.config_path - report["sis_command_line"] = str(self.sis_command_line) - - if self.compress: - with gzip.open(self.out_report.get_path(), "w") as f: - f.write( - self.report_format(report, **self.report_format_kwargs).encode() - + b"\n" - ) + if self.subject is None: + subject = f"Output {str(self.result)} is finished" else: - with open(self.out_report.get_path(), "w") as f: - f.write(self.report_format(report, **self.report_format_kwargs) + "\n") + subject = self.subject - if self.mail_address: - self.sh( - "zcat -f {out_report} | mail -s 'Report finished: {name} {config_path}' {mail_address}" + if self.send_contents: + value = self.sh( + f"zcat -f {self.result.get_path()} | mail -s '{subject}' {self.mail_address}" + ) + else: + value = self.sh( + f"echo '{subject}' | mail -s '{subject}' {self.mail_address}" ) - def tasks(self): - yield Task("run", mini_task=True) + self.out_status.set(value) def gmm_example_report_format(report: _Report_Type) -> str: From 7db11e1287d396958b009d9be61d548b6f202b71 Mon Sep 17 00:00:00 2001 From: Benedikt Hilmes Date: Tue, 27 Sep 2022 19:58:53 +0200 Subject: [PATCH 13/22] remove example --- report/report.py | 104 ----------------------------------------------- 1 file changed, 104 deletions(-) diff --git a/report/report.py b/report/report.py index ffddc6d3..7abebf87 100644 --- a/report/report.py +++ b/report/report.py @@ -1,6 +1,5 @@ from sisyphus import * -import sys import getpass import gzip from typing import Dict, Union, Callable, Optional @@ -98,106 +97,3 @@ def run(self): ) self.out_status.set(value) - - -def gmm_example_report_format(report: _Report_Type) -> str: - """ - Example report format for a GMM evaluated on dev-clean and dev-other - :param report: - :return: - """ - results = { - "dev-clean": { - "Monophone": {}, - "Triphone": {}, - "SAT": {}, - "VTLN": {}, - "VTLN+SAT": {}, - }, - "dev-other": { - "Monophone": {}, - "Triphone": {}, - "SAT": {}, - "VTLN": {}, - "VTLN+SAT": {}, - }, - } - for step_name, score in report.items(): - if not step_name.startswith("scorer"): - continue - if "dev-clean" in step_name: - set = "dev-clean" - else: - set = "dev-other" - if "mono" in step_name: - step = "Monophone" - elif "tri" in step_name: - step = "Triphone" - elif "vtln+sat" in step_name: - step = "VTLN+SAT" - elif "sat" in step_name: - step = "SAT" - else: - step = "VTLN" - if "iter08" in step_name: - results[set][step]["08"] = score - elif "iter10" in step_name: - results[set][step]["10"] = score - - out = [] - out.append( - f"""Name: {report["name"]} - Path: {report["config_path"]} - Date: {report["date"]} - Sis Command: {report["sis_command_line"]} - - Results:""" - ) - out.append("Step".ljust(20) + "dev-clean".ljust(10) + "dev-other") - out.append( - "Monophone".ljust(16) - + str(results["dev-clean"]["Monophone"]["10"]).ljust(14) - + str(results["dev-other"]["Monophone"]["10"]) - ) - out.append( - "Triphone 08".ljust(19) - + str(results["dev-clean"]["Triphone"]["08"]).ljust(14) - + str(results["dev-other"]["Triphone"]["08"]) - ) - out.append( - "Triphone 10".ljust(19) - + str(results["dev-clean"]["Triphone"]["10"]).ljust(14) - + str(results["dev-other"]["Triphone"]["10"]) - ) - out.append( - "VTLN 08".ljust(21) - + str(results["dev-clean"]["VTLN"]["08"]).ljust(14) - + str(results["dev-other"]["VTLN"]["08"]) - ) - out.append( - "VTLN 10".ljust(21) - + str(results["dev-clean"]["VTLN"]["10"]).ljust(14) - + str(results["dev-other"]["VTLN"]["10"]) - ) - out.append( - "SAT 08".ljust(23) - + str(results["dev-clean"]["SAT"]["08"]).ljust(14) - + str(results["dev-other"]["SAT"]["08"]) - ) - out.append( - "SAT 10".ljust(23) - + str(results["dev-clean"]["SAT"]["10"]).ljust(14) - + str(results["dev-other"]["SAT"]["10"]) - ) - out.append( - "VTLN+SAT 08".ljust(17) - + str(results["dev-clean"]["VTLN+SAT"]["08"]).ljust(14) - + str(results["dev-other"]["VTLN+SAT"]["08"]) - ) - out.append( - "VTLN+SAT 10".ljust(17) - + str(results["dev-clean"]["VTLN+SAT"]["10"]).ljust(14) - + str(results["dev-other"]["VTLN+SAT"]["10"]) - ) - - return "\n".join(out) From fcd0bd9dc6b334595499d9b0725bc0767b00279d Mon Sep 17 00:00:00 2001 From: Benedikt Hilmes Date: Fri, 30 Sep 2022 13:13:42 +0200 Subject: [PATCH 14/22] Update report/report.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Christoph M. Lüscher --- report/report.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/report/report.py b/report/report.py index 7abebf87..2b1db1eb 100644 --- a/report/report.py +++ b/report/report.py @@ -29,7 +29,7 @@ def __init__( self.report_template = report_template self.compress = compress - self.out_report = self.output_path("report.gz" if self.compress else "report") + self.out_report = self.output_path("report.txt.gz" if self.compress else "report.txt") def tasks(self): yield Task("run", mini_task=True) From 304e7f9eb6506c0caa3466ef870e1cd3d70db5b0 Mon Sep 17 00:00:00 2001 From: Benedikt Hilmes Date: Fri, 30 Sep 2022 13:16:21 +0200 Subject: [PATCH 15/22] updates --- report/report.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/report/report.py b/report/report.py index 2b1db1eb..e56759c0 100644 --- a/report/report.py +++ b/report/report.py @@ -15,7 +15,7 @@ class GenerateReportStringJob(Job): def __init__( self, - report_values: _Report_Type, + report_values: Union[_Report_Type, Callable], report_template: Optional[Callable[[_Report_Type], str]] = None, compress: bool = True, ): @@ -29,7 +29,9 @@ def __init__( self.report_template = report_template self.compress = compress - self.out_report = self.output_path("report.txt.gz" if self.compress else "report.txt") + self.out_report = self.output_path( + "report.txt.gz" if self.compress else "report.txt" + ) def tasks(self): yield Task("run", mini_task=True) @@ -43,12 +45,9 @@ def run(self): else: report = pprint.pformat(self.report_values, width=140) - if self.compress: - with gzip.open(self.out_report.get_path(), "wt") as f: - f.write(report) - else: - with open(self.out_report.get_path(), "wt") as f: - f.write(report) + fopen = gzip.open if self.compress else open + with fopen(self.out_report.get_path(), "wt") as f: + f.write(report) class MailJob(Job): From bdd858d9f7745ed518f559b7bd831c0c256bbb77 Mon Sep 17 00:00:00 2001 From: Benedikt Hilmes Date: Tue, 4 Oct 2022 10:58:11 +0200 Subject: [PATCH 16/22] add support for format string --- report/report.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/report/report.py b/report/report.py index e56759c0..a52e22b4 100644 --- a/report/report.py +++ b/report/report.py @@ -16,7 +16,7 @@ class GenerateReportStringJob(Job): def __init__( self, report_values: Union[_Report_Type, Callable], - report_template: Optional[Callable[[_Report_Type], str]] = None, + report_template: Optional[Union[Callable[[_Report_Type], str], str]] = None, compress: bool = True, ): """ @@ -39,7 +39,10 @@ def tasks(self): def run(self): if self.report_template: - report = self.report_template(self.report_values) + if isinstance(self.report_template, str): + report = self.report_template.format(**self.report_values) + else: + report = self.report_template(self.report_values) elif callable(self.report_values): report = str(self.report_values()) else: From 9bf08f576be92c2dad0b571c88fae4806bbf6ae9 Mon Sep 17 00:00:00 2001 From: Benedikt Hilmes Date: Sat, 8 Oct 2022 10:00:40 +0200 Subject: [PATCH 17/22] comments --- report/report.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/report/report.py b/report/report.py index a52e22b4..61569fd9 100644 --- a/report/report.py +++ b/report/report.py @@ -4,6 +4,7 @@ import gzip from typing import Dict, Union, Callable, Optional import pprint +import subprocess _Report_Type = Dict[str, Union[tk.AbstractPath, str]] @@ -90,12 +91,8 @@ def run(self): subject = self.subject if self.send_contents: - value = self.sh( - f"zcat -f {self.result.get_path()} | mail -s '{subject}' {self.mail_address}" - ) + p1 = subprocess.Popen(["zcat", "-f", self.result.get_path()], stdout=subprocess.PIPE) + value = subprocess.check_output(["mail", "-s", subject, self.mail_address], stdin=p1.stdout) else: - value = self.sh( - f"echo '{subject}' | mail -s '{subject}' {self.mail_address}" - ) - + value = subprocess.check_output(["mail", "-s", subject, self.mail_address]) self.out_status.set(value) From d067b5d032899e0fcc0811fcad22ae59c06ae14c Mon Sep 17 00:00:00 2001 From: Benedikt Hilmes Date: Mon, 10 Oct 2022 10:21:31 +0200 Subject: [PATCH 18/22] all + black --- report/report.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/report/report.py b/report/report.py index 61569fd9..8aea6178 100644 --- a/report/report.py +++ b/report/report.py @@ -1,3 +1,4 @@ +__all__ = ["GenerateReportStringJob", "MailJob"] from sisyphus import * import getpass @@ -91,8 +92,14 @@ def run(self): subject = self.subject if self.send_contents: - p1 = subprocess.Popen(["zcat", "-f", self.result.get_path()], stdout=subprocess.PIPE) - value = subprocess.check_output(["mail", "-s", subject, self.mail_address], stdin=p1.stdout) + p1 = subprocess.Popen( + ["zcat", "-f", self.result.get_path()], stdout=subprocess.PIPE + ) + value = subprocess.check_output( + ["mail", "-s", subject, self.mail_address], stdin=p1.stdout + ) else: - value = subprocess.check_output(["mail", "-s", subject, self.mail_address]) + value = subprocess.check_output( + ["mail", "-s", subject, self.mail_address], stdin=[""] + ) self.out_status.set(value) From 8ba2b14fa82578c774b62b50e5c817f4a2cb2efe Mon Sep 17 00:00:00 2001 From: Benedikt Hilmes Date: Mon, 10 Oct 2022 10:25:50 +0200 Subject: [PATCH 19/22] update --- report/report.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/report/report.py b/report/report.py index 8aea6178..166d7fda 100644 --- a/report/report.py +++ b/report/report.py @@ -99,7 +99,8 @@ def run(self): ["mail", "-s", subject, self.mail_address], stdin=p1.stdout ) else: + p1 = subprocess.run(stdin="", stdout=subprocess.PIPE) value = subprocess.check_output( - ["mail", "-s", subject, self.mail_address], stdin=[""] + ["mail", "-s", subject, self.mail_address], stdin=p1.stdout ) self.out_status.set(value) From da94a6d8b6bfdc1f33e1dfa89964480a9ef25523 Mon Sep 17 00:00:00 2001 From: Benedikt Hilmes Date: Wed, 19 Oct 2022 12:22:29 +0200 Subject: [PATCH 20/22] now with run --- report/report.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/report/report.py b/report/report.py index 166d7fda..57e73a80 100644 --- a/report/report.py +++ b/report/report.py @@ -99,8 +99,6 @@ def run(self): ["mail", "-s", subject, self.mail_address], stdin=p1.stdout ) else: - p1 = subprocess.run(stdin="", stdout=subprocess.PIPE) - value = subprocess.check_output( - ["mail", "-s", subject, self.mail_address], stdin=p1.stdout - ) + out = subprocess.run(["mail", "-s", subject, self.mail_address], input="", check=True) + value = out.returncode self.out_status.set(value) From 3b0e7cd494afe18d0b74e6a4e6acd76d2da9a09a Mon Sep 17 00:00:00 2001 From: Benedikt Hilmes Date: Wed, 19 Oct 2022 14:10:45 +0200 Subject: [PATCH 21/22] coffee --- report/report.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/report/report.py b/report/report.py index 57e73a80..e49a1bda 100644 --- a/report/report.py +++ b/report/report.py @@ -99,6 +99,8 @@ def run(self): ["mail", "-s", subject, self.mail_address], stdin=p1.stdout ) else: - out = subprocess.run(["mail", "-s", subject, self.mail_address], input="", check=True) + out = subprocess.run( + ["mail", "-s", subject, self.mail_address], input="", check=True + ) value = out.returncode self.out_status.set(value) From 53930a77396dc11ef56042b4760b772ad584c554 Mon Sep 17 00:00:00 2001 From: Benedikt Hilmes Date: Thu, 20 Oct 2022 11:51:07 +0200 Subject: [PATCH 22/22] doc, assert, content --- report/report.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/report/report.py b/report/report.py index e49a1bda..155cece4 100644 --- a/report/report.py +++ b/report/report.py @@ -7,6 +7,8 @@ import pprint import subprocess +from i6_core.util import uopen + _Report_Type = Dict[str, Union[tk.AbstractPath, str]] @@ -23,10 +25,12 @@ def __init__( ): """ - :param report_values: - :param report_template: - :param compress: + :param report_values: Can be either directly callable or a dict which then is handled by report_template + :param report_template: Function to handle report_values of type _Report_Type + :param compress: Whether to zip the report """ + if report_template is not None: + assert not isinstance(report_values, Callable) self.report_values = report_values self.report_template = report_template self.compress = compress @@ -50,8 +54,7 @@ def run(self): else: report = pprint.pformat(self.report_values, width=140) - fopen = gzip.open if self.compress else open - with fopen(self.out_report.get_path(), "wt") as f: + with uopen(self.out_report.get_path(), "wt") as f: f.write(report) @@ -100,7 +103,7 @@ def run(self): ) else: out = subprocess.run( - ["mail", "-s", subject, self.mail_address], input="", check=True + ["mail", "-s", subject, self.mail_address], input=subject, check=True ) value = out.returncode self.out_status.set(value)