Backend branch (#58)

* WIP: Tensorflow MNIST use-case

* UPDATE: Tensorflow MNIST version

* ADD: Backend

* ADD: Use-case init

* FIX: Paths and downloading of the data

* FIX: Paths and downloading of the data

* ADD: Setup, Config update

* ADD: Setup, Config update

* UPDATE: File movement into itwinai

* FIX: Move utils from tensorflow to global folder

* FIX: Add setup into torch Executable

* ADD: MNIST Torch Use-case

* FIX: Formatting

CODEOWNERS

@@ -7,4 +7,3 @@
 # These owners will be the default owners for everything in the repo.
 # Unless a later match takes precedence, they will be requested for
 # review when someone opens a pull request.
-

ai/setup.py

@@ -8,13 +8,7 @@
     version="0.1",
     packages=find_packages("src"),
     package_dir={"": "src"},
-    entry_points={
-        "console_scripts": [
-            "itwinai=itwinai.cli:app"
-        ]
-    },
+    entry_points={"console_scripts": ["itwinai=itwinai.cli:app"]},
     # Pip dependencies
-    install_requires=[
-        'jsonargparse[signatures]>=4.17.0'
-    ]
+    install_requires=["jsonargparse[signatures]>=4.17.0"],
 )

ai/src/itwinai/backend/components.py

@@ -0,0 +1,63 @@
+from abc import ABCMeta, abstractmethod
+
+
+class Executable(metaclass=ABCMeta):
+    @abstractmethod
+    def execute(self, args):
+        pass
+
+    @abstractmethod
+    def setup(self, args):
+        pass
+
+
+class Trainer(Executable):
+    @abstractmethod
+    def train(self, data):
+        pass
+
+
+class DataGetter(Executable):
+    @abstractmethod
+    def load(self, args):
+        pass
+
+
+class DataPreproc(Executable):
+    @abstractmethod
+    def preproc(self, args):
+        pass
+
+
+class StatGetter(Executable):
+    @abstractmethod
+    def stats(self, args):
+        pass
+
+
+class Evaluator(Executable):
+    @abstractmethod
+    def evaluate(self, args):
+        pass
+
+
+class Saver(Executable):
+    @abstractmethod
+    def save(self, args):
+        pass
+
+
+class Executor(Executable):
+    @abstractmethod
+    def execute(self, pipeline):
+        pass
+
+    @abstractmethod
+    def setup(self, pipeline):
+        pass
+
+
+class Logger(metaclass=ABCMeta):
+    @abstractmethod
+    def log(self):
+        pass

ai/src/itwinai/backend/tensorflow/executor.py

@@ -0,0 +1,15 @@
+from ..components import Executor
+
+
+class TensorflowExecutor(Executor):
+    def __init__(self, args):
+        self.args = args
+
+    def execute(self, pipeline):
+        args = None
+        for executable in pipeline:
+            args = executable.execute(args)
+
+    def setup(self, pipeline):
+        for executable in pipeline:
+            executable.setup(self.args)

ai/src/itwinai/backend/tensorflow/loggers.py

@@ -0,0 +1,22 @@
+import wandb
+import mlflow
+import mlflow.keras
+
+from ..components import Logger
+
+
+class WanDBLogger(Logger):
+    def __init__(self):
+        pass
+
+    def log(self):
+        wandb.init(config={"bs": 12})
+
+
+class MLFlowLogger(Logger):
+    def __init__(self):
+        mlflow.set_tracking_uri("http://127.0.0.1:5000")
+        mlflow.set_experiment("test-experiment")
+
+    def log(self):
+        mlflow.keras.autolog()

ai/src/itwinai/backend/tensorflow/utils.py

@@ -0,0 +1,13 @@
+import keras
+import json
+
+
+def model_to_json(model: keras.Model, filepath: str):
+    with open(filepath, "w") as f:
+        json.dump(model.to_json(), f)
+
+
+def model_from_json(filepath: str) -> keras.Model:
+    with open(filepath, "r") as f:
+        config = json.load(f)
+        return keras.models.model_from_json(config)

ai/src/itwinai/backend/torch/executor.py

@@ -0,0 +1,16 @@
+from ..components import Executor
+
+
+class TorchExecutor(Executor):
+    def __init__(self):
+        pass
+
+    def execute(self, pipeline):
+        args = None
+        for executable in pipeline:
+            args = executable.execute(args)
+
+    def setup(self, pipeline):
+        args = None
+        for executable in pipeline:
+            args = executable.setup(args)

ai/src/itwinai/backend/torch/loggers.py

@@ -0,0 +1,22 @@
+import wandb
+import mlflow
+import mlflow.keras
+
+from ..components import Logger
+
+
+class WanDBLogger(Logger):
+    def __init__(self):
+        pass
+
+    def log(self):
+        wandb.init(config={"bs": 12})
+
+
+class MLFlowLogger(Logger):
+    def __init__(self):
+        mlflow.set_tracking_uri("http://127.0.0.1:5000")
+        mlflow.set_experiment("test-experiment")
+
+    def log(self):
+        mlflow.pytorch.autolog()

ai/src/itwinai/backend/utils.py

@@ -0,0 +1,13 @@
+import yaml
+
+
+# Parse (part of) YAML loaded in memory
+def parse_pipe_config(yaml_file, parser):
+    with open(yaml_file, "r", encoding="utf-8") as f:
+        try:
+            config = yaml.safe_load(f)
+        except yaml.YAMLError as exc:
+            print(exc)
+            raise exc
+
+    return parser.parse_object(config)