Skip to content

Commit

Permalink
cleanup
Browse files Browse the repository at this point in the history
  • Loading branch information
stavros-k committed Oct 14, 2024
1 parent 817b4fb commit aa68f71
Show file tree
Hide file tree
Showing 6 changed files with 175 additions and 115 deletions.
4 changes: 2 additions & 2 deletions library/2.0.0/container.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
from .ports import Ports
from .restart import RestartPolicy
from .validations import valid_network_mode_or_raise, valid_cap_or_raise
from .volume_mount import VolumeMounts
from .volume_mounts import VolumeMounts
except ImportError:
from depends import Depends
from deploy import Deploy
Expand All @@ -30,7 +30,7 @@
from ports import Ports
from restart import RestartPolicy
from validations import valid_network_mode_or_raise, valid_cap_or_raise
from volume_mount import VolumeMounts
from volume_mounts import VolumeMounts


class Container:
Expand Down
7 changes: 7 additions & 0 deletions library/2.0.0/formatter.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,3 +15,10 @@ def escape_dollar(text):
def get_hashed_name_for_volume(prefix: str, config: dict):
config_hash = hashlib.sha256(json.dumps(config).encode("utf-8")).hexdigest()
return f"{prefix}_{config_hash}"


def merge_dicts_no_overwrite(dict1, dict2):
overlapping_keys = dict1.keys() & dict2.keys()
if overlapping_keys:
raise ValueError(f"Merging of dicts failed. Overlapping keys: {overlapping_keys}")
return {**dict1, **dict2}
17 changes: 5 additions & 12 deletions library/2.0.0/render.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,16 +6,14 @@
from .functions import Functions
from .notes import Notes
from .portals import Portals

# from .volumes import Volumes
from .volumes import Volumes
except ImportError:
from container import Container
from error import RenderError
from functions import Functions
from notes import Notes
from portals import Portals

# from volumes import Volumes
from volumes import Volumes


class Render(object):
Expand All @@ -27,8 +25,7 @@ def __init__(self, values):
self.funcs = Functions(render_instance=self).func_map()
self.portals: Portals = Portals(render_instance=self)
self.notes: Notes = Notes(render_instance=self)
# self.volumes = Volumes(render_instance=self)
self.new_volumes: dict = {}
self.volumes = Volumes(render_instance=self)

# self.networks = {}

Expand All @@ -55,12 +52,8 @@ def render(self):
"services": {c._name: c.render() for c in self._containers.values()},
}

# self.volumes.check_volumes()
# if self.volumes.has_volumes():
# result["volumes"] = self.volumes.render()
if self.new_volumes:
# TODO: add a method to "add" volumes
result["volumes"] = self.new_volumes
if self.volumes.has_volumes():
result["volumes"] = self.volumes.render()

# if self.networks:
# result["networks"] = {...}
Expand Down
116 changes: 15 additions & 101 deletions library/2.0.0/volume_mount.py → library/2.0.0/storage_types.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,108 +15,16 @@
from validations import valid_fs_path_or_raise


def merge_dicts_no_overwrite(dict1, dict2):
overlapping_keys = dict1.keys() & dict2.keys()
if overlapping_keys:
raise ValueError(f"Merging of dicts failed. Overlapping keys: {overlapping_keys}")
return {**dict1, **dict2}


class VolumeMounts:
def __init__(self, render_instance: "Render"):
self._render_instance = render_instance
self._volume_mounts: set[VolumeMount] = set()

def add_volume_mount(self, mount_path: str, config: dict):
mount_path = valid_fs_path_or_raise(mount_path)
if mount_path in [m.mount_path for m in self._volume_mounts]:
raise RenderError(f"Mount path [{mount_path}] already used for another volume mount")

volume_mount = VolumeMount(self._render_instance, mount_path, config)
self._volume_mounts.add(volume_mount)

def has_mounts(self) -> bool:
return bool(self._volume_mounts)

def render(self):
return [vm.render() for vm in sorted(self._volume_mounts, key=lambda vm: vm.mount_path)]


class VolumeMount:
# TODO: create a type for config
def __init__(self, render_instance: "Render", mount_path: str, config: dict):
self._render_instance = render_instance
self.mount_path: str = mount_path
self.read_only: bool = config.get("read_only", False)

self.source: str | None = None # tmpfs does not have a source
self.spec_type: str = ""

self.spec_config_for_type: dict = {}

self.volume_mount_spec: dict = {}

_type_spec_mapping = {
"host_path": {"class": HostPathIxStorage, "spec_type": "bind"},
"ix_volume": {"class": IxVolumeIxStorage, "spec_type": "bind"},
"tmpfs": {"class": TmpfsIxStorage, "spec_type": "tmpfs"},
"cifs": {"class": CifsIxStorage, "spec_type": "volume"},
"nfs": {"class": NfsIxStorage, "spec_type": "volume"},
# TODO: anonymous/temporary volumes
}

vol_type = config.get("type", None)
if vol_type not in _type_spec_mapping:
valid_types = ", ".join(_type_spec_mapping.keys())
raise RenderError(f"Volume type [{vol_type}] is not valid. Valid options are: [{valid_types}]")

# Fetch the class for the type
vol_spec_class = _type_spec_mapping[vol_type]["class"]

# Set the type in the mount type for the spec
self.spec_type = _type_spec_mapping[vol_type]["spec_type"]

# Parse the config for the type
storage_item = vol_spec_class(self._render_instance, config).render()

# Make sure source is not empty for all but tmpfs
if self.spec_type != "tmpfs":
if not storage_item.source:
raise RenderError(f"Missing source for volume type [{vol_type}]")

# Fetch the source (path for bind, volume name for volume)
self.source = storage_item.source
# Fetch the type specific config for the mount spec
self.spec_config_for_type = storage_item.mount_spec

if self.spec_type == "volume":
# TODO: self._render_instance.add_volume...
self._render_instance.new_volumes[self.source] = storage_item.volume_spec

def render(self) -> dict:
result = {
"type": self.spec_type,
"target": self.mount_path,
"read_only": self.read_only,
}

if self.source is not None:
result["source"] = self.source

if self.spec_config_for_type:
result = merge_dicts_no_overwrite(result, self.spec_config_for_type)

return result


class StorageItemResult:
def __init__(self, source: str | None, volume_spec: dict | None, mount_spec: dict):
self.source: str | None = source
self.volume_spec: dict | None = volume_spec
self.mount_spec: dict = mount_spec


class TmpfsIxStorage:
class TmpfsStorage:
"""Parses storage item with type tmpfs."""

def __init__(self, render_instance: "Render", config: dict):
self._render_instance = render_instance
tmpfs_config = config.get("tmpfs_config", {})
Expand All @@ -126,7 +34,9 @@ def render(self):
return StorageItemResult(source=None, volume_spec=None, mount_spec=self.mount_spec)


class HostPathIxStorage:
class HostPathStorage:
"""Parses storage item with type host_path."""

def __init__(self, render_instance: "Render", config: dict):
self._render_instance = render_instance
host_path_config = config.get("host_path_config", {})
Expand All @@ -150,7 +60,9 @@ def render(self):
return StorageItemResult(source=self.source, volume_spec=None, mount_spec=self.mount_spec)


class IxVolumeIxStorage:
class IxVolumeStorage:
"""Parses storage item with type ix_volume."""

def __init__(self, render_instance: "Render", config: dict):
self._render_instance = render_instance
ix_volume_config = config.get("ix_volume_config", {})
Expand Down Expand Up @@ -179,7 +91,9 @@ def render(self):
return StorageItemResult(source=self.source, volume_spec=None, mount_spec=self.mount_spec)


class CifsIxStorage:
class CifsStorage:
"""Parses storage item with type cifs."""

def __init__(self, render_instance: "Render", config: dict):
self._render_instance = render_instance
cifs_config = config.get("cifs_config", {})
Expand Down Expand Up @@ -220,8 +134,6 @@ def __init__(self, render_instance: "Render", config: dict):
path = cifs_config["path"].strip("/")
path = valid_fs_path_or_raise("/" + path).lstrip("/")

# TODO: probably include the target here as well.
# Reason is: what if we want multiple cifs with the same config, but mounted on different targets?
self.source = get_hashed_name_for_volume("cifs", cifs_config)
self.volume_spec = {
"driver_opts": {
Expand All @@ -235,7 +147,9 @@ def render(self):
return StorageItemResult(source=self.source, volume_spec=self.volume_spec, mount_spec=self.mount_spec)


class NfsIxStorage:
class NfsStorage:
"""Parses storage item with type nfs."""

def __init__(self, render_instance: "Render", config: dict):
self._render_instance = render_instance
nfs_config = config.get("nfs_config", {})
Expand Down
114 changes: 114 additions & 0 deletions library/2.0.0/volume_mounts.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,114 @@
from typing import TYPE_CHECKING

if TYPE_CHECKING:
from render import Render

try:
from .error import RenderError
from .formatter import merge_dicts_no_overwrite
from .validations import valid_fs_path_or_raise
from .storage_types import (
StorageItemResult,
TmpfsStorage,
HostPathStorage,
IxVolumeStorage,
CifsStorage,
NfsStorage,
)
except ImportError:
from error import RenderError
from formatter import merge_dicts_no_overwrite
from validations import valid_fs_path_or_raise
from storage_types import (
StorageItemResult,
TmpfsStorage,
HostPathStorage,
IxVolumeStorage,
CifsStorage,
NfsStorage,
)


class VolumeMounts:
def __init__(self, render_instance: "Render"):
self._render_instance = render_instance
self._volume_mounts: set[VolumeMount] = set()

def add_volume_mount(self, mount_path: str, config: dict):
mount_path = valid_fs_path_or_raise(mount_path)
if mount_path in [m.mount_path for m in self._volume_mounts]:
raise RenderError(f"Mount path [{mount_path}] already used for another volume mount")

volume_mount = VolumeMount(self._render_instance, mount_path, config)
self._volume_mounts.add(volume_mount)

def has_mounts(self) -> bool:
return bool(self._volume_mounts)

def render(self):
return [vm.render() for vm in sorted(self._volume_mounts, key=lambda vm: vm.mount_path)]


class VolumeMount:
# TODO: create a type for config
def __init__(self, render_instance: "Render", mount_path: str, config: dict):
self._render_instance = render_instance
self.mount_path: str = mount_path
self.read_only: bool = config.get("read_only", False)

self.source: str | None = None # tmpfs does not have a source
self.spec_type: str = ""

self.spec_config_for_type: dict = {}

self.volume_mount_spec: dict = {}

_type_spec_mapping = {
"host_path": {"class": HostPathStorage, "spec_type": "bind"},
"ix_volume": {"class": IxVolumeStorage, "spec_type": "bind"},
"tmpfs": {"class": TmpfsStorage, "spec_type": "tmpfs"},
"cifs": {"class": CifsStorage, "spec_type": "volume"},
"nfs": {"class": NfsStorage, "spec_type": "volume"},
# TODO: anonymous/temporary volumes
}

vol_type = config.get("type", None)
if vol_type not in _type_spec_mapping:
valid_types = ", ".join(_type_spec_mapping.keys())
raise RenderError(f"Volume type [{vol_type}] is not valid. Valid options are: [{valid_types}]")

# Parse the config for the type
storage_item: StorageItemResult = _type_spec_mapping[vol_type]["class"](self._render_instance, config).render()

# Set the type in the mount type for the spec
self.spec_type = _type_spec_mapping[vol_type]["spec_type"]

# Make sure source is not empty for all but tmpfs
if self.spec_type != "tmpfs":
if not storage_item.source:
raise RenderError(f"Missing source for volume type [{vol_type}]")

# Fetch the source (path for bind, volume name for volume)
self.source = storage_item.source
# Fetch the type specific config for the mount spec
self.spec_config_for_type = storage_item.mount_spec

if self.spec_type == "volume":
assert self.source is not None
assert storage_item.volume_spec is not None
self._render_instance.volumes.add_volume(self.source, storage_item.volume_spec)

def render(self) -> dict:
result = {
"type": self.spec_type,
"target": self.mount_path,
"read_only": self.read_only,
}

if self.source is not None:
result["source"] = self.source

if self.spec_config_for_type:
result = merge_dicts_no_overwrite(result, self.spec_config_for_type)

return result
32 changes: 32 additions & 0 deletions library/2.0.0/volumes.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
from typing import TYPE_CHECKING

if TYPE_CHECKING:
from render import Render


class Volumes:
def __init__(self, render_instance: "Render"):
self._render_instance = render_instance
self._volumes: dict[str, Volume] = {}

def add_volume(self, name: str, config: dict):
# This method can be called many times from the volume mounts
# Only add the volume if it is not already added, but dont raise an error
if name in self._volumes:
return
self._volumes[name] = Volume(self._render_instance, config)

def has_volumes(self) -> bool:
return bool(self._volumes)

def render(self):
return {name: v.render() for name, v in sorted(self._volumes.items())}


class Volume:
def __init__(self, render_instance: "Render", config: dict):
self._render_instance = render_instance
self._config = config

def render(self):
return self._config

0 comments on commit aa68f71

Please sign in to comment.