From 510c96d5a26836a313aece93c621dea9b66ab28e Mon Sep 17 00:00:00 2001 From: dcvan Date: Tue, 20 Mar 2018 11:09:06 -0400 Subject: [PATCH] [#7] collect port availability from cluster; [#3][#7] added methods to validate user input --- appliance/base.py | 15 +++++++++++ appliance/handler.py | 18 ++++++++++--- cluster/base.py | 62 +++++++++++++++++++++++++++++++++++++++++--- container/base.py | 11 ++++++++ container/handler.py | 12 --------- 5 files changed, 99 insertions(+), 19 deletions(-) diff --git a/appliance/base.py b/appliance/base.py index d8bd88c..b0a9254 100644 --- a/appliance/base.py +++ b/appliance/base.py @@ -4,11 +4,26 @@ class Appliance: + REQUIRED = frozenset(['id', 'containers']) + def __init__(self, id, containers=[], **kwargs): self.__id = id self.__containers = list(containers) self.__dag = ContainerDAG() + @classmethod + def pre_check(cls, data): + if not isinstance(data, dict): + return 422, None, "Failed to parse appliance request format: %s"%type(data) + missing = Appliance.REQUIRED - data.keys() + if missing: + return 400, None, "Missing required field(s) of appliance: %s"%missing + for c in data['containers']: + status, _, err = Container.pre_check(c) + if err: + return status, None, err + return 200, "Appliance %s is valid" % data['id'], None + @property def id(self): return self.__id diff --git a/appliance/handler.py b/appliance/handler.py index 8f71b9f..a8de869 100644 --- a/appliance/handler.py +++ b/appliance/handler.py @@ -3,6 +3,7 @@ from tornado.web import RequestHandler +from appliance.base import Appliance from appliance.manager import ApplianceManager from container.manager import ContainerManager from util import message, error @@ -16,10 +17,19 @@ def initialize(self, config): self.__contr_mgr = ContainerManager(config) async def post(self): - data = tornado.escape.json_decode(self.request.body) - status, app, err = await self.__app_mgr.create_appliance(data) - self.set_status(status) - self.write(json.dumps(app.to_render() if status == 201 else error(err))) + try: + data = tornado.escape.json_decode(self.request.body) + status, app, err = Appliance.pre_check(data) + if status != 200: + self.set_status(status) + self.write(error(err)) + return + status, app, err = await self.__app_mgr.create_appliance(data) + self.set_status(status) + self.write(json.dumps(app.to_render() if status == 201 else error(err))) + except json.JSONDecodeError as e: + self.set_status(422) + self.write(error("Ill-formatted request: %s"%e)) class ApplianceHandler(RequestHandler, Loggable): diff --git a/cluster/base.py b/cluster/base.py index 8819cbc..07d54ee 100644 --- a/cluster/base.py +++ b/cluster/base.py @@ -1,3 +1,5 @@ +import bisect + from collections import defaultdict from tornado.ioloop import PeriodicCallback @@ -6,19 +8,67 @@ class Host: - def __init__(self, hostname, attributes={}): + def __init__(self, hostname, resources, attributes={}): + self.__resources = resources self.__attributes = dict(**attributes, hostname=hostname) @property def hostname(self): return self.__attributes.get('hostname', None) + @property + def resources(self): + return self.__resources + @property def attributes(self): return dict(self.__attributes) def to_render(self): - return self.attributes + return dict(attributes=self.attributes, resources=self.resources.to_render()) + + +class Resources: + + def __init__(self, cpus, mem, disk, gpus, port_ranges): + self.__cpus = cpus + self.__mem = mem + self.__disk = disk + self.__gpus = gpus + self.__port_ranges = [tuple(map(lambda p: int(p), p.split('-'))) for p in port_ranges] + + @property + def cpus(self): + return self.__cpus + + @property + def mem(self): + return self.__mem + + @property + def disk(self): + return self.__disk + + @property + def gpus(self): + return self.__gpus + + @property + def port_ranges(self): + return self.__port_ranges + + def check_port_availability(self, p): + assert isinstance(p, int) + starts = [p[0] for p in self.port_ranges] + idx = bisect.bisect(starts, p, 0, len(starts)) + if idx == 0: + return False + port_range = self.port_ranges[idx - 1] + return port_range[0] <= p <= port_range[1] + + def to_render(self): + return dict(cpus=self.cpus, mem=self.mem, disk=self.disk, gpus=self.gpus, + port_ranges=['%d-%d'%p for p in self.port_ranges]) class Cluster(Loggable, metaclass=Singleton): @@ -45,7 +95,13 @@ async def query_mesos(): self.logger.debug(err) return self.logger.debug('Collect host info') - self.__hosts = [Host(hostname=h['hostname'], attributes=h['attributes']) + self.__hosts = [Host(hostname=h['hostname'], + resources=Resources(h['resources']['cpus'], + h['resources']['mem'], + h['resources']['disk'], + h['resources']['gpus'], + h['resources']['ports'][1:-1].split(',')), + attributes=h['attributes']) for h in body['slaves']] for h in self.__hosts: for kv_pair in h.attributes.items(): diff --git a/container/base.py b/container/base.py index 2f0f87a..be5a582 100644 --- a/container/base.py +++ b/container/base.py @@ -202,6 +202,8 @@ def to_request(self): class Container: + REQUIRED=frozenset(['id', 'type', 'image', 'resources']) + def __init__(self, id, appliance, type, image, resources, cmd=None, args=[], env={}, volumes=[], network_mode=NetworkMode.HOST, endpoints=[], ports=[], state=ContainerState.SUBMITTED, is_privileged=False, force_pull_image=True, @@ -229,6 +231,15 @@ def __init__(self, id, appliance, type, image, resources, cmd=None, args=[], env self.__dependencies = dependencies self.__last_update = last_update + @classmethod + def pre_check(cls, data): + if not isinstance(data, dict): + return 422, None, "Failed to parse container data format: %s"%type(data) + missing = Container.REQUIRED - data.keys() + if missing: + return 400, None, "Missing required field(s) of container: %s"%missing + return 200, "Container %s is valid"%data['id'], None + @property def id(self): return self.__id diff --git a/container/handler.py b/container/handler.py index 318d23f..e5cc685 100644 --- a/container/handler.py +++ b/container/handler.py @@ -37,18 +37,6 @@ class ContainersHandler(RequestHandler, Loggable): def initialize(self, config): self.__contr_mgr = ContainerManager(config) - async def post(self, app_id): - data = tornado.escape.json_decode(self.request.body) - data['appliance'] = app_id - status, contr, err = await self.__contr_mgr.create_container(data) - if err: - self.set_status(status) - self.write(json.dumps(error(err))) - return - status, contr, err = await self.__contr_mgr.provision_container(contr) - self.set_status(status) - self.write(json.dumps(contr.to_render() if status == 201 else error(err))) - class ContainerHandler(RequestHandler, Loggable):