api_scraper.py

#!/usr/bin/python

import sys
import re
import inflect

# Map HTTP method names to REST API names
REST_MAP = {'GET': 'get', 'PUT': 'update',
            'POST': 'create', 'DELETE': 'delete'}

# For plural->singular conversion
p = inflect.engine()


def underscorize(name):
    s1 = re.sub('(.)([A-Z][a-z]+)', r'\1_\2', name)
    return re.sub('([a-z0-9])([A-Z])', r'\1_\2', s1).lower()


# Write python indented lines
def pout(file, indent, text):
    file.write('    ' * indent)
    file.write(text)


def write_model_file(model):
    name = model['name']
    filename = underscorize(name)
    file = open("./decora_wifi/models/{0}.py".format(filename), 'w')
    pout(file, 0, "# Leviton Cloud Services API model {0}.\n".format(name))
    pout(file, 0, '# Auto-generated by api_scraper.py.\n')
    pout(file, 0, '#\n')
    pout(file, 0, '# Copyright 2017 Tim Lyakhovetskiy <tlyakhov@gmail.com>\n')
    pout(file, 0, '#\n')
    pout(file, 0, '# This code is released under the terms of the MIT license. ')
    pout(file, 0, 'See the LICENSE\n')
    pout(file, 0, '# file for more details.\n')
    pout(file, 0, 'from decora_wifi.base_model import BaseModel\n')
    pout(file, 0, '\n\n')
    pout(file, 0, "class {0}(BaseModel):\n".format(name))
    pout(file, 1, 'def __init__(self, session, model_id=None):\n')
    pout(file, 2, "super({0}, self).__init__(session, model_id)\n\n".format(name))

    for method in sorted(model['methods'].keys()):
        write_method(file, model, model['methods'][method])

    file.close()


def write_get_or_update(file, model, output_model, method, query):
    """Output code to query and instantiate models based on the result."""
    if method['is_array']:
        pout(file, 2, 'items = ' + query)
    else:
        pout(file, 2, 'data = ' + query)

    if method['type'] == 'class' or model['name'] != output_model:
        # If it's a class method or the output model doesn't match the parent
        # class, then we need to import the right class.
        if model['name'] != output_model:
            pkg = underscorize(output_model)
            pout(file, 2, "from .{0} import {1}\n".format(pkg, output_model))

        # Instantiate models with result JSON
        if method['is_array']:
            pout(file, 2, "result = []\n")
            pout(file, 2, "if items is not None:\n")
            pout(file, 3, "for data in items:\n")
            indent = 4
        else:
            indent = 2

        if method['type'] == 'class':
            construct_line = "model = {0}(session, data['id'])\n"
        else:
            construct_line = "model = {0}(self._session, data['id'])\n"

        pout(file, indent, construct_line.format(output_model))
        pout(file, indent, "model.data = data\n")

        if method['is_array']:
            pout(file, indent, "result.append(model)\n")
            pout(file, 2, "return result\n\n")
        else:
            pout(file, 2, "return model\n\n")
    else:
        if method['method'] == 'PUT':
            pout(file, 2, 'self.data.update(attribs)\n')
        elif method['method'] == 'GET':
            pout(file, 2, 'self.data.update(data)\n')
        pout(file, 2, 'return self\n\n')


def extract_model_from_url(api_split):
    """This method figures out the output model for a REST API method,"""
    # Get all the path elements that don't refer to IDs
    keys = [k for k in api_split if k and not k.startswith(':')]
    # The last one will be the type we want.
    last_key = keys[-1]
    # Some of them have a 'rel' path element...seems extraneous? not sure
    if last_key == 'rel':
        last_key = keys[-2]

    # Get a singular name, if it's plural
    last_key_singular = p.singular_noun(last_key) or last_key
    # Lowercase for matching to our list of models.
    last_key_singular = last_key_singular.lower()
    # Get all the model names
    model_names = list(extracted_models.keys())
    # Lowercase them for matching
    extracted_models_lower = [name.lower() for name in model_names]
    # Our path element key does not seem to match any model we know about.
    if last_key_singular not in extracted_models_lower:
        return None
    # Return the actual model name.
    model_index = extracted_models_lower.index(last_key_singular.lower())
    return model_names[model_index]


def write_method(file, model, method):
    """Output the code for a REST API method."""
    if method['type'] == 'loopback':
        # The loopback method is a special case - refreshes the data we have.
        url = method['url'].replace(':id', '{0}')
        pout(file, 1, 'def refresh(self):\n')
        pout(file, 2, "api = \"{0}\".format(self._id)\n".format(url))
        pout(file, 2, 'result = self._session.call_api(api, {}, \'get\')\n')
        pout(file, 2, 'if result is not None:\n')
        pout(file, 3, 'self.data.update(result)\n')
        pout(file, 2, 'return self\n')
        pout(file, 0, '\n')
        return

    # REST APIs, so slash delimited
    # print(method)
    api_split = method['url'].split('/')

    root_model = p.singular_noun(api_split[1]) or api_split[1]
    if root_model.lower() != model['name'].lower():
        print("Method {0} on {1} has a different root. Skipping...".format(
            method, model['name']))
        return

    # Gather all the foreign key stuff...
    fk_index = api_split.index(':fk') if ':fk' in api_split else None
    if fk_index is not None:
        fk_name = api_split[fk_index - 1]
        if fk_name == 'rel':
            fk_name = api_split[fk_index - 2]  # Why?
        fk_name = underscorize(fk_name)
        fk_name_singular = p.singular_noun(fk_name) or fk_name
        # For the parameter list
        fk_line = (", {0}_id".format(fk_name_singular))
    else:
        fk_name = None
        fk_name_singular = None
        fk_line = ''

    # Class methods don't have an :id.
    if method['type'] == 'class':
        api = method['url'].replace(':fk', '{0}')
    else:
        api = method['url'].replace(':id', '{0}').replace(':fk', '{1}')

    if method['type'] == 'class':
        decl = "def {0}(cls, session{1}, attribs=None):\n".format(
            method['name'], fk_line)
        pout(file, 1, "@classmethod\n")
        pout(file, 1, decl)
        pout(file, 2, 'if attribs is None:\n')
        pout(file, 3, 'attribs = {}\n')

        if fk_name_singular is not None:
            format_line = ".format({0}_id)".format(fk_name_singular)
        else:
            format_line = ''
        pout(file, 2, "api = \"{0}\"{1}\n".format(api, format_line))
        code = "session.call_api(api, attribs, \'{0}\')\n\n"
    else:
        decl = "def {0}(self{1}, attribs=None):\n".format(
            method['name'], fk_line)
        pout(file, 1, decl)
        pout(file, 2, 'if attribs is None:\n')
        pout(file, 3, 'attribs = {}\n')

        if fk_name is not None:
            format_line = ".format(self._id, {0}_id)".format(fk_name_singular)
        elif '{0}' in api:
            format_line = '.format(self._id)'
        else:
            format_line = ''
        pout(file, 2, "api = \"{0}\"{1}\n".format(api, format_line))
        code = "self._session.call_api(api, attribs, \'{0}\')\n\n"

    code = code.format(method['method'].lower())

    output_model = extract_model_from_url(api_split)

    if ((method['method'] == 'GET' or method['method'] == 'PUT') and
       (output_model is not None)):
        write_get_or_update(file, model, output_model, method, code)
    else:
        pout(file, 2, 'return ' + code)


# valid states = no-model, model, api
state = 'no-model'
RE_MODEL_DECL = re.compile("module\.factory\(\"(.*?)\"")
RE_MODEL_LBR = re.compile("LoopBackResource\(urlBase\s\+\s\"(.*?)\"")
RE_MODEL_END = re.compile("\s*\}\);")
RE_CLASS_API_DECL = re.compile("prototype\$_?_?(.*?):\s*{")
RE_MODEL_API_DECL = re.compile("\s*(.*?):\s*{")
RE_SUBMODEL_API_DECL = re.compile("\"::(.*?)\":\s*{")
RE_API_END = re.compile("^\s{16}\},?")
RE_API_URL = re.compile("urlBase\s*\+\s*\"(.*?)\"")
RE_API_METHOD = re.compile("method:\s*\"(.*?)\"")
RE_API_IS_ARRAY = re.compile("isArray:\s*!0")
RE_END = re.compile("module\.factory\(\"LoopBackAuth\"")

extracted_models = {}
model = None
method = None

for i, line in enumerate(open(sys.argv[1])):
    if state == 'no-model':
        match = re.search(RE_END, line)

        if match is not None:
            break

        match = re.search(RE_MODEL_DECL, line)
        if match is None:
            continue

        model = {'name': match.group(1), 'methods': {}}
        extracted_models[model['name']] = model
        state = 'model'
    elif state == 'model':
        match = re.search(RE_MODEL_END, line)

        if match is not None:
            state = 'no-model'
            continue

        match = re.search(RE_MODEL_LBR, line)

        if match is not None:
            method = {'name': 'get', 'type': 'loopback', 'is_array': False,
                      'url': match.group(1), 'method': 'get'}
            model['methods'][method['name']] = method
            continue

        match = re.search(RE_CLASS_API_DECL, line)

        if match is not None:
            method = {'name': match.group(1), 'type': 'class',
                      'is_array': False}
            method['name'] = underscorize(method['name']).replace('__', '_')
            model['methods'][method['name']] = method
            state = 'api'
            continue

        match = re.search(RE_SUBMODEL_API_DECL, line)

        if match is not None:
            method = {'name': match.group(1), 'type': 'subinstance',
                      'is_array': False}
            method['name'] = underscorize(
                method['name'].replace('::', '_')).replace('__', '_')
            model['methods'][method['name']] = method
            state = 'api'
            continue

        match = re.search(RE_MODEL_API_DECL, line)

        if match is not None:
            method = {'name': match.group(1), 'type': 'instance',
                      'is_array': False}
            method['name'] = underscorize(method['name'])
            model['methods'][method['name']] = method
            state = 'api'
            continue
    elif state == 'api':
        match = re.search(RE_API_END, line)

        if match is not None:
            state = 'model'
            continue

        match = re.search(RE_API_IS_ARRAY, line)

        if match is not None:
            method['is_array'] = True

        match = re.search(RE_API_URL, line)

        if match is not None:
            method['url'] = match.group(1)
            has_id = ':id' in method['url']
            has_fk = ':fk' in method['url']
            if method['type'] != 'class' and not has_id:
                method['type'] = 'class'
            elif method['type'] == 'class' and has_id:
                if has_fk:
                    method['type'] = 'subinstance'
                else:
                    method['type'] = 'instance'
            continue

        match = re.search(RE_API_METHOD, line)

        if match is not None:
            method['method'] = match.group(1)
            continue

for model in sorted(extracted_models.keys()):
    write_model_file(extracted_models[model])