25_week4_ML_django

lecture2 python data science stack/3. Pandas/pandas basics.ipynb

@@ -167,7 +167,9 @@
   {
    "cell_type": "code",
    "execution_count": 5,
-   "metadata": {},
+   "metadata": {
+    "collapsed": true
+   },
    "outputs": [],
    "source": [
     "df.to_excel('Excel_Sample.xlsx',sheet_name='Sheet1')"
@@ -203,7 +205,9 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "metadata": {},
+   "metadata": {
+    "collapsed": true
+   },
    "outputs": [],
    "source": [
     "df = pd.read_html('http://www.fdic.gov/bank/individual/failed/banklist.html')"
@@ -244,7 +248,9 @@
   {
    "cell_type": "code",
    "execution_count": 11,
-   "metadata": {},
+   "metadata": {
+    "collapsed": true
+   },
    "outputs": [],
    "source": [
     "from sqlalchemy import create_engine"
@@ -7330,7 +7336,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.6.2"
+   "version": "3.6.1"
   }
  },
  "nbformat": 4,

mlapiservertutorial/decisiontreeofgodapi/admin.py

@@ -0,0 +1,3 @@
+from django.contrib import admin
+
+# Register your models here.

mlapiservertutorial/decisiontreeofgodapi/apps.py

@@ -0,0 +1,5 @@
+from django.apps import AppConfig
+
+
+class DecisiontreeofgodapiConfig(AppConfig):
+    name = 'decisiontreeofgodapi'

mlapiservertutorial/decisiontreeofgodapi/models.py

@@ -0,0 +1,3 @@
+from django.db import models
+
+# Create your models here.

mlapiservertutorial/decisiontreeofgodapi/tests.py

@@ -0,0 +1,3 @@
+from django.test import TestCase
+
+# Create your tests here.

mlapiservertutorial/decisiontreeofgodapi/views.py

@@ -0,0 +1,3 @@
+from django.shortcuts import render
+
+# Create your views here.

mlapiservertutorial/manage.py

@@ -0,0 +1,22 @@
+#!/usr/bin/env python
+import os
+import sys
+
+if __name__ == "__main__":
+    os.environ.setdefault("DJANGO_SETTINGS_MODULE", "tutorial.settings")
+    try:
+        from django.core.management import execute_from_command_line
+    except ImportError:
+        # The above import may fail for some other reason. Ensure that the
+        # issue is really that Django is missing to avoid masking other
+        # exceptions on Python 2.
+        try:
+            import django
+        except ImportError:
+            raise ImportError(
+                "Couldn't import Django. Are you sure it's installed and "
+                "available on your PYTHONPATH environment variable? Did you "
+                "forget to activate a virtual environment?"
+            )
+        raise
+    execute_from_command_line(sys.argv)

mlapiservertutorial/mtcars.csv

@@ -0,0 +1,33 @@
+"","mpg","cyl","disp","hp","drat","wt","qsec","vs","am","gear","carb"
+"Mazda RX4",21,6,160,110,3.9,2.62,16.46,0,1,4,4
+"Mazda RX4 Wag",21,6,160,110,3.9,2.875,17.02,0,1,4,4
+"Datsun 710",22.8,4,108,93,3.85,2.32,18.61,1,1,4,1
+"Hornet 4 Drive",21.4,6,258,110,3.08,3.215,19.44,1,0,3,1
+"Hornet Sportabout",18.7,8,360,175,3.15,3.44,17.02,0,0,3,2
+"Valiant",18.1,6,225,105,2.76,3.46,20.22,1,0,3,1
+"Duster 360",14.3,8,360,245,3.21,3.57,15.84,0,0,3,4
+"Merc 240D",24.4,4,146.7,62,3.69,3.19,20,1,0,4,2
+"Merc 230",22.8,4,140.8,95,3.92,3.15,22.9,1,0,4,2
+"Merc 280",19.2,6,167.6,123,3.92,3.44,18.3,1,0,4,4
+"Merc 280C",17.8,6,167.6,123,3.92,3.44,18.9,1,0,4,4
+"Merc 450SE",16.4,8,275.8,180,3.07,4.07,17.4,0,0,3,3
+"Merc 450SL",17.3,8,275.8,180,3.07,3.73,17.6,0,0,3,3
+"Merc 450SLC",15.2,8,275.8,180,3.07,3.78,18,0,0,3,3
+"Cadillac Fleetwood",10.4,8,472,205,2.93,5.25,17.98,0,0,3,4
+"Lincoln Continental",10.4,8,460,215,3,5.424,17.82,0,0,3,4
+"Chrysler Imperial",14.7,8,440,230,3.23,5.345,17.42,0,0,3,4
+"Fiat 128",32.4,4,78.7,66,4.08,2.2,19.47,1,1,4,1
+"Honda Civic",30.4,4,75.7,52,4.93,1.615,18.52,1,1,4,2
+"Toyota Corolla",33.9,4,71.1,65,4.22,1.835,19.9,1,1,4,1
+"Toyota Corona",21.5,4,120.1,97,3.7,2.465,20.01,1,0,3,1
+"Dodge Challenger",15.5,8,318,150,2.76,3.52,16.87,0,0,3,2
+"AMC Javelin",15.2,8,304,150,3.15,3.435,17.3,0,0,3,2
+"Camaro Z28",13.3,8,350,245,3.73,3.84,15.41,0,0,3,4
+"Pontiac Firebird",19.2,8,400,175,3.08,3.845,17.05,0,0,3,2
+"Fiat X1-9",27.3,4,79,66,4.08,1.935,18.9,1,1,4,1
+"Porsche 914-2",26,4,120.3,91,4.43,2.14,16.7,0,1,5,2
+"Lotus Europa",30.4,4,95.1,113,3.77,1.513,16.9,1,1,5,2
+"Ford Pantera L",15.8,8,351,264,4.22,3.17,14.5,0,1,5,4
+"Ferrari Dino",19.7,6,145,175,3.62,2.77,15.5,0,1,5,6
+"Maserati Bora",15,8,301,335,3.54,3.57,14.6,0,1,5,8
+"Volvo 142E",21.4,4,121,109,4.11,2.78,18.6,1,1,4,2

mlapiservertutorial/requirements.txt

@@ -0,0 +1,80 @@
+alabaster==0.7.10
+appnope==0.1.0
+astroid==1.5.3
+Babel==2.5.0
+bleach==1.5.0
+boto3==1.4.5
+botocore==1.5.92
+certifi==2016.2.28
+chardet==3.0.4
+colorlover==0.2.1
+cufflinks==0.12.0
+cycler==0.10.0
+decorator==4.1.2
+Django==1.11.5
+djangorestframework==3.6.4
+docutils==0.14
+entrypoints==0.2.3
+html5lib==0.999
+imagesize==0.7.1
+ipykernel==4.6.1
+ipython==6.1.0
+ipython-genutils==0.2.0
+isort==4.2.15
+jedi==0.10.2
+Jinja2==2.9.6
+jmespath==0.9.0
+jsonschema==2.6.0
+jupyter-client==5.1.0
+jupyter-core==4.3.0
+lazy-object-proxy==1.3.1
+MarkupSafe==1.0
+matplotlib==2.0.2
+mccabe==0.6.1
+mistune==0.7.4
+nbconvert==5.2.1
+nbformat==4.4.0
+notebook==5.0.0
+numpy==1.13.1
+numpydoc==0.7.0
+pandas==0.20.3
+pandocfilters==1.4.2
+patsy==0.4.1
+pexpect==4.2.1
+pickleshare==0.7.4
+plotly==2.0.15
+prompt-toolkit==1.0.15
+psutil==5.2.2
+ptyprocess==0.5.2
+pycodestyle==2.3.1
+pyflakes==1.6.0
+Pygments==2.2.0
+pylint==1.7.2
+pyparsing==2.2.0
+python-dateutil==2.6.1
+pytz==2017.2
+pyzmq==16.0.2
+QtAwesome==0.4.4
+qtconsole==4.3.1
+QtPy==1.3.1
+requests==2.14.2
+rope==0.10.7
+rope-py3k==0.9.4.post1
+s3transfer==0.1.10
+scikit-learn==0.19.0
+scipy==0.19.1
+seaborn==0.8
+simplegeneric==0.8.1
+singledispatch==3.4.0.3
+six==1.10.0
+snowballstemmer==1.2.1
+Sphinx==1.6.3
+sphinxcontrib-websupport==1.0.1
+spyder==3.2.3
+statsmodels==0.8.0
+terminado==0.6
+testpath==0.3
+tornado==4.5.2
+traitlets==4.3.2
+wcwidth==0.1.7
+wrapt==1.10.11

mlapiservertutorial/sidmlapi/admin.py

@@ -0,0 +1,3 @@
+from django.contrib import admin
+
+# Register your models here.

mlapiservertutorial/sidmlapi/apps.py

@@ -0,0 +1,5 @@
+from django.apps import AppConfig
+
+
+class SidmlapiConfig(AppConfig):
+    name = 'sidmlapi'

mlapiservertutorial/sidmlapi/models.py

@@ -0,0 +1,3 @@
+from django.db import models
+
+# Create your models here.

mlapiservertutorial/sidmlapi/serializers.py

@@ -0,0 +1,14 @@
+from django.contrib.auth.models import User, Group
+from rest_framework import serializers
+
+
+class UserSerializer(serializers.HyperlinkedModelSerializer):
+    class Meta:
+        model = User
+        fields = ('url', 'username', 'email', 'groups')
+
+
+class GroupSerializer(serializers.HyperlinkedModelSerializer):
+    class Meta:
+        model = Group
+        fields = ('url', 'name')

mlapiservertutorial/sidmlapi/tests.py

@@ -0,0 +1,3 @@
+from django.test import TestCase
+
+# Create your tests here.

mlapiservertutorial/sidmlapi/views.py

@@ -0,0 +1,128 @@
+from django.contrib.auth.models import User, Group
+from rest_framework import viewsets
+from .serializers import UserSerializer, GroupSerializer
+from rest_framework.views import APIView
+from rest_framework.response import Response
+from rest_framework import status
+from django.http import JsonResponse
+from django.shortcuts import render, redirect
+from django.contrib.auth.decorators import login_required
+from django.contrib.auth import authenticate, login
+import json
+import math
+from django.http import HttpResponse, HttpResponseRedirect
+import os
+
+#import data science libraries
+from sklearn.model_selection import train_test_split
+from sklearn.linear_model import LinearRegression
+import pandas as pd
+import numpy as np
+from sklearn.neighbors import NearestNeighbors
+from sklearn import model_selection
+from sklearn.linear_model import LogisticRegression
+import pickle
+import json
+
+class UserViewSet(viewsets.ModelViewSet):
+    """
+    API endpoint that allows users to be viewed or edited.
+    """
+    queryset = User.objects.all().order_by('-date_joined')
+    serializer_class = UserSerializer
+
+class GroupViewSet(viewsets.ModelViewSet):
+    """
+    API endpoint that allows groups to be viewed or edited.
+    """
+    queryset = Group.objects.all()
+    serializer_class = GroupSerializer
+
+class LinearRegressPredict(APIView):
+
+    def get(self, request, *args, **kw):
+        print("calling get method")
+        # Any URL parameters get passed in **kw
+        result = {"result": "model trained"}
+        # model training ----------------------------------------------
+        filename = './finalized_model.sav'
+        print("import data")
+        USAhousing = pd.read_csv('USA_Housing.csv')
+        X = USAhousing[['Avg. Area Income', 'Avg. Area House Age', 'Avg. Area Number of Rooms',
+                        'Avg. Area Number of Bedrooms', 'Area Population']]
+        y = USAhousing['Price']
+        X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.4, random_state=101)
+        print("Start Training Process...")
+        lm = LinearRegression()
+        lm.fit(X_train, y_train)
+        pickle.dump(lm, open(filename, 'wb'))
+        response = Response(result, status=status.HTTP_200_OK)
+        return response
+
+    def post(self, request, *args, **kw):
+        msg_body = json.loads(request.body)
+        # TODO check data validity before publish mqtt message to Azure
+        print("msg_body: {}".format(msg_body))
+        print(type(msg_body))
+        X_test = pd.DataFrame.from_dict(msg_body, orient='index')
+        print(X_test.T)
+        filename = './finalized_model.sav'
+        loaded_model = pickle.load(open(filename, 'rb'))
+        pred = loaded_model.predict(X_test.T)
+        print(pred)
+        result = {"result": pred}
+        response = Response(result, status=status.HTTP_200_OK)
+        return response
+
+class RecommenderSystem(APIView):
+
+    def get(self, request, *args, **kw):
+        print("calling get method")
+        # Any URL parameters get passed in **kw
+        result = {"result": "model trained"}
+        # model training ----------------------------------------------
+        filename = './finalized_model.sav'
+        print("import data: mtcars")
+        cars = pd.read_csv('mtcars.csv')
+        cars.columns = ['car_names', 'mpg', 'cyl', 'disp', 'hp', 'drat', 'wt', 'qsec', 'vs', 'am', 'gear', 'carb']
+        X = cars.ix[:, (1, 3, 4, 6)].values
+        nbrs = NearestNeighbors(n_neighbors=2).fit(X)
+        print("Start Training Process...")
+        pickle.dump(nbrs, open(filename, 'wb'))
+        response = Response(result, status=status.HTTP_200_OK)
+        return response
+
+    def post(self, request, *args, **kw):
+        msg_body = json.loads(request.body)
+        # TODO check data validity before publish mqtt message to Azure
+        print("msg_body: {}".format(msg_body))
+        print(type(msg_body))
+        X_test = pd.DataFrame.from_dict(msg_body, orient='index')
+
+        # Verify features
+        ind = X_test.index.values.tolist()
+
+        if ind == ['mpg','disp','hp','wt']:
+
+            X_temp = X_test.iloc[:, 0].tolist()
+            print(X_temp)
+            filename = './finalized_model.sav'
+            loaded_model = pickle.load(open(filename, 'rb'))
+            pred = loaded_model.kneighbors([X_temp])
+            print(pred)
+
+            # Result
+            cars = pd.read_csv('mtcars.csv')
+            cars.columns = ['car_names', 'mpg', 'cyl', 'disp', 'hp', 'drat', 'wt', 'qsec', 'vs', 'am', 'gear', 'carb']
+            car1 = cars.loc[pred[1][0][0], :]
+            car2 = cars.loc[pred[1][0][1], :]
+            print(car1)
+            print(car2)
+            result = {"result": [car1,car2]}
+        else:
+            print("Input format has to be [\"mpg\":10,\"disp\":10,\"hp\":10,\"wt\":10]")
+            result = {"result": "Invalid Input"}
+
+        response = Response(result, status=status.HTTP_200_OK)
+        return response
+