project_p3(2).ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'C:\\\\spark-2.4.4'"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import findspark\n",
    "findspark.init()\n",
    "findspark.find()\n",
    "import pyspark\n",
    "findspark.find()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyspark import SparkContext, SparkConf\n",
    "from pyspark.sql.functions import *\n",
    "from pyspark.sql.session import SparkSession\n",
    "\n",
    "sc = SparkContext.getOrCreate()\n",
    "spark = SparkSession(sc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "users = spark.read.format(\"csv\")\\\n",
    ".options(header = True, inferSchema = True)\\\n",
    ".load(\"hdfs://localhost:9000/MovieData/input/users.csv\")\\\n",
    ".persist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "movies = spark.read.format(\"csv\")\\\n",
    ".options(header = True, inferSchema = True)\\\n",
    ".load(\"hdfs://localhost:9000/MovieData/input/movie.csv\")\\\n",
    ".persist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "ratings = spark.read.format(\"csv\")\\\n",
    ".options(header = True, inferSchema = True)\\\n",
    ".load(\"hdfs://localhost:9000/MovieData/input/rating.csv\")\\\n",
    ".persist()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "age_avg_rating = users.alias(\"t1\")\\\n",
    ".join(ratings.alias(\"t2\"), col(\"t1.user_id\") == col(\"t2.userId\"))\\\n",
    ".select(col(\"rating\"), col(\"age\"))\\\n",
    ".groupBy(col(\"age\"))\\\n",
    ".agg(count(col(\"age\")).alias(\"age_category\"), avg(\"rating\").alias(\"avg_rating\")) \\\n",
    ".orderBy(desc(\"avg_rating\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------+------------+------------------+\n",
      "|     age|age_category|        avg_rating|\n",
      "+--------+------------+------------------+\n",
      "|   50-55|       70641|3.5862954941181466|\n",
      "|   18-24|      159607| 3.538341050204565|\n",
      "|   35-44|      180083|3.5352143178423283|\n",
      "|   45-49|       85968|3.5168492927600967|\n",
      "|Under 18|       34969|3.5156710229060026|\n",
      "|   25-34|      314284| 3.514257805042573|\n",
      "|     56+|       55696|3.4560830221200805|\n",
      "+--------+------------+------------------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "age_avg_rating.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "age_avg_rating.coalesce(1).write.csv(\"hdfs://localhost:9000/MovieData/output/average_rating_age.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "root\n",
      " |-- age: string (nullable = true)\n",
      " |-- age_category: long (nullable = false)\n",
      " |-- avg_rating: double (nullable = true)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "age_avg_rating.printSchema()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAb4AAAFACAYAAADDFRmAAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAGllJREFUeJzt3Xu0ZGV95vHvY3eLRFCjnEQGGjqJJEYdBWwJxsQgOjNIVOISVzA3yZhhZImKlySaySBinHgZkxhJZFCJqIgoXoIsvGAU8QbSkAZB1HQMDq3M0IoBexS0yW/+2PtIUZzTfaq7dtfpfr+ftWqdfa/fW+eceurde9feqSokSWrFvWZdgCRJO5PBJ0lqisEnSWqKwSdJaorBJ0lqisEnSWqKwSdJaorBJ0lqisEnSWrKylkXsD322WefWrNmzazLkCQtI1deeeW3q2puW8vtksG3Zs0a1q1bN+syJEnLSJJvLGU5d3VKkppi8EmSmmLwSZKaYvBJkppi8EmSmmLwSZKaYvBJkppi8EmSmmLwSZKaYvBJkppi8EmSmrJLXqtTknamG9/4klmXMJHVL3zDrEtY1uzxSZKaYvBJkpoy6K7OJPcBLgX26J/r/Kp6xdgyxwOvB77ZTzq9qt46ZF27ot966DGzLmFi7/7K38+6BEm6h6GP8d0BHFlVm5OsAj6b5CNVddnYcudV1UkD1yLNzEWHPmvWJUzs6KvOnXUJ0iAGDb6qKmBzP7qqf9RQz/eo1b801KYHcfWNl8+6hGXjNY/4nVmXMLGXXfuuWZewbHzl2c+cdQkTe+jZ75t1CZqRwY/xJVmRZD1wM3BxVS30bv+MJNckOT/J6kW2c0KSdUnWbdq0adCaJUm7r8GDr6rurKqDgf2Bw5I8YmyRDwNrquqRwCeAsxfZzplVtbaq1s7NzQ1btCRpt7XTzuqsqn8FLgGOGpv+naq6ox99C/DonVWTJKk9Q5/VOQf8qKr+NcmewJOA144ts29V3dSPPg24fsiaJEl3+fYl7551CRPb54jf2qH1hz6rc1/g7CQr6HqX762qC5OcBqyrqguAFyR5GrAFuAU4fuCaJEkNG/qszmuAQxaYfsrI8MuBlw9ZhyRJ87xyiySpKQafJKkpBp8kqSkGnySpKQafJKkpBp8kqSkGnySpKQafJKkpBp8kqSkGnySpKQafJKkpBp8kqSkGnySpKQafJKkpBp8kqSkGnySpKQafJKkpBp8kqSkGnySpKQafJKkpBp8kqSkGnySpKQafJKkpBp8kqSkGnySpKQafJKkpBp8kqSmDBl+S+yT5YpKrk1yX5JULLLNHkvOSbEhyeZI1Q9YkSWrb0D2+O4Ajq+pRwMHAUUkOH1vmOcB3q+ohwF8Crx24JklSwwYNvups7kdX9Y8aW+wY4Ox++HzgiUkyZF2SpHYNfowvyYok64GbgYur6vKxRfYDbgSoqi3ArcCDFtjOCUnWJVm3adOmocuWJO2mBg++qrqzqg4G9gcOS/KIsUUW6t2N9wqpqjOram1VrZ2bmxuiVElSA3baWZ1V9a/AJcBRY7M2AqsBkqwE7g/csrPqkiS1ZeizOueSPKAf3hN4EvCVscUuAJ7dDx8LfLKq7tHjkyRpGlYOvP19gbOTrKAL2fdW1YVJTgPWVdUFwNuAdybZQNfTO27gmiRJDRs0+KrqGuCQBaafMjJ8O/DMIeuQJGmeV26RJDXF4JMkNcXgkyQ1xeCTJDXF4JMkNcXgkyQ1xeCTJDXF4JMkNcXgkyQ1xeCTJDXF4JMkNcXgkyQ1xeCTJDXF4JMkNcXgkyQ1xeCTJDXF4JMkNcXgkyQ1xeCTJDXF4JMkNcXgkyQ1xeCTJDXF4JMkNcXgkyQ1xeCTJDXF4JMkNcXgkyQ1ZdDgS7I6yaeSXJ/kuiQvXGCZI5LcmmR9/zhlyJokSW1bOfD2twAvqaqrkuwNXJnk4qr68thyn6mqpwxciyRJw/b4quqmqrqqH/4ecD2w35DPKUnS1uy0Y3xJ1gCHAJcvMPuxSa5O8pEkD19k/ROSrEuybtOmTQNWKknane2U4EuyF/B+4OSqum1s9lXAgVX1KOBNwIcW2kZVnVlVa6tq7dzc3LAFS5J2W4MHX5JVdKF3TlV9YHx+Vd1WVZv74YuAVUn2GbouSVKbhj6rM8DbgOur6i8WWebB/XIkOayv6TtD1iVJatfQZ3U+Dvhd4EtJ1vfT/gQ4AKCqzgCOBU5MsgX4AXBcVdXAdUmSGjVo8FXVZ4FsY5nTgdOHrEOSpHleuUWS1BSDT5LUFINPktQUg0+S1BSDT5LUFINPktQUg0+S1BSDT5LUFINPktQUg0+S1BSDT5LUFINPktQUg0+S1BSDT5LUFINPktSUJd+PL8mhC0y+FfhGVW2ZXkmSJA1nkhvR/i1wKHAN3c1lH9EPPyjJc6vq4wPUJ0nSVE2yq/MG4JCqWltVjwYOAa4FngS8boDaJEmaukmC76FVdd38SFV9mS4Ivz79siRJGsYkuzq/muTNwHv68d8EvpZkD+BHU69MkqQBTNLjOx7YAJwMvAj4ej/tR8ATpl2YJElDWHKPr6p+ALyhf4zbPLWKJEka0CRfZ3gccCpw4Oh6VfWz0y9LkqRhTHKM7210uzivBO4cphxJkoY1SfDdWlUfGawSSZJ2gkmC71NJXg98ALhjfmJVXTX1qiRJGsgkwfdL/c+1I9MKOHJ65UiSNKxJzuqc+CsLSVYD7wAeDPwbcGZVvXFsmQBvBI4Gvg8cby9SkjSUbQZfkt+pqnclefFC86vqL7ay+hbgJVV1VZK9gSuTXNxf9WXek4GD+scvAW/mrt6lJElTtZQe3337n3svMK+2tmJV3QTc1A9/L8n1wH7AaPAdA7yjqgq4LMkDkuzbrytJ0lRtM/iq6n/1g5+oqs+Nzuu/27ckSdbQXdj68rFZ+wE3joxv7KfdLfiSnACcAHDAAQcs9WklSbqbSS5Z9qYlTruHJHsB7wdOrqrbxmcvsMo9epJVdWZ/Z4i1c3NzS3laSZLuYSnH+B4L/DIwN3ac737AiiWsv4ou9M6pqg8ssMhGYPXI+P7At7a1XUmStsdSenz3BvaiC8m9Rx63AcdubcX+jM23Addv5SSYC4DfS+dwui/Ke3xPkjSIpRzj+zTw6SRvr6pvTLj9xwG/C3wpyfp+2p8AB/TbPgO4iO6rDBvovs7w+xM+hyRJSzbJF9i/31+55eHAfeYnVtWiX2Cvqs+y8DG80WUKeN4EdUiStN0mObnlHOArwM8ArwRuAK4YoCZJkgYzSfA9qKreBvyoqj5dVf8ZOHyguiRJGsQkuzp/1P+8Kcmv0515uf/0S5IkaTiTBN+fJbk/8BK67+/dj+7+fJIk7TKWFHxJVgAHVdWFwK3AxBesliRpOVjSMb6quhN42sC1SJI0uEl2dX4+yenAecD/m5/oLYQkSbuSSYLvl/ufp41M80a0kqRdytRuRJvk2VV19o6XJEnScCb5Ht+2vHCK25IkaRDTDL6tXppMkqTlYJrBt9W7sUuStBzY45MkNWWawfe5KW5LkqRBLPmszrG7r8+7FbiyqtZX1UnTK0uSpGFM0uNbCzwX2K9/nAAcAbwlyR9NvzRJkqZvki+wPwg4tKo2AyR5BXA+8HjgSuB10y9PkqTpmqTHdwDww5HxHwEHVtUPgDumWpUkSQOZpMf3buCyJH/fjz8VODfJfYEvT70ySZIGMMkly16V5CLgV+i+uvDcqlrXz/7tIYqTJGnaJjmr843AeVX1xgHrkSRpUJMc47sK+NMkG5K8PsnaoYqSJGkoSw6+qjq7qo4GDgO+Brw2yT8NVpkkSQPYniu3PAR4KLAG+MpUq5EkaWBLDr4k8z2804BrgUdX1VMHq0ySpAFM8nWGf6G7C/vPAnsAj0xCVV06SGWSJA1gkuC7E/gksD+wHjgc+AJw5AB1SZI0iEmO8b0AeAzwjap6AnAIsGlrKyQ5K8nNSa5dZP4RSW5Nsr5/nDJBPZIkTWySHt/tVXV7EpLsUVVfSfIL21jn7cDpwDu2ssxnquopE9QhSdJ2myT4NiZ5APAh4OIk3wW+tbUVqurSJGu2vzxJkqZrkkuWPb0fPDXJp4D7Ax+dQg2PTXI1XYi+tKqum8I2JUla0CQ9vh+rqk9P6fmvorvDw+YkR9P1Jg9aaMEkJ9DdA5ADDjhgSk8vSWrN9nyBfWqq6rb5+/tV1UXAqiT7LLLsmVW1tqrWzs3N7dQ6JUm7j5kGX5IHJ0k/fFhfz3dmWZMkafe2Xbs6lyrJucARwD5JNgKvAFYBVNUZwLHAiUm2AD8AjquqGrImSVLbBg2+qnrWNuafTvd1B0mSdoqZ7uqUJGlnM/gkSU0x+CRJTTH4JElNMfgkSU0x+CRJTTH4JElNMfgkSU0x+CRJTTH4JElNMfgkSU0x+CRJTTH4JElNMfgkSU0x+CRJTTH4JElNMfgkSU0x+CRJTTH4JElNMfgkSU0x+CRJTTH4JElNMfgkSU0x+CRJTTH4JElNMfgkSU0x+CRJTRk0+JKcleTmJNcuMj9J/jrJhiTXJDl0yHokSRq6x/d24KitzH8ycFD/OAF488D1SJIaN2jwVdWlwC1bWeQY4B3VuQx4QJJ9h6xJktS2WR/j2w+4cWR8Yz9NkqRBzDr4ssC0WnDB5IQk65Ks27Rp08BlSZJ2V7MOvo3A6pHx/YFvLbRgVZ1ZVWurau3c3NxOKU6StPuZdfBdAPxef3bn4cCtVXXTjGuSJO3GVg658STnAkcA+yTZCLwCWAVQVWcAFwFHAxuA7wO/P2Q9kiQNGnxV9axtzC/geUPWIEnSqFnv6pQkaacy+CRJTTH4JElNMfgkSU0x+CRJTTH4JElNMfgkSU0x+CRJTTH4JElNMfgkSU0x+CRJTTH4JElNMfgkSU0x+CRJTTH4JElNMfgkSU0x+CRJTTH4JElNMfgkSU0x+CRJTTH4JElNMfgkSU0x+CRJTTH4JElNMfgkSU0x+CRJTTH4JElNGTz4khyV5KtJNiR52QLzj0+yKcn6/vEHQ9ckSWrXyiE3nmQF8DfAfwA2AlckuaCqvjy26HlVddKQtUiSBMP3+A4DNlTV16vqh8B7gGMGfk5JkhY1dPDtB9w4Mr6xnzbuGUmuSXJ+ktUD1yRJatjQwZcFptXY+IeBNVX1SOATwNkLbig5Icm6JOs2bdo05TIlSa0YOvg2AqM9uP2Bb40uUFXfqao7+tG3AI9eaENVdWZVra2qtXNzc4MUK0na/Q0dfFcAByX5mST3Bo4DLhhdIMm+I6NPA64fuCZJUsMGPauzqrYkOQn4GLACOKuqrktyGrCuqi4AXpDkacAW4Bbg+CFrkiS1bdDgA6iqi4CLxqadMjL8cuDlQ9chSRJ45RZJUmMMPklSUww+SVJTDD5JUlMMPklSUww+SVJTDD5JUlMMPklSUww+SVJTDD5JUlMMPklSUww+SVJTDD5JUlMMPklSUww+SVJTDD5JUlMMPklSUww+SVJTDD5JUlMMPklSUww+SVJTDD5JUlMMPklSUww+SVJTDD5JUlMMPklSUww+SVJTBg++JEcl+WqSDUletsD8PZKc18+/PMmaoWuSJLVr0OBLsgL4G+DJwMOAZyV52NhizwG+W1UPAf4SeO2QNUmS2jZ0j+8wYENVfb2qfgi8BzhmbJljgLP74fOBJybJwHVJkho1dPDtB9w4Mr6xn7bgMlW1BbgVeNDAdUmSGrVy4O0v1HOr7ViGJCcAJ/Sjm5N8dQdrm8Q+wLenvdFl1LEdpH3n7ubte3nOmfYmt8cgbQMg7xlksxMarn3vWBZ/n8O07+S/mPomt9NAv7/fXmzGgUtZe+jg2wisHhnfH/jWIstsTLISuD9wy/iGqupM4MyB6tyqJOuqau0snntnsH27rt25bWD7dnXLtX1D7+q8Ajgoyc8kuTdwHHDB2DIXAM/uh48FPllV9+jxSZI0DYP2+KpqS5KTgI8BK4Czquq6JKcB66rqAuBtwDuTbKDr6R03ZE2SpLYNvauTqroIuGhs2ikjw7cDzxy6jh00k12sO5Ht23Xtzm0D27erW5bti3sVJUkt8ZJlkqSmGHySpKbsNsGXZE2Sa8emnZrkpRNu55Ik2336bZLHJ7kqyZYkx47Ne12S65Jcn+Svp32FmiRnJbl59HVIcnCSy5KsT7IuyWGLrHtOf03Va/vtrBqb/5gkd463aWdKsjrJp/rX77okL+ynn5rkm30b1yc5epH1X5Xkmn6Zjyf5d2PzZ9rGJPdJ8sUkV/fte2U//e1J/mWkfQdvYztvSrJ5genHJqkd+fveUUlWJPnHJBf24zvUtiQHJvmH/vd6SZL9h27DVmq7IcmX5v/X+mkPTHJxkn/qf/7kIusu+DokOSLJrSPTT1lo/VlYqL399Of37yXXJXndLGtcVFXtFg9gDXDt2LRTgZdOuJ1LgLUTLL9igToeCbwDOHZk+i8Dn6M7u3UF8AXgiCm/Bo8HDh19HYCPA0/uh48GLllk3aPpLiYQ4FzgxNE2Ap+kO0np2GnWPGH79gUO7Yf3Br5Gdw3YJf2egfuNDL8AOGM5tbF/7ffqh1cBlwOHA29fak3AWuCdwOax6XsDlwKXTfL3PUAbXwy8G7iwH9+htgHvA57dDx8JvHOGbbsB2Gds2uuAl/XDLwNeu8i6C74OwBHzr9VyeyzS3icAnwD26Md/aoH1TgWOn2Xtu02Pb1v6T4Ov7T9Rfy3Jr/bT90zynv4T43nAniPr/MckX+h7cO9Lslc//YYkpyT5LGNnpFbVDVV1DfBvYyUUcB/g3sAedG9s/3eabayqS7nnl/8LuF8/fH/ueQGB+XUvqh7wRbqLDcx7PvB+4OZp1jupqrqpqq7qh78HXM89L4G3tfVvGxm9L3e/QtDM29i//PO9mVX9Y8lnn6W7KPzrgT9aYPar6N6Eb9/ROrdX3xv7deCt27HuYm17GPAP/fCnuOe1gGdt9FrEZwO/McNadoYTgddU1R0AVTXT94zFNBN8vZVVdRhwMvCKftqJwPer6pHAq4FHAyTZB/hT4ElVdSiwju7T6rzbq+pXqmpJ13Wqqi/Q/WPe1D8+VlXXT6FN23Iy8PokNwL/E3j51hbud3H+LvDRfnw/4OnAGQPXOZF0t686hK5XBHBS/+HlrMV2J/Xrvbp/LX4bOKWftmza2O8KXE8XwBdX1Xz7Xt237y+T7LHI6icBF1TVTWPbPARYXVUXDlf5kvwVXXCNfyjc7rYBVwPP6IefDuydZFbX+i3g40muTHeJRYCfnq+5//lTW1l/sdfhsf3u748kefhAtW+Phdr788CvprvF3KeTPGaG9S1qdwq+xT4Zj07/QP/zSrpdktDtHnwXQN9Tu6affjjdp8nP9W9Ez+bu14E7b5LikjwE+EW6ntR+wJFJHj/JNrbTicCLqmo18CK6CwZszd8Cl1bVZ/rxvwL+uKruHLDGifQ97/cDJ/e9uDcDPwccTPeh4g2LrVtV/61/Lc6hezOFZdTGqrqzqg6m+zs5LMkj6D6sPBR4DPBA4I/H1+uPVz4TeNPY9HvR3e7rJQOXvlVJngLcXFVXjs3a7rb1Xgr8WpJ/BH4N+CawZYqlT+Jx/YfkJwPPm/D/e7HX4SrgwKp6FF37PzTFenfUQu1dCfwk3fvnHwLvTeffzx+nBJ4LnDZy3HLnf1CZ9X7iaT2AvYBvjk37a+7a/38J/bENugun3tAPfwh4wsg6V9EdS3gqcO4iz3UDY/u2F1jm7dz9GN8fAv99ZPwU4I8GeB3WcPdjfLdy1/c1A9zWD38MWA+8dWTZV/Svx71Gpv1L394bgM10PZHfmOHveVVf+4u31X7g7/o2XrTAcgeOLLes2jj2+3jp2LQjuOv42I9/h3S7EP/PSDv+DdhAt3v72yPTb6fb3b1Tj/MBf053Xd4b+jq/D7xrR9q2wHPsBWyc9e+tr+VUulD+KrBvP21f4Kvj7Vtg3R+/DgvMu4FtvPfMuL0fZeTcBeCfgbkFlj1+pvXO+gWb8ou/DnhiP/xAupMffq4fv4SFg+/F8398wCPoPi2uBeaA/w08pJ/3E8DP98Pb/OPjnsH3m3QHfVfSvXn/A/DUAV6DNdw9+K6f/0MEnghcuch6fwB8HthzqW2awe83dCcN/dXY9H1Hhl8EvGeR9Q8aGX4+cP5yamP/N/eAfnhP4DPAU0beOEPXO33NEra1eZHpP/4/mOHv8cdv7Dvatv5/+V798KuB02bUpvsCe48Mfx44iu645OjJLa9bZP0FXwfgwdz1wfWw/j0ps/z9baO9z53/HdDt9rxxvF6WQfANfsmynez3gL9JMr+r65VV9c/bWOfNwN8luYbuE9gXAapqU5LjgXNH9rf/KV2YLqrfp/1Buu7+U5O8sqoeTneT3SOBL9Htfv1oVX140gZu47nPpXtT2SfJRroew38B3pjuzhe3c9etncadAXwD+EK6b1l8oKpOm2Z9U/A4uuOPX+p3mQD8CfCs/vTvovtQ8l8XWf81SX6BrsfwDbp/0uVkX+Ds/kSOewHvraoLk3wyyRzdm+L8rqLdxTk72LYjgD9PUnRnrT5vuuUt2U8DH+z/d1YC766qjya5gm5333PoQmuxyzMu9jocC5yYZAvwA+C46tNjxhZr772Bs9J9peqHdHvclkO9d+MlyyRJTdmdTm6RJGmbDD5JUlMMPklSUww+SVJTDD5JUlMMPklSUww+SVJTDD5pmUvyof5CwNfNXww4yXP6u4xckuQtSU7vp88leX+SK/rH42ZbvbT8+AV2aZlL8sCquiXJnsAVwH+iu7fjocD36O4jeHVVnZTk3cDfVtVnkxxAdxeQX5xZ8dIytLtdskzaHb0gydP74dV0l237dFXdApDkfXTXRQR4EvCw/lJSAPdLsnd19y+UhMEnLWtJjqALs8dW1feTXEJ3xf/FenH36pf9wc6pUNr1eIxPWt7uD3y3D72H0t3n7Cfo7kH3k/3Fx58xsvzHues+g/QX75Y0wuCTlrePAiv7u4e8CriM7mar/4Pu7vOfAL5Md99FgBcAa/s7eX+Z3etODtJUeHKLtAtKsldVbe57fB8EzqqqD866LmlXYI9P2jWd2t+T8Fq6O8h/aMb1SLsMe3ySpKbY45MkNcXgkyQ1xeCTJDXF4JMkNcXgkyQ15f8DAOvZtZaba4gAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 504x360 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sb\n",
    "%matplotlib inline \n",
    "\n",
    "plt.figure(figsize=(7, 5))\n",
    "\n",
    "df = age_avg_rating.toPandas()\n",
    "sb.barplot(data = df, x = 'age', y = 'avg_rating', ci = None, palette = 'rocket', order = [\"Under 18\", \"18-24\", \"25-34\", \"35-44\", \"45-49\", \"50-55\", \"56+\"]);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "age_profession_avg_rating = users.alias(\"t1\")\\\n",
    ".join(ratings.alias(\"t2\"), col(\"t1.user_id\") == col(\"t2.userId\"))\\\n",
    ".select(col(\"rating\"), col(\"occupation\"))\\\n",
    ".groupBy(col(\"occupation\"))\\\n",
    ".agg(count(col(\"occupation\")).alias(\"category\"), avg(\"rating\").alias(\"avg_rating\")) \\\n",
    ".orderBy(desc(\"avg_rating\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------------------+--------+------------------+\n",
      "|          occupation|category|        avg_rating|\n",
      "+--------------------+--------+------------------+\n",
      "|              farmer|    3094|3.6872979961215253|\n",
      "|          unemployed|   10658| 3.678692062300619|\n",
      "|              artist|   35857| 3.609239479041749|\n",
      "|       self-employed|   33323|3.6032320019205955|\n",
      "|              writer|   33616|3.5818955259400287|\n",
      "|      clerical/admin|   23156|3.5788132665399894|\n",
      "|   academic/educator|   71038|3.5780286607167993|\n",
      "|               other|  105715|3.5483706191174385|\n",
      "|        K-12 student|   28282|3.5443921929142213|\n",
      "| tradesman/craftsman|    9382|3.5369857173310595|\n",
      "|executive/managerial|  101014| 3.535757419763597|\n",
      "|              lawyer|   19581| 3.522394157601757|\n",
      "| technician/engineer|   85038| 3.522007808273948|\n",
      "|college/grad student|  110429|3.5194061342582112|\n",
      "|          programmer|   62103| 3.498116033041882|\n",
      "|  doctor/health care|   39845| 3.490450495670724|\n",
      "|     sales/marketing|   45809|3.4622781549477177|\n",
      "|           scientist|   19155| 3.462176977290525|\n",
      "|           homemaker|   14685|3.4351719441607083|\n",
      "|             retired|   26161|3.4063491456748594|\n",
      "+--------------------+--------+------------------+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "age_profession_avg_rating.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "age_profession_avg_rating.coalesce(1).write.csv(\"hdfs://localhost:9000/MovieData/output/average_rating_profession.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAsEAAAGuCAYAAABx+9tmAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzs3XecnGW9/vHPRRJIQkJCaCLFANJbIKGEIokih+IBKYoaFQGNwIGAHg4Hy1EEC4jCQSwQ+VEUVKQe1CNFeieFkELzUFQEQaRIDSG5fn/MvTKsszuzSXZnd+d6v1772mfu5y7f59n88c0933lGtomIiIiIaCXLNDuAiIiIiIieliQ4IiIiIlpOkuCIiIiIaDlJgiMiIiKi5SQJjoiIiIiWkyQ4IiIiIlpOkuCIiIiIaDlJgiMiIiKi5SQJjoiIiIiWM7DZAUTvt/LKK3v06NHNDiMiIiKirhkzZjxre5V6/ZIER12jR49m+vTpzQ4jIiIioi5Jf2ion+3ujiX6uIEDl/PIYWs0O4yIiIjow5594dEeWUfSDNvj6vVLTXBEREREtJwkwRERERHRcpIER0RERETLSRLcS0iaIukBSRc1O5aIiIiI/i5Ph+g9jgD2sP1YvY6SBtp+c2ktvLTni4iIiOjtkgT3ApLOAtYFrpJ0IbAPMAR4DTjY9kOSPgXsBQwGlpd0IvA14GlgDHA5MAc4uoz9oO1HJK0CnAWsXZY7xvbtkk4A3gmMBp4FPtYDlxoRERHRKyQJ7gVsHyZpd2Ai8AbwXdtvStoV+Cawf+k6HtjC9nOSJgBbAhsDzwGPAufY3lbS0cBRwDHAGcDptm+TtDZwTRkDMBbYyfZr7WOSNBmYDLCMBnTHZUdEREQ0TZLg3mcEcIGk9QEDg6rOXWf7uarX02w/BSDpEeDa0j6HSkINsCuwiaS2MStIGl6Or6qVAAPYngpMhcpzgpfskiIiIiJ6lyTBvc9JwI2295U0Grip6twr7frOrzpeVPV6EW/9bZcBxrdPdktS3H6+iIiIiJaQp0P0PiOAP5fjTy2F+a4Fjmx7IWnMUpgzIiIiok9LEtz7fBv4lqTbgaVRjDsFGCdptqT7gcOWwpwRERERfZrslHtG5wYOXM4jh63R7DAiIiKiD3v2hUd7ZB1JM2yPq9cvO8ERERER0XKSBEdEREREy8nTIaKuMWM2Z/r06c0OIyIiImKpyU5wRERERLScJMERERER0XLydIioa7lBQ73mSu9udhgRERHRBz3yl9k9ul6eDhERERER0YEkwRERERHRcpIER0RERETLSRLci0maIOnX3bzG45JW7s41IiIiInqbJMERERER0XKSBNcgabSkuVWvj5V0gqSbJJ0i6R5JD0vauZwfIOlUSdMkzZb02dI+QdLNkn5Z+p8saVIZP0fSeqXf+ZLOknRr6feBGjGNknRlmf8uSVtIWkbS7yWtUvosI+n/JK0saRVJl5WYpknasfRZSdK1ku6VdDagHrilEREREb1KkuCuG2h7W+AY4Kul7VDgRdvbANsAn5G0Tjm3JXA0sDnwCWCDMv4c4KiqeUcDuwB7AWdJGtxu3a8B99reAvgi8BPbi4ALgUmlz67AfbafBc4ATi8x7V/Wo8R8m+2tgKuAtZfkZkRERET0Rfna5K67vPyeQSVxBdgN2ELSAeX1CGB94A1gmu2nACQ9Alxb+swBJlbN+8uS1P5e0qPARu3W3YlKMovtG8qO7gjgXOB/gP8GDgHOK/13BTaR/rHRu4Kk4cB7gP3KPL+R9Hyti5Q0GZgMMHCZQXVuSURERETfkiS4tjd5+y559a7s/PJ7IW/dPwFH2b6mehJJE6r6Ayyqer2It9//9t9a0v51rbIF2/6TpKclvRfYjrd2hZcBxtt+rV1MteauNfFUYCpUviyjXv+IiIiIviTlELU9DaxadluXA/6pRreda4DDJQ0CkLSBpOW7uOaHSk3vesC6wEPtzt9CSXBLcv2s7b+Xc+dQKYv4pe2Fpe1a4Mi2wZLG1JhnD2DFLsYZERER0edlJ7gG2wsknQjcDTwGPFhnyDlUSiNmqrLV+lfgg11c9iHgZmA14DDbr1eVMgCcAJwnaTbwKnBQ1bmrqJRBnFfVNgX4Qek/kEryexiV2uKfS5pZ1vtjF+OMiIiI6PNk553uZpN0PvBr25cu5vhxVD4Et/NSDaxYbtBQr7nSu7tj6oiIiOjnHvnL7B5dT9IM2+Pq9ctOcB8n6XjgcN6qBY6IiIiIOrITHHVlJzgiIiIWV3aCo8/afMtNmD59erPDiIiIiFhq8nSIiIiIiGg5SYIjIiIiouWkJjjqGrrsMG/4js2aHUZERET0Ivf+8a5mh1BTozXB2QmOiIiIiJaTJDgiIiIiWk6S4IiIiIhoOUmC+xFJx0gaWvX6fyWNbLR/RERERKtIEtxPSBoAHAP8I6m1vaftFzoZ9rb+EREREa0iX5bRR0i6ElgLGAycYXuqpJeB04B/AX4DvBO4UdKztidKehwYB7wG/BJYExgAnASs1r5/T19TRERERLMkCe47DrH9nKQhwDRJlwHLA3NtfwVA0iHARNvPthu7O/Ck7b1KvxG2X5T0+Q76R0RERPRrKYfoO6ZIug+4i8qO8PrAQuCyBsbOAXaVdIqknW2/WG+ApMmSpkua/uaiBUsUeERERERvkyS4D5A0AdgVGG97S+BeKmURr9teWG+87YeBsVSS4W9J+koDY6baHmd73MBlBi1R/BERERG9Tcoh+oYRwPO2X5W0EbB9B/1eAoYDbytvkPRO4DnbF5Y64k911j8iIiKiv0sS3DdcDRwmaTbwEJWSiFqmAr+V9FS7D7ptDpwqaRGwADi8Tv+IiIiIfk22mx1D9HJDlx3mDd+xWbPDiIiIiF7k3j92tCfXXJJm2B5Xr19qgiMiIiKi5SQJjoiIiIiWk5rgqGuTLTZi+vTe+ZZHRERExOLITnBEREREtJwkwRERERHRcpIER0RERETLySPSoq7hy63gcWvUfdJIREREtIgbH72h2SF0KI9Ii4iIiIjoQJLgiIiIiGg5SYI7Iel8SQeU450lzZM0S9KQHlr/BEnHduP8oyXN7a75IyIiInqrJMGNmwR8x/YY2681O5iIiIiIWHwtlwRLWl7SbyTdJ2mupAMljZV0s6QZkq6RtHq7MZ8GPgx8RdJFNeZcRdJlkqaVnx1L+wmSLpB0raTHJe0n6duS5ki6WtKg0u9xSadIuqf8vLvGGmMk3SVptqQrJK0oaT1JM6v6rC9pRjmueU2l/T5JdwL/thRvbURERESf0XJJMLA78KTtLW1vBlwNnAkcYHsscC7wjeoBts8BrgL+w/akGnOeAZxuextgf+CcqnPrAXsB+wAXAjfa3hx4rbS3+bvtbYHvA/9dY42fAP9pewtgDvBV248AL0oaU/ocDJxfkuuOruk8YIrt8Z3epYiIiIh+rBW/NnkO8B1JpwC/Bp4HNgOukwQwAHiqi3PuCmxSxgOsIGl4Of6t7QWS5pS5r66KY3TVHD+v+n169eSSRgAjbd9cmi4ALinH5wAHS/o8cCCwLbBhrWuqMc9PgT1qXZCkycBkgOUGLNfALYiIiIjoO1ouCbb9sKSxwJ7At4DrgHld2RmV9A3KLq7tMVR21Me3rxUuCej80m+RpAV+68HMi3j7/XcHx/VcBnwVuAGYYftvkt5Z65okjWx0bttTgalQeU5wF+KJiIiI6PVarhyiJIiv2r4Q+A6wHbCKpPHl/CBJm3Y2h+0vlQ/ItZUhXAscWbXGmNojO3Vg1e872633IvC8pJ1L0yeAm8u514FrgB9RKXUAeKjWNdl+gUr5xE6lX63SjoiIiIh+r+V2goHNgVMlLQIWAIcDbwLfK+UCA6nU5M7rwpxTgB9Iml3G3wIc1sW4lpN0N5X/mHy0xvmDgLMkDQUepVL/2+YiYD8qyTi23yiPdqt1TQcD50p6lUryHBEREdFy8rXJvYCkx4Fxtp9dzPHHAiNs/9dSDazI1yZHREREtf7wtcmtuBPcr0i6gsoTKN7b7FgiIiIi+ookwb2A7dFLMHbfpRhKREREREtIEhx1bbj5Btw4vfe+7RERERHRVS33dIiIiIiIiCTBEREREdFykgRHRERERMvJI9KirhUHj/CEtXeq3zEiIiL6rSse/k2zQ2hIo49Iy05wRERERLScJMERERER0XKSBPcDks6RtEk5/mKz44mIiIjo7ZIE93GSBtj+tO37S1OXk2BJA5ZyWBERERG9WpLgXkjScZKmlOPTJd1Qjt8n6UJJL0s6UdLdwHhJN0kaJ+lkYIikWZIuKmM+Lume0nZ2W8Lbfo4mXWpEREREUyQJ7p1uAXYux+OAYZIGATsBtwLLA3Ntb2f7trZBto8HXrM9xvYkSRsDBwI72h4DLAQmle4154iIiIhoBfna5N5pBjBW0nBgPjCTSjK8MzCFSjJ7WQPzvA8YC0yTBDAEeKac63QOSZOByQBDBg5erIuIiIiI6K2SBPdCthdIehw4GLgDmA1MBNYDHgBet72wgakEXGD7CzXOdTqH7anAVKg8J7hrVxARERHRu6Ucove6BTi2/L4VOAyY5frfbrKglE4AXA8cIGlVAEmjJL2ruwKOiIiI6CuSBPdetwKrA3fafhp4vbTVMxWYLemi8sSILwPXSpoNXFfmjIiIiGhp+drkqCtfmxwRERH52uSIiIiIiD4uSXBEREREtJw8HSLqWm+z9bliet94CyQiIiKiEdkJjoiIiIiWkyQ4IiIiIlpOkuCIiIiIaDmpCY66Hp/3CAdvvF+zw4iIiIgmOu+By5sdwlKVneCIiIiIaDlJgiMiIiKi5SQJBiSdIOnYLo7ZW9Lxi7ne45JWrnp9tqQduzD+5S6ud5ikT3ZlTERERER/lprgxSBpoO2rgKuW0pTbAUcspbn+ie2zumvuiIiIiL6oJZPgsit6LGBgNvBI1bn1gB8AqwCvAp+x/aCk84HngK2AmZLmAONsHylpNeAsYN0yzeG275B0JbAWMBg4w/bUGrFsDDxse6GkzwCTgWWB/wM+YftVSesAP6Py97q6auwE4GvA08AY4HJgDnA0MAT4oO1HJJ0AvGz7O5JuAu4GJgIjgUNt37rYNzMiIiKiD2q5cghJmwJfAt5re0sqCWO1qcBRtsdSSZR/WHVuA2BX2//ebsz3gJvLfFsD80r7IWWeccAUSSvVCGkP3kpsL7e9TZnnAeDQ0n4G8CPb2wB/aTe+7Ro2Bz4BbGB7W+Ac4KgObsPA0ucY4Ksd9ImIiIjot1pxJ/i9wKW2nwWw/ZwkACQNA3YALmlrA5arGnuJ7YUdzPnJMt9C4MXSPkXSvuV4LWB94G/txv4LcHA53kzS16ns0A4DrintOwL7l+OfAqdUjZ9m+6kS/yPAtaV9DpXd3lrannEyAxhdq4OkyVR2pVl+4JAOpomIiIjom1oxCRaVMohalgFesD2mg/OvNLxIpVRhV2B8KWm4iUpZRHWfocBI20+WpvOplDDcJ+lTwISq7h3FPL/qeFHV60V0/Pdt67Owoz6ldGMqwMpDVuxo7YiIiIg+qeXKIYDrgQ+3lSZIGtV2wvbfgcckfaick6QtG5zz8DJmgKQVgBHA8yUB3gjYvsa4icCNVa+HA09JGgRMqmq/HfhIOa5uj4iIiIjF0HJJsO15wDeAmyXdB5zWrssk4NBybh6wTwPTHg1MLB+WmwFsSqXOd6Ck2cBJwF01xlXXAwP8F5UPrV0HPNhu/n+TNI1Kch0RERERS0B23uluFkkzge1sL2h2LJ1ZeciK/tfRHZUXR0RERCvoK1+bLGmG7XH1+rViTXCvYXvrZscQERER0YparhwiIiIiIiI7wVHX6E3X47zpfeMtkIiIiIhGZCc4IiIiIlpOkuCIiIiIaDlJgiMiIiKi5aQmOOr607xH+fymH6nfMSIiIvqt0+b9otkhLFXZCY6IiIiIlpMkOCIiIiJaTr9PgiXdJKnut4Y0MM87JV3aQL+PSvpSF+ZdKvG1m/OLS3O+iIiIiP6m3yfBS4vtJ20f0EDX3YGruzueOrqUBKsi/xYiIiKiZfS6xEfSlZJmSJonaXJp213STEn3Sbq+tG0r6Q5J95bfG5b2IZJ+IWm2pIuBIVVz7ybpzjLXJZKGlfbHJX2znJsuaWtJ10h6RNJhpc9oSXPL8QBJ35E0p6xzVGkXMAaYKWl5SedKmlZi3KeB+F6uOj5A0vnleDVJV5Trv0/SDp3cq5OBIZJmSbqotH1e0tzyc0zV9Twg6YfATGCtpfuXjIiIiOi9euPTIQ6x/ZykIcA0Sf8D/Bh4j+3HJI0q/R4sbW9K2hX4JrA/cDjwqu0tJG1BJcFD0srAl4Fdbb8i6T+BzwMnlvn+ZHu8pNOB84EdgcHAPOCsdjFOBtYBtirrt8W0FXCfbZeSiBtsHyJpJHCPpN8Bn60VXx3fA262va+kAcCwDu7VZbaPl3Sk7THluscCBwPbAQLulnQz8DywIXCw7SMaiCEiIiKi3+iNSfAUSfuW47WoJJy32H4MwPZz5dwI4AJJ6wMGBpX291BJGrE9W9Ls0r49sAlwe2XDlmWBO6vWvar8ngMMs/0S8JKk10sSW21X4Czbb7aLaXfgt+V4N2BvSceW14OBtTuJrzPvBT5ZxiwEXizt7e/V+sDf2o3dCbjC9isAki4Hdi7X+wfbd9VasOwsTwYYPmhoAyFGRERE9B29KgmWNIFKgjne9quSbgLuo7Jj2d5JwI1ld3Q0cFPVOdeaHrjO9kc7WH5++b2o6rjtdfv7pA7W2I3KbnRbn/1tP/S2gZUEvNbY9u2DO+jTNs8E/vle1RqjTqZ5paMTtqcCUwFWGzKqo3gjIiIi+qTeVhM8Ani+JHUbUdm9XQ7YRdI6AFWlByOAP5fjT1XNcQswqfTdDNiitN8F7Cjp3eXcUEkbLGac1wKHSRrYFpOkEcBA2207sdcAR5U6YSRtVSc+gKclbVw+pLZvVfv1VMo82uqRV6D2vWqzQFLbzvgtwAfL9S5f5r11Ma87IiIiol/obUnw1cDAUiJwEpXE9a9U3pa/XNJ9wMWl77eBb0m6HRhQNcePgGFljuOAewBs/5VKsvzzcu4uYKPFjPMc4I/A7BLTx4D3A7+r6nMSlRKN2eUDdSd1Fl9xPPBr4Abgqar2o4GJkuYAM4BNqX2v2kwt615keyaVGud7gLuBc2zfu5jXHREREdEvyM473UuDpHOoJJg1a2z7stWGjPKkdXdrdhgRERHRRH3la5MlzbBd9zsYelVNcF9m+9PNjiEiIiIiGtPbyiEiIiIiIrpddoKjrrU2XZfTpveNt0AiIiIiGpGd4IiIiIhoOUmCIyIiIqLlJAmOiIiIiJaTmuCo66l5j/H1zSY1O4yIiIjoIV+ee1GzQ+h22QmOiIiIiJaTJDgiIiIiWk6S4IiIiIhoOUmC+yBJIyUdUfV6gqRfNzOmiIiIiL4kSXDfNBI4om6vBknKByQjIiKipSQJ7gMkfV7S3PJzDHAysJ6kWZJOLd2GSbpU0oOSLpKkMnaspJslzZB0jaTVS/tNkr4p6Wbg6OZcWURERERzZAewl5M0FjgY2A4QcDfwcWAz22NKnwnAVsCmwJPA7cCOku4GzgT2sf1XSQcC3wAOKdOPtL1LB+tOBiYDjBg0tHsuLiIiIqJJkgT3fjsBV9h+BUDS5cDONfrdY/uJ0mcWMBp4AdgMuK5sDA8Anqoac3FHi9qeCkwFWGPISl7iq4iIiIjoRZIE935qsN/8quOFVP62AubZHt/BmFeWJLCIiIiIvio1wb3fLcAHJQ2VtDywL5Vyh+ENjH0IWEXSeABJgyRt2n2hRkRERPQN2Qnu5WzPlHQ+cE9pOsf2DEm3S5oL/Bb4TQdj35B0APA9SSOo/L3/G5jXA6FHRERE9FqyU+4ZnVtjyEo+fL3dmx1GRERE9JAvz72o2SEsNkkzbI+r1y/lEBERERHRcpIER0RERETLSU1w1LX6puvw5el9922RiIiIiPayExwRERERLSdJcERERES0nJRDRF3PzHuM72/6iWaHERERET3kyHk/bXYI3S47wRERERHRcpIER0RERETLSRIcERERES0nSXAVSS9XHe8p6feS1q7R72RJT0h6oV37cZIekHSfpOskrdWFtT8vaXAX4x3YPoYujj9E0jsWd3xEREREX5UkuAZJ7wPOBHa3/ccaXf4H2L5G+3Rga9tbAlcBJ3dh2c8DXUqCl4JDgCTBERER0XKSBLcjaWfgx8Beth+p1cf2ncBfarTfYPu18vIuYM0a8w+X9NuyWzxX0gGSPgesCtwq6Xftd3glfUTSOeV4PUl3S5oGnNBu7uMl3SNptqSvlLZ3l3X+n6R5Ze3Bkg4ExgAXS5oladku36yIiIiIPipJ8NstR2WX94O2H1zCuQ4FflujfU/gcdtb2t4MuM726cAzwM62d60z75nAGba3Af7a1ihpT2BtYDsqye0OknYopzcE/tv2psBrVK7vYmAWcKDtMbbfqF5E0mRJ0yVNf3nh/C5eekRERETvliT47RYAd1BJYBebpIOAzYHTapyeDexe6op3tP1iF6cfD1xcjqsf4rcbsAdwLzATeDewQTn3f7bnlOMZwOh6i9ieanuc7XHDBizXxRAjIiIierckwW+3CPgwsI2kLwJIWraUC8xqKzHojKTdgeOAfdrvrgLYfgAYB8wDTm1bp0YcqnpdXSvs8vNPSwNfL7u6Y2y/2/b55Vz1Vu5C8iUpERER0eKSBLdj+1XgA8AkSYfafqMqsTyxs7GSxgE/APa2/WwHfdYAXrb9Uyo7xVuXUy8Bw0sMi4DnJa0vaRlg36op7qKSqANMqmq/BjhU0vJlnTUlrVzncv+xZkREREQrSRJcg+3ngN2BL0vap/15SacBjwMrlEelfbmc+g6wPHBZ2Tm+osb0WwLTJM2ismP8zdI+FfidpN+V1/8JXA1cDzxRNX4K8DlJ9wDDqmL+X+BS4C5Jc4BfVp/vwHnAOflgXERERLQa2bXeWY94y9pDVvJx6+7Z7DAiIiKihxw576f1O/VSkmbYHlevX3aCIyIiIqLlJAmOiIiIiJaTpwREXatuug5HTu+7b4tEREREtJed4IiIiIhoOUmCIyIiIqLlpBwi6npu3mNctOnHmx1GRERE9IBJ8y5sdgg9oqGdYEn7Sfq9pBcl/V3SS5L+3t3BRURERER0h0Z3gr8N/Gv5yt+IiIiIiD6t0Zrgp5MAR0RERER/0WgSPF3SxZI+Wkoj9pO0X2cDJI2UdMSSBCfpU5K+vyRzdAdJZ0vacSnM8yFJD0i6UdIYSflatoiIiIge0GgSvALwKrAb8K/l5wN1xowE/ikJljSgKwH2UtsBd9Xr1MC1HgocYXsiMAZIEhwRERHRAxqqCbZ98GLMfTKwnqRZwALgZeApKsneJpKuBNYCBgNn2J4KIOlg4Aul78PA/NK+CnAWsHaZ/xjbt0vaBTijLVTgPcBY4GvA02W9y4E5wNHAEOCDth+R9K/Al4Flgb8Bk2w/LemEss665fd/2/5eiWNj4GHbCyW9u8S0CrAQ+FC5pq/Wu1ZJXwF2AtaR9L/A/sAQSTsB3wL+0pPXFREREdFKGkqCJa0JnAnsSCUhuw042vYTnQw7HtjM9hhJE4DflNePlfOH2H5O0hBgmqTLqCRtX6OS7L0I3AjcW/qfAZxu+zZJawPXABsDxwL/VhLiYcDrpf+W5fxzwKPAOba3lXQ0cBRwTLmO7W1b0qeB44B/L+M3AiYCw4GHJP3I9gJgD+Dq0uci4GTbV0gaTGVnfS1g23rXavtESe8FjrU9XdJ9wDjbR5Z7/qsevq6IiIiIltHo0yHOA35GZacT4OOl7f1dWOueqqQQYIqkfcvxWsD6wDuAm2z/FUDSxcAGpc+uVHZV28avIGk4cDtwmqSLgMttP1H6TLP9VJnnEeDaMm4OlSQQYE3gYkmrU0nAq+P7je35wHxJzwCrAU8A/wIcXNZew/YVALZfL2s1eq1/q3O/evq63kbSZGAywEqDhtYJNSIiIqJvabQmeBXb59l+s/ycT6UEoCteaTsoO8O7AuNtb0llt3dwOe1OYh1ve0z5WcP2S7ZPBj5NpRzgLkkblf7zq8Yuqnq9iLeS/zOB79veHPhsVQztxy8EBkoaCoy0/SQgOtbotXaoJ6+rg/Wn2h5ne9wKA+qGGxEREdGnNJoEPyvp45IGlJ+PU38n8yUqb7nXMgJ43varJbnbvrTfDUyQtJKkQby18wyVHc8j215IGlN+r2d7ju1TgOlU3u5v1Ajgz+X4oAb6T6RSooHtvwNPSPpgiWO5kiTXWqPWtbb3tvvVw9cVERER0VIaTYIPAT5M5cNaTwEHlLYO2f4bcLukucCp7U5fTWVndTZwEuVJC+Vt/hOAO4HfATOrxkwBxkmaLel+4LDSfoykuaWm9jXgtw1eE2WtSyTdCjzbQP/qemCAT1ApdZgN3EGlnKO9mtdaw41Uyj1mSTqQnr2uiIiIiJYiu6Pqg2hP0kxgu1b7INm6Q1bySevu0ewwIiIiogdMmndhs0NYIpJm2B5Xr1+nH4yTdJztb0s6kxq1uranLEGMfY7trZsdQ0REREQsuXpPh2j7quTp3R1IRERERERP6TQJtv2rcviq7Uuqz0n6UI0h0Q+N2nQdJk3v22+NRERERFRr9INxX2iwLSIiIiKi16tXE7wHsCewhqTqr9ddAXizOwOLiIiIiOgu9WqCn6RSD7w3MKOq/SXgc90VVPQuL97/KP+79UebHUZERET0gD1n/rzZIfSIejXB9wH3SfpZqz0WLCIiIiL6r3o7wW1GS/oWsAlVX8Fre91uiSoiIiIiohs1+sG484AfUakDngj8BPhpdwUVEREREdGdGk2Ch9i+nso3zP3B9gnAe7svrMUjaaSkI6pev1PSpUs45wxJyy55dD1P0h0N9Hm5J2KJiIiI6E0aTYJfl7QM8HtJR0raF1i1G+NaXCOBfyTBtp+0fcDiTiZpNPBn228seWg9R9IAANs7NDuWiIiIiN6o0ST4GGAoMAUYC3wCOKjeIEkfl3SPpFmSzpb0Lkm/l7SypGUk3Spptw7r7YwPAAAgAElEQVT6Dijtu0uaKek+SdeXthMkHVu1ztySsJ4MrFfmOFXSaElzS5+7JW1aNeYmSWMlLS/pXEnTJN0raZ+qS9gDuLr0f1nSKWVn+HeSti1zPCpp79JndLmmmeVnh9I+ofS9VNKDki6SpHLuK2XtuZKmVrVvI2m2pDvLtbRdx4Dyelo5/9mqNW6U9DNgTlvM5fcwSdeXmOa0u8aIiIiIltNQEmx7mu2Xgb8DU2zvZ/uuzsZI2hg4ENjR9hhgIbALcApwFvDvwP22r+2g7yRJqwA/Bva3vSVQ71vqjgcesT3G9n+0O/cL4MMlttWBd9qeAXwJuMH2NlTqnU+VtHwZszslCQaWB26yPZbKI+K+Drwf2Bc4sfR5Bni/7a3L9VQ/W3krKv+Z2ARYF9ixtH/f9ja2NwOGAB8o7ecBh9keX+5Hm0OBF0u82wCfkbROObct8CXbm7S79teBfUtcE4HvtiXbEREREa2ooadDSBpHJSkbXl6/CBxSksiOvI/KrvG0km8NAZ6xfUL5yuXDgDGd9QW2B26x/RiA7ee6dHVv90vgOuCrVJLhtq+B3g3Yu2pneTCwtqRHgDVtP1ra3+CthHgOMN/2AklzgNGlfRDwfUltifwGVevfY/sJAEmzypjbgImSjqOy0z4KmCfpVmC47baa3p/xVnK8G7CFpLYyjxHA+iW+e9ruVTsCvinpPcAiYA1gNeAvHd0sSZOByQCrDBraUbeIiIiIPqnRR6SdCxxh+1YASTtRSYq36GSMgAtsv+3rlSUNBdYsL4dR2VXtqO/egGvM/SZv38UeXKPP29j+s6S/SdqCyi7tZ6vi3N/2Q+3Wfh+VJLXNAtttsSwC5pd5F0lqu4+fA54GtizxvV41fn7V8UJgoKTBwA+Bcbb/JOmEci2d7dIKOMr2Ne3inQC80sGYScAqwNiSuD9OnXtmeyowFWD9oaNq/Q0iIiIi+qxGa4JfakuAAWzfRiV57cz1wAGSVgWQNErSu6iUQ1wEfIVKqUNnfe8Edml7u1/SqNL/cWDr0rY10FYO8BJlt7oDvwCOA0bYnlPargGOqqrF3aq07w78ts41tjcCeMr2Iip10wPq9G9LRJ+VNAw4AMD288BLkrYv5z9SNeYa4HBJg0q8G1SVb3QW1zMlAZ4IvKvhK4qIiIjohxpNgu8pH1abIGkXST8EbpK0dUlC/4nt+4EvA9dKmk2lFGE0lTrWU2xfBLwh6eAO+q5u+69U3pK/XNJ9wMVl+suAUaWs4HDg4bLm34Dby4fMTq0R1qVUEspfVrWdRKWMYXb58NlJpX0CcHOD96fND4GDJN1FpRSio51ZSrwvUPmPwBzgSmBa1elDgamS7qSy+/tiaT8HuB+YWeI9m/o7+hcB4yRNp7Ir/GBXLioiIiKiv9Fb7/B30km6sZPTtt3rnhm8JCStCfzY9h5NjGFY+TAiko6n8p+Co5sRy/pDR/mMjf6lGUtHRERED9tz5s+bHcISkTTD9rh6/RqqCbY9cclD6jvKB9ialgAXe0n6ApW/0R+ATzU3nIiIiIj+o9GnQ3ylVrvtE2u1x5KzfTFvlX9ERERExFLU6NMhqmtbB1N5XNcDSz+c6I1GbLIue07v22+NRERERFRrtBziu9WvJX0HuKpbIoqIiIiI6GaNPh2ivaFUvvUsIiIiIqLPabQmeA5vfWnFACpfvJB64IiIiIjokxqtCf5A1fGbwNO23+yGeKIXevmhR7ltlwObHUZERER0o51ubq3P4zdaDrE68JztP9j+MzBY0nbdGFdERERERLdpNAn+EfBy1etXS1tERERERJ/TaBIsV321nO1FNF5KEQ2Q9HL9XhERERGxNDSaBD8qaYqkQeXnaODR7gwsepakAc2OISIiIqKnNJoEHwbsAPwZeALYDpjcXUG1MknDJF0vaaakOZL2Ke3HSZpSjk+XdEM5fp+kCyUdKun0qnk+I+m0cvxxSfdImiXp7LaEV9LLkk6UdDcwvscvNiIiIqJJGkqCbT9j+yO2V7W9mu2P2X6mu4NrUa8D+9reGpgIfFeSgFuAnUufccAwSYOAnYBbgV8Ae5c2gIOB8yRtDBwI7Gh7DLAQmFT6LA/Mtb2d7dt64NoiIiIieoWGkmBJF0gaWfV6RUnndl9YLU3ANyXNBn4HrAGsBswAxkoaDswH7qSSDO8M3Gr7FeAG4AOSNgIG2Z4DvA8YC0yTNKu8bvuik4XAZTWDkCZLmi5p+gsL5nfTpUZEREQ0R6MfbtvC9gttL2w/L2mrboqp1U2i8mUkY20vkPQ4MLjq+GDgDmA2lZ3i9YAHythzgC8CDwLnlTYBF9j+Qo21Xre9sFYQtqcCUwE2Gj7KtfpERERE9FWN1gQvI2nFtheSRpGnQ3SXEcAzJemdCLyr6twtwLHl961UarVntT25w/bdwFrAx4CflzHXAwdIWhUqfztJ1XNGREREtJxGE9nvAndKuqS8/hDwje4JqeVdBPxK0nRgFpVd3Ta3Al8C7rT9iqTXS1u1XwJjbD8PYPt+SV8GrpW0DLAA+DfgD918HRERERG9VkNJsO2fSPo/KjWoi4CDbd/ZrZG1GNvDyu9n6eBJDbavBwZVvd6gRredgNOrG2xfDPzTdyG2rRkRERHRahr9YNzRwNnASsCqwNmSjurOwKJrJI2U9DDwWkmWIyIiIqIDjZZDHApsX55AgKRTqDyd4MzuCiy6pnxwsdbOcERERES002gSLCqP02qzsLRFCxi24brsdPM/VVNERERE9FmNJsHnAXdLuqK8/iDw/7onpIiIiIiI7tXoB+NOk3QTlQ9dicoH4+7tzsAiIiIiIrpLw8/6tT0TmNmNsURERERE9Ih84UXU9dojjzJ73w81O4yIiIjoRltccUn9Tv1Io98YFxERERHRbyQJjoiIiIiW07QkuHy5wxGLOfZ8SQd0ccw5kjbp5Pzeko5fnHiq5jhb0o5LMkeNOTuNOyIiIiK6rpk7wSOBxUqCF4ftT9u+v5PzV9k+eQmX2Q64awnneJt6cS8pSakLj4iIiJbTzCT4ZGA9SbMknSrpPyRNkzRb0tfaOkn6ZGm7T9JPq8a/R9Idkh5t2xWWNEHSTZIulfSgpIskqZy7SdK4cry7pJllzutL26ckfb8c/6ukuyXdK+l3klYr7SdIOrfM9aikKVVxbgw8bHuhpPUkXS1phqRbJW1U+pwv6Xs14l5G0g8lzZP0a0n/W3WuOu6XJX2jxH1XVVyrSLqs3L9pbbvRkpYv8U4r17JP1bVeIulXwLVL+w8bERER0ds1cxfweGAz22Mk7QYcAGxL5TnEV0l6D/A34EvAjraflTSqavzqVJ5bvBFwFXBpad8K2BR4Ergd2BG4rW2QpFWAHwPvsf1Yuznb3Ebla6It6dPAccC/l3MbAROB4cBDkn5kewGwB3B16TMVOMz27yVtB/wQeG8nce8HjAY2B1YFHgDOrRHX8sBdtr8k6dvAZ4CvA2cAp9u+TdLawDXAxuXe3WD7EEkjgXsk/a7MNR7YwvZzNdaJiIiI6Nd6y1vhu5Wfti/gGAasD2wJXGr7WYB2CduVthcB97ftiBb32H4CQNIsKsnlbVXntwdusf1YjTnbrAlcLGl1YFngsapzv7E9H5gv6RlgNeAJ4F+AgyUNA3YALimb0ADL1Yl7J+CS0v4XSTd2cJ/eAH5djmcA7y/HuwKbVK23gqThVO7p3pKOLe2DgbXL8XWdJcCSJgOTAVYfMrSjbhERERF9Um9JggV8y/bZb2uslBu4gzHz242v1b6Qf75GdTJnmzOB02xfJWkCcEJn80saCoy0/aSkFYAXbI/pQtyq1bGGBbbbYq++tmWA8bZfq+5cSkH2t/1Qu/btgFc6W8j2VCo72my64qh69ysiIiKiT2lmTfBLVEoKoPL2/SFlFxVJa0haFbge+LCklUp7rdKFrroT2EXSOp3MOQL4czk+qIE5JwI3Atj+O/CYpA+V+SVpyzrjbwP2L7XBqwETGliz2rXAkW0vJLUl4NcAR1XVRW/VxXkjIiIi+qWmJcG2/wbcLmkulbf1fwbcKWkOlTrZ4bbnAd8AbpZ0H3DaUlj3r1Te5r+8zHlxjW4nUClnuBV4toFpq+uBASYBh5b55wH71Bl/GZWSirnA2cDdwIsNrNtmCjCufIDwfuCw0n4SMAiYXe7zSV2YMyIiIqLf0lvvrsfikjQT2K58QG5x5xhm++Wy630PlQ8D/mWpBbkENl1xlH8+4X3NDiMiIiK6UX/52mRJM2yPq9evt9QE92m2t14K0/y6PMFhWeCk3pIAR0RERPRHSYJ7CdsTmh1DRERERKtIEhx1DVlv3X7zFklEREQENPfpEBERERERTZEkOCIiIiJaTpLgiIiIiGg5qQmOul5//FEePORDzQ4jIiIiuslG57beZ3+yExwRERERLSdJcERERES0nF6bBEs6QdKx5fh8SQd00zpXS1qjG+ZtOGZJEyTt0J1r1Bg7RtKeizM2IiIioq/rtUlwT5A0BBhl+88N9u+uGuoJQJeT4CU0BkgSHBERES2pR5NgSZ+UNFvSfZJ+WtreJen60n69pLXrzDFW0s2SZki6RtLqpX2bMsedkk6VNLe0Dyivp5Xzn62abgJwU+m3p6QHJd0m6XuSfl3aT5A0VdK1wE8kjZZ0q6SZ5WeH0k+Svi/pfkm/AVbtIP4ppc9sSb+QNBo4DPicpFmSdm6/wyvp5XprdHJfbpJ0iqR7JD1c5l8WOBE4sKx5YEN/wIiIiIh+oseeDiFpU+BLwI62n5U0qpz6PvAT2xdIOgT4HvDBDuYYBJwJ7GP7ryV5+wZwCHAeMNn2HZJOrhp2KPCi7W0kLQfcLula248BewBXShoMnA28x/Zjkn7ebumxwE62X5M0FHi/7dclrQ/8HBgH7AtsCGwOrAbcD5xb4zKOB9axPV/SSNsvSDoLeNn2d8p1HtrBbay5Rp37AjDQ9ral/OGrtneV9BVgnO0jO1grIiIiot/qyUekvRe41PazALafK+3jgf3K8U+Bb3cyx4bAZsB1kgAGAE9JGgkMt31H6fcz4APleDdgi6qd1RHA+sBjwI7AscAmwKMlMYZKYju5at2rbL9WjgcB35c0BlgIbFDa3wP83PZC4ElJN3RwDbOBiyRdCVzZybXW0tEaNe9L1bjLy+8ZwOhGFpI0mXIP3rn80C6GGREREdG79WQSLMAN9Ousj4B5tse/rVFasc6Yo2xf027MusCfbL+hkjl24pWq488BTwNbUikneb3B2NvsRSWZ3Rv4r7JD3t6bZW5KbMvWWaPmfakyv/xeSIN/c9tTgakAm608qpHrioiIiOgzerIm+Hrgw5JWAqgqh7gD+Eg5ngTc1skcDwGrSBpf5hgkaVPbzwMvSdq+9PtI1ZhrgMNLyQCSNpC0PJVSiKtLnweBdUt9LkBnNbIjgKdsLwI+QWXXFeAW4COlBnl1YGL7gZKWAdayfSNwHDASGAa8BAyv6vo4lRIMgH2o7D53tkbN+9LJNVBjzYiIiIiW0WNJsO15VOpUb5Z0H3BaOTUFOFjSbCpJ5dGdzPEGcABwSpljFm89VeFQYKqkO6nsjL5Y2s+hUjs7s3xY7mwqu6G7U5LgUupwBHC1pNuo7PS2jW/vh8BBku6iUgrRtkt8BfB7YA7wI+DmGmMHABdKmgPcC5xu+wXgV8C+bR+MA34M7CLpHmC7emvUuS8duRHYJB+Mi4iIiFYku3+80y1pmO22pygcD6xuu2ZC3fYBOdvj2o8v5Qc/AH5v+/SeiL2322zlUb507/c1O4yIiIjoJv3pa5MlzajO8TrSn54TvFfZ1ZwL7Ax8vaOOtufXuDmfkTQLmEel5OHs7gs1IiIiIpqpJz8Y161sXwxcvATjTwey8xsRERHRAvpNEhzdZ/DodfvV2yQRERER/akcIiIiIiKiIUmCIyIiIqLlJAmOiIiIiJaTmuCoa/4Tj/HIcZ9odhgRERGxFK337Z82O4Smyk5wRERERLScJMERERER0XKSBEdEREREy0kSvBgkDVjC8b2qFru3xRMRERHR3ZIEtyNptKQHJV0gabakSyUNlfS4pK9Iug34kKQxku4qfa6QtGIZv01pu1PSqeVrnJH0KUmXSPoVcK2kYZKulzRT0hxJ+7Rb/xxJcyVdJGlXSbdL+r2kbUu/E0qM15bY9pP07TLX1ZIGlX5jJd0saYakayStXtpvkvRNSTcDRzfhVkdEREQ0TZLg2jYEptreAvg7cERpf932TrZ/AfwE+M/SZw7w1dLnPOAw2+OBhe3mHQ8cZPu9wOvAvra3BiYC35Wk0u/dwBnAFsBGwMeAnYBjgS9WzbcesBewD3AhcKPtzYHXgL1KInwmcIDtscC5wDeqxo+0vYvt7y7WXYqIiIjoo/I2eG1/sn17Ob4QmFKOLwaQNIJKAnlzab8AuETSSGC47TtK+8+AD1TNe53t58qxgG9Keg+wCFgDWK2ce8z2nLLWPOB625Y0BxhdNd9vbS8o7QOAq0t7W78Ngc2A60p+PQB4qmr8xR3dAEmTgckA71xh+Y66RURERPRJSYJrcwevX6kzTnXOV4+fBKwCjC2J7OPA4HJuflW/RVWvF/H2v9l8ANuLJC2w7Xb9BMwru9L14nkb21OBqQCbv2Ol9vcjIiIiok9LOURta0tqSxw/CtxWfdL2i8DzknYuTZ8Abrb9PPCSpO1L+0c6WWME8ExJgCcC71p64f/DQ8AqbdciaZCkTbthnYiIiIg+JUlwbQ8AB0maDYwCflSjz0HAqaXPGODE0n4oMFXSnVR2Yl/sYI2LgHGSplPZFX5wKcYPgO03gAOAUyTdB8wCdlja60RERET0NXrrHfSAytMZgF/b3mwxxw+z/XI5Ph5Y3XaffvrC5u9YyVd+cs9mhxERERFLUX/92mRJM2yPq9cvNcFL316SvkDl3v4B+FRzw4mIiIiI9pIEt2P7cSpPVFjc8RfTyVMXIiIiIqL5kgRHXcutuU6/fcskIiIiWlM+GBcRERERLSdJcERERES0nCTBEREREdFyUhMcdb3xlz/yh5P/rdlhRERExFLyruN/0OwQmi47wRERERHRcpIER0RERETLSRIcERERES2nzyTBkk6QdOxijBst6WNLsO7VktaQ9LiklRd3nnbxzC3HYyTtWXVusa4xIiIiIrqmzyTBS2A00KUkWNLA8nsIMMr2n7shLoAxwJ51ey0lbdcVERHx/9u782i5qjL949/HEEwCMRFEmQnKJEJISECQwaCRhTJEBDu0tBDRFaUFnNCmxVYbmkFhdSMgrUFpQPnZ/FBABjWEeRIJCSEhDOICXCIoIAgEJATy9B9nXyjKe2/VDeHWvfc8n7Wy6tQ+e+/znp0D9WbXPqci6m5AJ8GSjpZ0r6Qrgc0byidIukXSQkkXSXpzKd9E0pWS7pA0X9I7gBOBXSQtkPQFSSMk/Y+kRZJul7RbaTtD0gWSLgWuKIeaAlzbENLhpd9FkrYo7VaTdJakuaW/aaV8nKQbSv35kt7TdG6rAscA00ts08uuLSVdK+l+SUf0MC57lD7vkHRVKdte0s0lhpslbd7TeUn6col3oaR/X6G/nIiIiIhBbMDODEqaBBwATKSKcz4wr+w+Fzjc9nWSjgG+AXweOA840fZFkkZQJflHAUfa3qv0+yUA21uXRPYKSZuVfncExtt+orz/IHBxQ1iP295W0j8DRwKfAo4GrrZ9iKSxwK0laX8U+IDt5yVtCvwEmNzVke0XJH0dmGz7sBLbN4EtgN2A0cC9kv7b9rKGcVkLOBPY1fYDktYou+4pZS9KmgocD+zXfF6Sdgc2BbYHBFwiaVfb1zeN/0xgJsB6Y1fv6a8pIiIiYlAasEkwsAtwke3nACRdUl7HAGNtX1fqnQNcIGk0sJ7tiwBsP1/qN/e7M3BaqXOPpN8DXUnwnIYEGGAnqmS3y4XldR7wkbK9O7BPw1reEcCGwMPA6ZImAC81HKOVy20vBZZKehR4G/BQw/4dgOttP1DOoSveMcA5JeE2MLyhTeN57V7+3F7er06VFL8qCbY9C5gFMH79t7rN2CMiIiIGhYGcBEOVzLXr77LdFaj37MuVpLcDf7D9QsP+peX1JV4ZOwH72b73VQepZnX/DGxDNSP9fJvxLW3YbjxOY/zdjcuxwDW295U0jlcv43i2YVvACba/32Y8EREREUPOQF4TfD2wr6SRZZZ3bwDbTwFPStql1Ps4cJ3tp4GHJH0YQNIbJY0CnqFaWtDY74GlzmZUs7avSmCLDwK/aiPO2VRrhVX6nFjKxwCP2F5eYhzWTdvm2Nrxa+C9kjYux+taDjEG6LqBb0aLeA+RtHppv56kt/YxhoiIiIhBbcAmwbbnA+cDC4CfATc07D4YOEnSQqonLBxTyj8OHFHKbwbWBhYCL5abyL4AnAEMk7So9D+jLD9otgftJcHHUi09WFgefXZsKT8DOFjSLVRLIZ7tpu01VDfCNd4Y1yvbj1Gt1b1Q0h3lHAC+DZwg6Sa6T7i72l8B/D/g12UMfkrfE/GIiIiIQU12lns2k/RG4Cbbk1tWroHx67/Vlx720U6HERERESvJRkd9t9MhvG4kzWsnhxvoa4I7oswMJwGOiIiIGKKSBEdLq6694ZD+F2NERETUz4BdExwRERER8XpJEhwRERERtZMkOCIiIiJqJ2uCo6UXHvsjD33/q50OIyIiIlbQ+p8+vtMhDDiZCY6IiIiI2kkSHBERERG1kyQ4IiIiImqnlkmwpLMl7b+CbYdLmvc6xDRF0mV9qD9B0oca3u8j6aiVHVdERETEUFTLJPg12hm4eWV2KGlFblCcALycBNu+xPaJKy+qiIiIiKFryCTBklaTdLmkOyTdKWm6pK9Lmlvez5KkbtpNknSdpHmSZktap5QfIekuSQsl/W9Dkz2AX0oaJ+keST8o/Z8naaqkmyTdJ2n70s/2km6WdHt53byUz5B0gaRLgSuaYtqu1H97Oa+zynncLmmapFWBY4DpkhaUc50h6fTS/mxJp5bj3d816y3pDZLOkLRY0mWSfrGiM+IRERERg9mQSYKpktOHbW9jeyvgV8Dptrcr70cCezU2kDQcOA3Y3/Yk4CzguLL7KGCi7fHAZxqa7QZcW7Y3Ab4DjAe2AD5GNVN8JND1TLF7gF1tTwS+DjQ+o2RH4GDb72uI6T3A94Bptu8Hjgautr1dOfZJwPDS1/m2J9g+v5vxWKfEshfQNUP8EWAcsDXwqXL8bkmaKek2Sbc9seS5nqpFREREDEpD6TnBi4CTJX0LuMz2DZL2k/QVYBSwBrAYuLShzebAVsCcMkk8DHik7FsInCfpYuBiAEnrAk/Yfq7Uf8D2orJvMXCVbUtaRJVsAowBzpG0KWCqBLbLHNtPNLx/JzAL2N32w6Vsd2AfSUeW9yOADdsYj4ttLwfukvS2UrYzcEEp/5Oka3pqbHtWiYXxG63jNo4XERERMWgMmSTY9m8lTaJaJ3uCpCuAzwKTbf9B0jepEshGAhbb7m5GdE9gV2Af4N8kvQv4IDC7oc7Shu3lDe+X88rYHgtcY3tfSeN4ZRYZ4NmmYz5SYpwIdCXBAvazfe+rApfe3U3MjRpjU9NrRERERK0NmeUQZZb2Ods/Bk4Gti27Hpe0OtDd2td7gbUk7Vj6GC7pXZLeAGxg+xrgK8BYYHXKeuA+hjYG+GPZntGi7l+pku/jJU0pZbOBw7vWM0uaWMqfAUb3MZYbgf3K2uC3AVNa1I+IiIgYkobMTDDVOteTJC0HlgGHAh+mWibxIDC3uYHtF8qNYadKGkM1HqcAvwV+XMoE/BdV0rmp7Xv6GNe3qZZDfBG4ulVl23+WtDfVzXeHUM0knwIsLInwg1TrfK8BjpK0ADihzVh+BrwfuJPqHH8DPNW304mIiIgY/GRnuWc7JO0M/JPtz7SsPIBJWt32EklrArcCO9n+U29txm+0jn/x1U/0T4ARERGx0q3/6eNbVxoiJM2zPblVvaE0E/y6sn0j1XKCwe4ySWOBVYFjWyXAEREREUNRkuCasT2l0zFEREREdFqS4Ghp1bXWq9XXKBERETH0DZmnQ0REREREtCtJcERERETUTpZDREvLnvwzf7rg5E6HERERESto7Y8e2bpSzWQmOCIiIiJqJ0lwRERERNROkuCIiIiIqJ0kwQOIpMmSTn0N7b/a9P7mvtSPiIiIqIskwQOI7dtsH/EaunhVUmv7PX2pHxEREVEXSYL7gaTVJF0u6Q5Jd0qaLmk7STeXslsljZY0RdJlDW3OkjRX0u2SppXyGZIulPQrSfdJ+nYpPxEYKWmBpPNK2ZLyuo6k68u+OyXt0l39iIiIiLrII9L6xx7Aw7b3BJA0BrgdmG57rqQ3AX9ranM0cLXtQySNBW6VdGXZNwGYCCwF7pV0mu2jJB1me0I3x/8YMNv2cZKGAaNs39BLfSTNBGYCrPeWsa/p5CMiIiIGmswE949FwFRJ35K0C7Ah8IjtuQC2n7b9YlOb3YGjJC0ArgVGlHYAV9l+yvbzwF3ARi2OPxf4hKRvAlvbfqZVwLZn2Z5se/Kab1q9vbOMiIiIGCSSBPcD278FJlElwycA+wJu0UzAfrYnlD8b2r677FvaUO8lWszo274e2BX4I/AjSQetwGlEREREDBlJgvuBpHWB52z/GDgZ2AFYV9J2Zf9oSc2J7GzgcEkqdSa2cahlkoZ3c/yNgEdtnwn8ENi2t/oRERERQ13WBPePrYGTJC0HlgGHUs30niZpJNV64KlNbY4FTgEWlkT4QWCvFseZVerPt31gQ/kU4MuSlgFLgINa1I+IiIgY0mS3+lY+6m6bd2zg2Sd+rtNhRERExApa+6NHdjqEfiNpnu3JreplOURERERE1E6S4IiIiIionawJjpaGv/lttfoaJSIiIoa+zARHRERERO0kCY6IiIiI2slyiGjpxTTw/rkAABAgSURBVKf/wmNXnt3pMCIiIqJNa02d0ekQBrzMBEdERERE7SQJjoiIiIjaSRIcEREREbWTJLgXksZJurPTcbRD0hRJl3U6joiIiIjBIElwICk3SEZEREStJAlubZikMyUtlnSFpJGSJki6RdJCSRdJejOApGsl/Zek6yXdLWk7SRdKuk/Sf3R1KOmfJN0qaYGk70saVsqXSPqWpHmSrpS0fenzfkn7lDrjJN0gaX75857mgMtxb5f0dkmrSTpL0txSNq3UmSHpAkmXAlf0y0hGREREDBBJglvbFPiu7XcBfwX2A84F/sX2eGAR8I2G+i/Y3hX4HvBz4LPAVsAMSWtKeicwHdjJ9gTgJeDA0nY14Frbk4BngP8APgDsCxxT6jwKfMD2tqWfUxuDLUnx94Bptu8Hjgautr0dsBtwkqTVSvUdgYNtv++1DlJERETEYJKvwVt7wPaCsj0PeAcw1vZ1pewc4IKG+peU10XAYtuPAEi6H9gA2BmYBMyVBDCSKrEFeAH4VUP7pbaXSVoEjCvlw4HTJXUl0Js1HPudwCxgd9sPl7LdgX0kdf3u8Qhgw7I9x/YT3Z20pJnATID137pm9yMTERERMUglCW5tacP2S8DYNusvb2q7nGq8BZxj+1+7abvMtpvb217esG73C8CfgW2oZvKfb2j/CFWSOxHoSoIF7Gf73sYDSXo38GxPJ2F7FlVCzYTNNnZP9SIiIiIGoyyH6LungCcl7VLefxy4rpf6za4C9pf0VgBJa0jaqA/txwCP2F5ejj2sYd9fgT2B4yVNKWWzgcNVpp0lTezDsSIiIiKGpCTBK+ZgqrW1C4EJvLJetyXbdwFfA64o7ecA6/Th2GcAB0u6hWopxKtmc23/Gdgb+G6Z7T2WagnFwvK4t2P7cKyIiIiIIUmvfPse0b0Jm23sOWd8o3XFiIiIGBDWmjqj0yF0jKR5tie3qpeZ4IiIiIionSTBEREREVE7eTpEtLTKm9as9dcqERERMfRkJjgiIiIiaidJcERERETUTpZDREsvPvcUT9x2eafDiIiIiB6sMXnPTocw6GQmOCIiIiJqJ0lwRERERNROkuCIiIiIqJ0kwYOIpM9LGtXw/heSxr6G/qZIumzlRBcRERExeCQJHmBU6env5fPAy0mw7Q/Z/msf2kdEREQESYIHBEnjJN0t6QxgPvBxSb+WNF/SBZJWl3QEsC5wjaRrSrsHJb2lm/YbSNq9uY/SZg9J90i6EfhIh045IiIioqOSBA8cmwPnAh8APglMtb0tcBvwRdunAg8Du9neraf2ticCzwJfa+5D0gjgTGBvYBdg7df5nCIiIiIGpDwneOD4ve1bJO0FbAncJAlgVeDX7bYv2zv00McWwAO27wOQ9GNgZnedSZrZtW/9tdda0XOKiIiIGJCSBA8cz5ZXAXNs/+MKtu+xD0kTALfTme1ZwCyACVtu2labiIiIiMEiyyEGnluAnSRtAiBplKTNyr5ngNGvoY97gI0lvaPU62uiHRERETEkJAkeYGw/BswAfiJpIVVCu0XZPQv4ZdeNcX3tw/bzVEscLi83xv3+dTmJiIiIiAFOdr7pjt5N2HJTX33uKZ0OIyIiInqwxuQ9Ox3CgCFpnu3JreplJjgiIiIiaidJcERERETUTp4OES2tMmpMvmaJiIiIISUzwRERERFRO0mCIyIiIqJ2kgRHRERERO1kTXC0tHzpczzzu9s6HUZERMSgN3qTlk/uin6SmeCIiIiIqJ0kwRERERFRO7VNgiV9tdMxvB4kHSNpaqfjiIiIiBjIapsEA/2aBEtaaeuve+vL9tdtX7myjhURERExFA3KJFjSQZIWSrpD0o9K2dmS9m+os6S8riPpekkLJN0paRdJJwIjS9l5pd4Xy/47JX2+lI2TdI+kH5Ty8yRNlXSTpPskbV/qrSbpLElzJd0uaVopnyHpAkmXAlc0ncNqki4v53CnpOmlfJKk6yTNkzRb0jql/FpJx0u6Djha0oOS3lD2jZL0B0nDG8dB0naSbi7HuFXSaEnDJJ1UYl0o6dOv419VRERExIA06J4OIeldwNHATrYfl7RGiyYfA2bbPk7SMGCU7RskHWZ7QulzEvAJ4N2AgN+UZPNJYBPgo8BMYG7pb2dgH6rZ5A+XeK62fYikscCtkrpmY3cExtt+oimuPYCHbe9ZYhgjaThwGjDN9mMlMT4OOKS0GWv7vaX+tsB7gWuAvcs5LpPUNU6rAucD023PlfQm4G/AJ4GnbG8n6Y3ATZKusP1Aq7GPiIiIGCoGXRIMvA/4qe3HAbpJLpvNBc4qCebFthd0U2dn4CLbzwJIuhDYBbgEeMD2olK+GLjKtiUtAsaV9rsD+0g6srwfAWxYtuf0EOMi4GRJ3wIuK4n5VsBWwJySzA4DHmloc37T9nSqJPgA4Iym/jcHHrE9F8D20+UcdgfGN8yajwE2BV6VBEuaSZX4s8G6a3cTfkRERMTgNRiTYAHupvxFyvIOVRnkqgC2r5e0K7An8CNJJ9k+t5s+e7K0YXt5w/vlvDJ+Avazfe+rOpXeDTzbXae2f1tmoD8EnCDpCuAiYLHtHXuIpbGvS0q7NYBJwNXdnFN34yTgcNuzezhGV3yzgFkA2269ZXf9RERERAxag3FN8FXAP0haE6BhOcSDVMkgwDRgeNm/EfCo7TOBHwLbljrLyuwwwPXAh8va2tWAfYEb+hDTbODwknwjaWKrBpLWBZ6z/WPg5BLXvcBaknYsdYaX5R9/x/YS4FbgO1QzyS81VbkHWFfSdqWv0eWGutnAoV3nLmmzcs4RERERtTHoZoJtL5Z0HHCdpJeA24EZwJnAzyXdSpUod82aTgG+LGkZsAQ4qJTPAhZKmm/7QElnUyWVAD+wfbukcW2GdSxwSulPVAn5Xi3abA2cJGk5sAw41PYLZZnCqZLGUP39nAIs7qGP84ELyjm+SulrOnCapJFU64GnAj+gWsYxv8T6GNW65oiIiIjakJ1vuqN32269pa+7qHkFSURERPRVfjb59Sdpnu2WAz0Yl0NERERERLwmSYIjIiIionYG3Zrg6H9veOOofH0TERERQ0pmgiMiIiKidnJjXLQk6Rmqx7dF994CPN7pIAawjE/vMj69y/j0LuPTu4xPa0NxjDayvVarSlkOEe24t527LOtK0m0Zn55lfHqX8eldxqd3GZ/eZXxaq/MYZTlERERERNROkuCIiIiIqJ0kwdGOWZ0OYIDL+PQu49O7jE/vMj69y/j0LuPTWm3HKDfGRURERETtZCY4IiIiImonSXC8TNIeku6V9DtJR3Wz/42Szi/7fyNpXP9H2TltjM8MSY9JWlD+fKoTcXaCpLMkPSrpzh72S9KpZewWStq2v2PspDbGZ4qkpxquna/3d4ydJGkDSddIulvSYkmf66ZOba+hNsentteQpBGSbpV0Rxmff++mTm0/v9ocn1p+fuURaQGApGHAd4EPAA8BcyVdYvuuhmqfBJ60vYmkA4BvAdP7P9r+1+b4AJxv+7B+D7DzzgZOB87tYf8HgU3Ln3cD/11e6+Jseh8fgBts79U/4Qw4LwJfsj1f0mhgnqQ5Tf991fkaamd8oL7X0FLgfbaXSBoO3Cjpl7ZvaahT288v2hsfqOHnV2aCo8v2wO9s32/7BeB/gWlNdaYB55TtnwLvl6R+jLGT2hmf2rJ9PfBEL1WmAee6cgswVtI6/RNd57UxPrVm+xHb88v2M8DdwHpN1Wp7DbU5PrVVrokl5e3w8qf5hqfafn61OT61lCQ4uqwH/KHh/UP8/f9kX65j+0XgKWDNfomu89oZH4D9yle1P5W0Qf+ENii0O351tmP5uvKXkt7V6WA6pXxNPRH4TdOuXEP0Oj5Q42tI0jBJC4BHgTm2e7x+avj51c74QA0/v5IER5fu/kXc/C/FduoMVe2c+6XAONvjgSt5ZdYh6n3ttGM+1c98bgOcBlzc4Xg6QtLqwM+Az9t+unl3N01qdQ21GJ9aX0O2X7I9AVgf2F7SVk1Van39tDE+tfz8ShIcXR4CGv/ltz7wcE91JK0CjKE+X/G2HB/bf7G9tLw9E5jUT7ENBu1cX7Vl++murytt/wIYLuktHQ6rX5W1ij8DzrN9YTdVan0NtRqfXEMV238FrgX2aNpV58+vl/U0PnX9/EoSHF3mAptK2ljSqsABwCVNdS4BDi7b+wNXuz4Pmm45Pk3rE/ehWrcXlUuAg8od/jsAT9l+pNNBDRSS1u5anyhpe6r/N/+ls1H1n3LuPwTutv2fPVSr7TXUzvjU+RqStJaksWV7JDAVuKepWm0/v9oZn7p+fuXpEAFUa6QkHQbMBoYBZ9leLOkY4Dbbl1D9T/hHkn5H9S/oAzoXcf9qc3yOkLQP1Z3cTwAzOhZwP5P0E2AK8BZJDwHfoLr5AtvfA34BfAj4HfAc8InORNoZbYzP/sChkl4E/gYcUJcP6GIn4OPAorJuEeCrwIaQa4j2xqfO19A6wDnlKT5vAP6/7cvy+fWydsanlp9f+cW4iIiIiKidLIeIiIiIiNpJEhwRERERtZMkOCIiIiJqJ0lwRERERNROkuCIiIiIqJ0kwRERERFRO0mCIyJiwJE0RdJ7Gt5/RtJBnYwpIoaW/FhGRER0hKRVbL/Yw+4pwBLgZnj5ByEiIlaa/FhGRET8HUkXAxsAI4DvUP1S4sa2v1L2zwAm2T5c0r8BBwJ/AB4H5tk+uYd+r6VKbHei+inb3wJfA1al+pnfA4GRwC3AS8BjwOHA+4Eltk8uffwG2A0YC3zS9g2SRgFnA1tQ/ezrOOCztm9bScMSEUNIZoIjIqI7h9h+QtJIYC5VEnoT8JWyfzpwnKTJwH7ARKrPlPnAvBZ9j7X9XgBJbwZ2sG1JnwK+YvtLkr5HSXpLvfc39bGK7e0lfYjqZ6inAv8MPGl7vKStgAVERPQgSXBERHTnCEn7lu0NgI2B+yXtANwHbE6VFH8O+LntvwFIurSNvs9v2F4fOF/SOlSzwQ+0Gd+F5XUe1YwvwM5Us9bYvlPSwjb7iogayo1xERHxKpKmUM2s7mh7G+B2qmUR5wP/QDXze5Gr9XRagUM827B9GnC67a2BT5fjtGNpeX2JVyZ0ViSWiKipJMEREdFsDNWyguckbQHsUMovBD4M/COvzObeCOwtaYSk1YE9V+BYfyzbBzeUPwOM7mNfN1Il6UjaEti6j+0jokaSBEdERLNfAauU5QTHUt2khu0ngbuAjWzfWsrmUt3gdgdVknwb8FQfjvVN4AJJN1DdVNflUmBfSQsk7dJmX2cAa5W4/wVY2MdYIqJG8nSIiIh4TSStbntJeTrD9cBM2/M7EMcwYLjt5yW9A7gK2Mz2C/0dS0QMfLkxLiIiXqtZZfnBCOCcTiTAxSjgGknDqdYHH5oEOCJ6kpngiIhY6SR9l+pZwI2+Y/t/OhFPRESzJMERERERUTu5MS4iIiIiaidJcERERETUTpLgiIiIiKidJMERERERUTtJgiMiIiKidv4PBXZ80M++gy8AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 720x504 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sb\n",
    "%matplotlib inline \n",
    "\n",
    "plt.figure(figsize=(10, 7))\n",
    "\n",
    "df = age_profession_avg_rating.toPandas()\n",
    "sb.barplot(data = df, x = 'avg_rating', y = 'occupation', ci = None, palette = 'rocket');"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyspark.sql.functions import split, explode\n",
    "\n",
    "age_genres_avg_rating = users.alias(\"t1\")\\\n",
    ".join(ratings.alias(\"t2\"), col(\"t1.user_id\") == col(\"t2.userId\"))\\\n",
    ".join(movies.alias(\"t3\"), col(\"t2.movieId\") == col(\"t3.movieId\"))\\\n",
    ".select(explode(split(col(\"genres\"), \"\\\\|\")).alias(\"genres\"), col(\"age\"), col(\"rating\"))\\\n",
    ".groupBy(col(\"age\"), col(\"genres\"))\\\n",
    ".agg(count(col(\"age\")).alias(\"age_category\"), count(col(\"genres\")).alias(\"genres_category\"), avg(\"rating\").alias(\"avg_rating\")) \\\n",
    ".orderBy(desc(\"avg_rating\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------+------------------+------------+---------------+------------------+\n",
      "|     age|            genres|age_category|genres_category|        avg_rating|\n",
      "+--------+------------------+------------+---------------+------------------+\n",
      "|   45-49|(no genres listed)|           1|              1|               4.5|\n",
      "|   50-55|         Film-Noir|         795|            795| 3.993081761006289|\n",
      "|   25-34|         Film-Noir|        3522|           3522| 3.989068710959682|\n",
      "|     56+|         Film-Noir|         549|            549|3.9799635701275045|\n",
      "|   18-24|         Film-Noir|        1686|           1686| 3.974199288256228|\n",
      "|   35-44|         Film-Noir|        1966|           1966| 3.950915564598169|\n",
      "|   50-55|               War|        3781|           3781|3.8841576302565457|\n",
      "|Under 18|         Film-Noir|         318|            318|3.8726415094339623|\n",
      "|   50-55|       Documentary|         925|            925|3.8567567567567567|\n",
      "|Under 18|       Documentary|         460|            460|3.8434782608695652|\n",
      "|   18-24|               War|        8464|           8464|3.8234286389413987|\n",
      "|   45-49|               War|        4281|           4281|3.8221209997664096|\n",
      "|Under 18|               War|        1775|           1775|3.8180281690140845|\n",
      "|   45-49|         Film-Noir|         890|            890|3.8162921348314605|\n",
      "|   35-44|               War|        9295|           9295| 3.813986013986014|\n",
      "|   25-34|               War|       16668|          16668| 3.809995200383969|\n",
      "|     56+|               War|        2620|           2620|3.7927480916030536|\n",
      "|   25-34|       Documentary|        3673|           3673|3.7704873400490064|\n",
      "|   35-44|       Documentary|        2107|           2107| 3.750830564784053|\n",
      "|   18-24|       Documentary|        2010|           2010|3.7460199004975125|\n",
      "+--------+------------------+------------+---------------+------------------+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "age_genres_avg_rating.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}