P5 segmentation.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b76f7451",
   "metadata": {},
   "outputs": [],
   "source": [
    "# pip install yellowbrick\n",
    "# pour pep8\n",
    "#!pip install pycodestyle pycodestyle_magic\n",
    "# %load_ext pycodestyle_magic\n",
    "#!pip install autopep8\n",
    "# https://stackoverflow.com/questions/47784440/how-can-i-solve-error-in-jupyter-notebook\n",
    "#%load_ext pycodestyle_magic \n",
    "#%pycodestyle_off"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1c05a16d",
   "metadata": {},
   "source": [
    "# IMPORT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ac4c67b3",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "\n",
    "\"\"\"Import des fichiers olist.\"\"\"\n",
    "\n",
    "\n",
    "geo = pd.read_csv('input/archive/olist_geolocation_dataset.csv',\n",
    "                  delimiter=',',\n",
    "                  error_bad_lines=False,\n",
    "                  low_memory=False)\n",
    "cust = pd.read_csv('input/archive/olist_customers_dataset.csv',\n",
    "                   delimiter=',',\n",
    "                   error_bad_lines=False,\n",
    "                   low_memory=False)\n",
    "order_payment = pd.read_csv('input/archive/olist_order_payments_dataset.csv',\n",
    "                            delimiter=',',\n",
    "                            error_bad_lines=False,\n",
    "                            low_memory=False)\n",
    "order_review = pd.read_csv('input/archive/olist_order_reviews_dataset.csv',\n",
    "                           delimiter=',',\n",
    "                           error_bad_lines=False,\n",
    "                           low_memory=False)\n",
    "order = pd.read_csv('input/archive/olist_orders_dataset.csv',\n",
    "                    delimiter=',',\n",
    "                    error_bad_lines=False,\n",
    "                    low_memory=False)\n",
    "\n",
    "order_item = pd.read_csv('input/archive/olist_order_items_dataset.csv',\n",
    "                         delimiter=',',\n",
    "                         error_bad_lines=False,\n",
    "                         low_memory=False)\n",
    "product = pd.read_csv('input/archive/olist_products_dataset.csv',\n",
    "                      delimiter=',',\n",
    "                      error_bad_lines=False,\n",
    "                      low_memory=False)\n",
    "seller = pd.read_csv('input/archive/olist_sellers_dataset.csv',\n",
    "                     delimiter=',',\n",
    "                     error_bad_lines=False,\n",
    "                     low_memory=False)\n",
    "product_cat = pd.read_csv(\n",
    "    'input/archive/product_category_name_translation.csv',\n",
    "    delimiter=',',\n",
    "    error_bad_lines=False,\n",
    "    low_memory=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3fe765ed",
   "metadata": {},
   "source": [
    "# Jointure product,product cat, seller, order_item"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bccf09aa",
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"Jointure des tables: product, product cat, seller, order_item.\"\"\"\n",
    "\n",
    "##############################################################################\n",
    "# fusion product/cat anglais\n",
    "# cat englais\n",
    "product2 = pd.merge(product,\n",
    "                    product_cat, how='outer',\n",
    "                    on='product_category_name')\n",
    "# volume\n",
    "product2['vol'] = product2.product_length_cm * \\\n",
    "    product2.product_height_cm*product2.product_width_cm*(1e-6)\n",
    "# drop\n",
    "product2 = product2.drop(columns=['product_category_name',\n",
    "                                  'product_name_lenght',\n",
    "                                  'product_length_cm',\n",
    "                                  'product_height_cm',\n",
    "                                  'product_width_cm'])\n",
    "product2.columns = ['product_id',\n",
    "                    'prod_desc_l',\n",
    "                    'prod_foto_nb',\n",
    "                    'prod_w',\n",
    "                    'prod_cat',\n",
    "                    'prod_vol']\n",
    "\n",
    "##############################################################################\n",
    "# fusion seller (adresse)/order_item/product\n",
    "product3 = order_item.set_index('seller_id').join(\n",
    "    seller.set_index('seller_id'))\n",
    "product3.columns = ['order_id',\n",
    "                    'order_item_id',\n",
    "                    'product_id',\n",
    "                    'ord_item_ship_lim_dat',\n",
    "                    'ord_item_price',\n",
    "                    'ord_item_fdp',\n",
    "                    'sel_zip',\n",
    "                    'sel_city',\n",
    "                    'sel_state']\n",
    "\n",
    "##############################################################################\n",
    "# fusion product/cat anglais et seller (adresse)/order_item/product\n",
    "product3 = product3.reset_index().set_index(\n",
    "    'product_id').join(product2.set_index('product_id'))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ed09ab18",
   "metadata": {},
   "source": [
    "# Fusion order_payment/order_review/cust/order/geo"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6b647402",
   "metadata": {},
   "outputs": [],
   "source": [
    "import datetime as dt\n",
    "\n",
    "\"\"\"Jointure des tables olist: order et order_payment.\"\"\"\n",
    "\n",
    "pay1 = order_payment.groupby(by=\"order_id\").agg({\n",
    "    'payment_sequential': ['max'],\n",
    "    'payment_installments': ['max'],\n",
    "    'payment_value': ['sum']})\n",
    "pay1.columns = pay1.columns.droplevel(0)\n",
    "pay1.columns = ['pay_seq', 'pay_ins', 'pay_val']\n",
    "\n",
    "##############################################################################\n",
    "# calcul des montant par type de paiement\n",
    "pay4 = order_payment.loc[\n",
    "    order_payment.payment_type == 'credit_card', :].groupby(\n",
    "    by=\"order_id\").agg({'payment_value': ['sum']})\n",
    "pay4.columns = pay4.columns.droplevel(0)\n",
    "pay4.columns = ['pay_val_creditc']\n",
    "pay5 = order_payment.loc[\n",
    "    order_payment.payment_type == 'debit_card', :].groupby(\n",
    "    by=\"order_id\").agg({'payment_value': ['sum']})\n",
    "pay5.columns = pay5.columns.droplevel(0)\n",
    "pay5.columns = ['pay_val_debitc']\n",
    "pay6 = order_payment.loc[\n",
    "    order_payment.payment_type == 'boleto', :].groupby(\n",
    "    by=\"order_id\").agg({'payment_value': ['sum']})\n",
    "pay6.columns = pay6.columns.droplevel(0)\n",
    "pay6.columns = ['pay_val_cash']\n",
    "pay7 = order_payment.loc[\n",
    "    order_payment.payment_type == 'voucher', :].groupby(\n",
    "    by=\"order_id\").agg({'payment_value': ['sum']})\n",
    "pay7.columns = pay7.columns.droplevel(0)\n",
    "pay7.columns = ['pay_val_voucher']\n",
    "pay8 = order_payment.loc[\n",
    "    order_payment.payment_type == 'not_defined', :].groupby(\n",
    "    by=\"order_id\").agg({'payment_value': ['sum']})\n",
    "pay8.columns = pay8.columns.droplevel(0)\n",
    "pay8.columns = ['pay_val_nd']\n",
    "\n",
    "pay1 = pay1.join(pay4)\n",
    "pay1 = pay1.join(pay5)\n",
    "pay1 = pay1.join(pay6)\n",
    "pay1 = pay1.join(pay7)\n",
    "pay1 = pay1.join(pay8)\n",
    "\n",
    "pay1[['pay_val_creditc', 'pay_val_debitc', 'pay_val_cash',\n",
    "      'pay_val_voucher', 'pay_val_nd']] = pay1[[\n",
    "          'pay_val_creditc', 'pay_val_debitc', 'pay_val_cash',\n",
    "          'pay_val_voucher', 'pay_val_nd']].fillna(0)\n",
    "pay1['pay_val_pct_creditc'] = pay1['pay_val_creditc']/pay1['pay_val']\n",
    "pay1['pay_val_pct_debitc'] = pay1['pay_val_debitc']/pay1['pay_val']\n",
    "pay1['pay_val_pct_cash'] = pay1['pay_val_cash']/pay1['pay_val']\n",
    "pay1['pay_val_pct_voucher'] = pay1['pay_val_voucher']/pay1['pay_val']\n",
    "pay1['pay_val_pct_nd'] = pay1['pay_val_nd']/pay1['pay_val']\n",
    "\n",
    "##############################################################################\n",
    "# petit nettoyage\n",
    "pay1 = pay1.loc[pay1.pay_val_pct_nd != 1, :]\n",
    "pay1 = pay1.drop(columns=['pay_val_nd', 'pay_val_pct_nd'])\n",
    "\n",
    "pay1.loc[pay1.pay_val_pct_cash == 1, ['pay_type']] = 'cash'\n",
    "pay1.loc[pay1.pay_val_pct_debitc == 1, ['pay_type']] = 'debit'\n",
    "pay1.loc[((pay1.pay_val_pct_creditc == 1) & (\n",
    "    pay1.pay_val_pct_voucher == 0)), ['pay_type']] = 'credit'\n",
    "pay1.loc[((pay1.pay_val_pct_voucher == 1) & (\n",
    "    pay1.pay_val_pct_creditc == 0)), ['pay_type']] = 'voucher'\n",
    "pay1.loc[((pay1.pay_val_pct_voucher > 0) & (pay1.pay_val_pct_creditc > 0)), [\n",
    "    'pay_type']] = 'mix credit voucher'\n",
    "\n",
    "##############################################################################\n",
    "# autre nettoyage pour données inutiles\n",
    "pay1 = pay1.drop(columns=['pay_val_pct_creditc',\n",
    "                 'pay_val_pct_debitc', 'pay_val_pct_cash'])\n",
    "pay1 = pay1.drop(columns=['pay_val_creditc',\n",
    "                          'pay_val_debitc', 'pay_val_cash'])\n",
    "\n",
    "##############################################################################\n",
    "# on fusionne avec order\n",
    "order1 = order.set_index('order_id')\n",
    "order1 = order1.join(pay1)\n",
    "\n",
    "##############################################################################\n",
    "# \"datetime\"\n",
    "order1.order_purchase_timestamp = pd.to_datetime(\n",
    "    order1.order_purchase_timestamp, errors='coerce')\n",
    "order1.order_approved_at = pd.to_datetime(\n",
    "    order1.order_approved_at, errors='coerce')\n",
    "order1.order_delivered_carrier_date = pd.to_datetime(\n",
    "    order1.order_delivered_carrier_date, errors='coerce')\n",
    "order1.order_delivered_customer_date = pd.to_datetime(\n",
    "    order1.order_delivered_customer_date, errors='coerce')\n",
    "order1.order_estimated_delivery_date = pd.to_datetime(\n",
    "    order1.order_estimated_delivery_date, errors='coerce')\n",
    "\n",
    "##############################################################################\n",
    "# difference de temps (c est vraimeent histoire 2)\n",
    "order1['tim2approv'] = order1['order_approved_at'] - \\\n",
    "    order1['order_purchase_timestamp']\n",
    "order1['tim2carrier'] = order1['order_delivered_carrier_date'] - \\\n",
    "    order1['order_approved_at']\n",
    "order1['tim2deliv'] = order1['order_delivered_customer_date'] - \\\n",
    "    order1['order_delivered_carrier_date']\n",
    "order1['tim2estim'] = order1['order_estimated_delivery_date'] - \\\n",
    "    order1['order_delivered_customer_date']\n",
    "order1['tim'] = order1['order_delivered_customer_date'] - \\\n",
    "    order1['order_purchase_timestamp']\n",
    "\n",
    "##############################################################################\n",
    "# transfo des types timedelta en float\n",
    "order1.tim2approv = order1.tim2approv.dt.total_seconds()/3600/24\n",
    "order1.tim2carrier = order1.tim2carrier.dt.total_seconds()/3600/24\n",
    "order1.tim2deliv = order1.tim2deliv.dt.total_seconds()/3600/24\n",
    "order1.tim2estim = order1.tim2estim.dt.total_seconds()/3600/24\n",
    "order1.tim = order1.tim.dt.total_seconds()/3600/24"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c018dc9a",
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"Jointure des tables olist: order,customer et order review.\"\"\"\n",
    "##############################################################################\n",
    "# fusion order et customer\n",
    "ordercust = order1.copy()\n",
    "ordercust = ordercust.reset_index()\n",
    "cust.columns = ['customer_id', 'customer_unique_id',\n",
    "                'cust_zip', 'cust_city', 'cust_state']\n",
    "ordercust1 = pd.merge(ordercust, cust, how='outer', on='customer_id')\n",
    "##############################################################################\n",
    "# on ne garde que les delivred\n",
    "ordercust1 = ordercust1.loc[ordercust1.order_status == \"delivered\", :]\n",
    "ordercust1 = ordercust1.drop(columns=['order_status',\n",
    "                                      'order_approved_at',\n",
    "                                      'order_delivered_carrier_date',\n",
    "                                      'order_delivered_customer_date',\n",
    "                                      'order_estimated_delivery_date'])\n",
    "\n",
    "##############################################################################\n",
    "# il peut y avoir plusieurs echanges pour les review : on garde la finale\n",
    "def doublonrem(datain, datanam, keynam):\n",
    "    temp = datain.copy()\n",
    "    temp2 = temp.drop_duplicates(subset=keynam, keep='last', inplace=False)\n",
    "    # inplace -> renvoie une table filtrée\n",
    "    # ou modifie directement la table de base sans rien renvoyer\n",
    "    print('extraction de ', temp2.shape[0]-datain.shape[0],\n",
    "          'doublons de la table ', datanam, ' basée sur la variable ', keynam)\n",
    "    return temp2\n",
    "\n",
    "##############################################################################\n",
    "# fusion order et order review\n",
    "order_review1 = doublonrem(order_review, 'order_review', 'order_id')\n",
    "# *le *1 permet de transformer le boolean en 0/1\n",
    "order_review1['rev_title_flag'] =\\\n",
    "    order_review1.review_comment_title.notna()*1\n",
    "order_review1['rev_comment_flag'] =\\\n",
    "    order_review1.review_comment_message.notna()*1\n",
    "\n",
    "order_review1 = order_review1.drop(columns=['review_id',\n",
    "                                            'review_comment_title',\n",
    "                                            'review_comment_message',\n",
    "                                            'review_creation_date',\n",
    "                                            'review_answer_timestamp',\n",
    "                                            'rev_title_flag'])\n",
    "order_review1.columns = ['order_id', 'rev_score', 'rev_com_flag']\n",
    "\n",
    "ordercust2 = ordercust1.set_index('order_id').join(\n",
    "    order_review1.set_index('order_id'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7462fa1e",
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "    Jointure des tables ordercust avec la table olist geo\n",
    "    \n",
    "    un peu de nettoyage sur les données geo\n",
    "    aggregation des données par zip et état\n",
    "    calcul de la distance entre zip et Sao Paulo (considéré comme centre)\n",
    "    calcul de la distance entre etat et etat de Sao Paulo.\n",
    "\"\"\"\n",
    "##############################################################################\n",
    "geo1 = geo.copy()\n",
    "# petit remplissage pour empecher les trous de seller geo\n",
    "geoapp = pd.DataFrame([[2285, -23.54, -46.63, 'sao paulo', 'SP'],\n",
    "                       [7412, -23.3916478, -46.38547, 'aruja', 'SP'],\n",
    "                       [37708, -21.8199612, -46.825645,\n",
    "                        'pocos de caldas', 'MG'],\n",
    "                       [71551, -15.7215826, -48.2182329, 'brasilia', 'DF'],\n",
    "                       [72580, -15.7215826, -48.2182329, 'brasilia', 'DF'],\n",
    "                       [82040, -25.4950497, -49.4302258, 'curitiba', 'PR'],\n",
    "                       [91901, -30.1087954, -51.3175691, 'porto alegre',\n",
    "                        'RS']], columns=['geolocation_zip_code_prefix',\n",
    "                                         'geolocation_lat', 'geolocation_lng',\n",
    "                                         'geolocation_city',\n",
    "                                         'geolocation_state'])\n",
    "\n",
    "geo1 = geo1.append(geoapp)\n",
    "##############################################################################\n",
    "# nettoyage\n",
    "# on enleve les positions hors du bresil aberrantes\n",
    "# et on fixe le sproblemes d'orthographe\n",
    "geo1.loc[((geo1.geolocation_lng > -28) | (geo1.geolocation_lng < -75) |\n",
    "          (geo1.geolocation_lat > 5)\n",
    "          | (geo1.geolocation_lat < -34)), ['geolocation_lat',\n",
    "                                            'geolocation_lng']] = np.nan\n",
    "# minuscule\n",
    "geo1['geolocation_city'] = geo1['geolocation_city'].str.lower()\n",
    "# accent\n",
    "geo1['geolocation_city'] = geo1['geolocation_city'].str.replace('ã', 'a')\n",
    "geo1['geolocation_city'] = geo1['geolocation_city'].str.replace('í', 'i')\n",
    "geo1['geolocation_city'] = geo1['geolocation_city'].str.replace('é', 'e')\n",
    "geo1['geolocation_city'] = geo1['geolocation_city'].str.replace('ê', 'e')\n",
    "geo1['geolocation_city'] = geo1['geolocation_city'].str.replace('ô', 'o')\n",
    "geo1['geolocation_city'] = geo1['geolocation_city'].str.replace('á', 'a')\n",
    "geo1['geolocation_city'] = geo1['geolocation_city'].str.replace('ú', 'u')\n",
    "geo1['geolocation_city'] = geo1['geolocation_city'].str.replace('â', 'a')\n",
    "geo1['geolocation_city'] = geo1['geolocation_city'].str.replace('ó', 'o')\n",
    "geo1['geolocation_city'] = geo1['geolocation_city'].str.replace(\"t'\", 't')\n",
    "geo1['geolocation_city'] = geo1['geolocation_city'].str.replace('õ', 'o')\n",
    "geo1['geolocation_city'] = geo1['geolocation_city'].str.replace('ç', 'c')\n",
    "geo1['geolocation_city'] = geo1['geolocation_city'].str.replace(\"d'\", \"d \")\n",
    "geo1['geolocation_city'] = geo1['geolocation_city'].str.replace(\n",
    "    \"embu\", 'embu das artes')\n",
    "geo1['geolocation_city'] = geo1['geolocation_city'].str.replace(\n",
    "    \"das artes das artes\", \"das artes\")\n",
    "geo1['geolocation_city'] = geo1['geolocation_city'].str.replace(\"-\", ' ')\n",
    "\n",
    "geo1['geolocation_city'] = geo1['geolocation_city'].str.replace(\n",
    "    \"piumhi\", 'piumhii')\n",
    "geo1['geolocation_city'] = geo1['geolocation_city'].str.replace(\n",
    "    \"santa terezinha\", 'santa teresinha')\n",
    "geo1['geolocation_city'] = geo1['geolocation_city'].str.replace(\n",
    "    \"belem de sao francisco\", 'belem do sao francisco')\n",
    "\n",
    "##############################################################################\n",
    "# aggregation des données geo par zip\n",
    "geo1_a = geo1.groupby(by=\"geolocation_zip_code_prefix\").agg(\n",
    "    {'geolocation_lat': ['median'], 'geolocation_lng': ['median']})\n",
    "geo1_a.columns = geo1_a.columns.droplevel(0)\n",
    "geo1_a.columns = ['lat_zip', 'lng_zip']\n",
    "\n",
    "geo1 = doublonrem(geo1, 'geo', 'geolocation_zip_code_prefix')\n",
    "geo2 = geo1.loc[:, ['geolocation_zip_code_prefix', 'geolocation_city',\n",
    "                    'geolocation_state']].set_index(\n",
    "    'geolocation_zip_code_prefix')\n",
    "geo2 = geo2.join(geo1_a)\n",
    "\n",
    "##############################################################################\n",
    "geo1_b = geo2.groupby(by=\"geolocation_state\").agg(\n",
    "    {'lat_zip': ['median'], 'lng_zip': ['median']})\n",
    "geo1_b.columns = geo1_b.columns.droplevel(0)\n",
    "geo1_b.columns = ['lat_state', 'lng_state']\n",
    "\n",
    "geo2 = geo2.reset_index()\n",
    "geo2 = geo2.set_index('geolocation_state')\n",
    "\n",
    "geo2 = geo2.join(geo1_b)\n",
    "geo2 = geo2.reset_index()\n",
    "\n",
    "##############################################################################\n",
    "# calcul distance depuis sao paulo (qu on qualifie de centre)\n",
    "sao = geo2.loc[geo2.geolocation_city == 'sao paulo', :].groupby(\n",
    "    by=\"geolocation_city\").agg({'lat_zip': ['median'], 'lng_zip': ['median']})\n",
    "sao.columns = sao.columns.droplevel(0)\n",
    "sao.columns = ['sao_lat', 'sao_lng']\n",
    "geo2['sao_lat'] = sao.sao_lat[0]\n",
    "geo2['sao_lng'] = sao.sao_lng[0]\n",
    "\n",
    "def distance(s_lat, s_lng, e_lat, e_lng):\n",
    "    \"\"\"calul de la distance par la formule de haversine.\n",
    "    \n",
    "    prend en entrée la latitude et longitude du point A puis du point B\n",
    "    \"\"\"\n",
    "\n",
    "    # approximate radius of earth in km\n",
    "    R = 6373.0\n",
    "\n",
    "    s_lat = s_lat*np.pi/180.0\n",
    "    s_lng = np.deg2rad(s_lng)\n",
    "    e_lat = np.deg2rad(e_lat)\n",
    "    e_lng = np.deg2rad(e_lng)\n",
    "\n",
    "    d = np.sin((e_lat - s_lat)/2)**2 + np.cos(s_lat) * \\\n",
    "        np.cos(e_lat) * np.sin((e_lng - s_lng)/2)**2\n",
    "\n",
    "    return 2 * R * np.arcsin(np.sqrt(d))\n",
    "\n",
    "\n",
    "geo2['sdist_zip'] = distance(\n",
    "    geo2.lat_zip, geo2.lng_zip, geo2.sao_lat, geo2.sao_lng)\n",
    "geo2['sdist_state'] = distance(\n",
    "    geo2.lat_state, geo2.lng_state, geo2.sao_lat, geo2.sao_lng)\n",
    "geo2['sdist_dif'] = geo2['sdist_zip']-geo2['sdist_state']\n",
    "# filtre coordonnées fausses car ne semble pas dans le bon etat :\n",
    "# coordonnées de zip trop lointaines\n",
    "# parfois dues à la presence de 2 villes differentes mais de meme nom\n",
    "\n",
    "geo2.loc[(geo2['sdist_dif'].abs() > 600), ['lat_zip', 'lng_zip']] = np.nan\n",
    "geo2.loc[geo2.lat_zip.isna(), ['lat_zip']] = geo2.lat_state\n",
    "geo2.loc[geo2.lat_zip.isna(), ['lng_zip']] = geo2.lng_state\n",
    "geo2['sdist_zip'] = distance(\n",
    "    geo2.lat_zip, geo2.lng_zip, geo2.sao_lat, geo2.sao_lng)\n",
    "geo2 = geo2.drop(columns='sdist_dif')\n",
    "\n",
    "##############################################################################\n",
    "ordercust2 = ordercust2.reset_index()\n",
    "ordercust2 = ordercust2.set_index('cust_zip')\n",
    "\n",
    "# minuscule\n",
    "ordercust2['cust_city'] = ordercust2['cust_city'].str.lower()\n",
    "\n",
    "ordercust2['cust_city'] = ordercust2['cust_city'].str.replace('ã', 'a')\n",
    "ordercust2['cust_city'] = ordercust2['cust_city'].str.replace('í', 'i')\n",
    "ordercust2['cust_city'] = ordercust2['cust_city'].str.replace('é', 'e')\n",
    "ordercust2['cust_city'] = ordercust2['cust_city'].str.replace('ê', 'e')\n",
    "ordercust2['cust_city'] = ordercust2['cust_city'].str.replace('ô', 'o')\n",
    "ordercust2['cust_city'] = ordercust2['cust_city'].str.replace('á', 'a')\n",
    "ordercust2['cust_city'] = ordercust2['cust_city'].str.replace('ú', 'u')\n",
    "ordercust2['cust_city'] = ordercust2['cust_city'].str.replace('â', 'a')\n",
    "ordercust2['cust_city'] = ordercust2['cust_city'].str.replace('ó', 'o')\n",
    "ordercust2['cust_city'] = ordercust2['cust_city'].str.replace(\"t'\", 't')\n",
    "ordercust2['cust_city'] = ordercust2['cust_city'].str.replace('õ', 'o')\n",
    "ordercust2['cust_city'] = ordercust2['cust_city'].str.replace('ç', 'c')\n",
    "ordercust2['cust_city'] = ordercust2['cust_city'].str.replace(\"d'\", \"d \")\n",
    "ordercust2['cust_city'] = ordercust2['cust_city'].str.replace(\n",
    "    \"embu\", 'embu das artes')\n",
    "ordercust2['cust_city'] = ordercust2['cust_city'].str.replace(\n",
    "    \"das artes das artes\", \"das artes\")\n",
    "ordercust2['cust_city'] = ordercust2['cust_city'].str.replace(\"-\", ' ')\n",
    "\n",
    "ordercustgeo = ordercust2.join(geo2.set_index('geolocation_zip_code_prefix'))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "015c8528",
   "metadata": {},
   "source": [
    "# Jointure finale"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "11dc5191",
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "    Jointure des tables product3 avec la table olist geo\n",
    "    puis jointure avec ordercustgeo\n",
    "    pour memoire : product3 : jointure product, product cat, seller\n",
    "                    ordercustgeo : order,customer, order review et geo\n",
    "                    \n",
    "    calcul distance seller-customer   \n",
    "    aggregation de données\n",
    "\"\"\"\n",
    "\n",
    "##############################################################################\n",
    "product4 = product3.reset_index()\n",
    "product4 = product4.set_index('sel_zip')\n",
    "geo3 = geo2.set_index('geolocation_zip_code_prefix')\n",
    "productgeo = product4.join(geo3)\n",
    "\n",
    "\n",
    "productgeo.columns = ['product_id', 'seller_id', 'order_id', 'order_item_id',\n",
    "                      'ord_item_ship_lim_dat', 'ord_item_price',\n",
    "                      'ord_item_fdp', 'sel_city',\n",
    "                      'sel_state_', 'prod_desc_l', 'prod_foto_nb',\n",
    "                      'prod_w', 'prod_cat',\n",
    "                      'prod_vol', 'sel_state', 'sel_city_2', 'sel_lat_zip',\n",
    "                      'sel_lng_zip', 'sel_lat_state', 'sel_lng_state',\n",
    "                      'sao_lat', 'sao_lng', 'sel_sdist_zip',\n",
    "                      'sel_sdist_state']\n",
    "productgeo = productgeo.drop(columns='sel_state_')\n",
    "# on doit faire confiance a sel_state_2 (plus fiable) issu de geo ,\n",
    "# plutot que sel_state\n",
    "\n",
    "##############################################################################\n",
    "# lien entre les 2: order_id\n",
    "ordercustgeo_light = ordercustgeo.copy()\n",
    "ordercustgeo_light = ordercustgeo_light.reset_index()\n",
    "ordercustgeo_light = ordercustgeo_light[[\n",
    "    'order_id', 'lat_zip', 'lng_zip', 'lat_state', 'lng_state']]\n",
    "ordercustgeo_light = ordercustgeo_light.set_index('order_id')\n",
    "\n",
    "productgeo = productgeo.reset_index()\n",
    "productgeo = productgeo.set_index('order_id')\n",
    "\n",
    "productgeo2 = productgeo.join(ordercustgeo_light)\n",
    "\n",
    "##############################################################################\n",
    "# distance seller-customer\n",
    "productgeo2['selcustdist_zip'] = distance(\n",
    "    productgeo2.lat_zip, productgeo2.lng_zip, productgeo2.sel_lat_zip,\n",
    "    productgeo2.sel_lng_zip)\n",
    "productgeo2['selcustdist_state'] = distance(\n",
    "    productgeo2.lat_state, productgeo2.lng_state, productgeo2.sel_lat_state,\n",
    "    productgeo2.sel_lng_state)\n",
    "\n",
    "listodr1 = ['index', 'sel_city', 'sel_state', 'sel_city_2',\n",
    "            'sel_lat_zip', 'sel_lng_zip',\n",
    "            'sel_lat_state', 'sel_lng_state', 'sao_lat', 'sao_lng',\n",
    "            'lat_zip', 'lng_zip', 'lat_state', 'lng_state']\n",
    "productgeo3 = productgeo2.drop(columns=listodr1)\n",
    "\n",
    "listodr2 = ['cust_city', 'cust_state', 'geolocation_state',\n",
    "            'geolocation_city', 'lat_zip',\n",
    "            'lng_zip', 'lat_state', 'lng_state', 'sao_lat', 'sao_lng']\n",
    "ordercustgeo = ordercustgeo.drop(columns=listodr2)\n",
    "\n",
    "##############################################################################\n",
    "# agg par order de productgeo3\n",
    "productgeo3 = productgeo3.reset_index()\n",
    "productgeo_agg = productgeo3.groupby(by=\"order_id\").agg({\n",
    "    'order_item_id': ['count', 'max'],\n",
    "    'ord_item_price': ['sum'],\n",
    "    'ord_item_fdp': ['sum'],\n",
    "    'prod_desc_l': ['mean'],\n",
    "    'prod_foto_nb': ['mean'],\n",
    "    'prod_w': ['sum'],\n",
    "    'prod_vol': ['sum'],\n",
    "    'sel_sdist_zip': ['sum', 'mean'],\n",
    "    'sel_sdist_state': ['sum', 'mean'],\n",
    "    'selcustdist_zip': ['sum', 'mean'],\n",
    "    'selcustdist_state': ['sum', 'mean']})\n",
    "productgeo_agg.columns = productgeo_agg.columns.droplevel(0)\n",
    "productgeo_agg.columns = ['item_nb', 'item_max', 'price', 'fdp',\n",
    "                          'desc_l', 'foto', 'w', 'vol',\n",
    "                          'sel2s_zip_sum', 'sel2s_zip_mean',\n",
    "                          'sel2s_state_sum', 'sel2s_state_mean',\n",
    "                          'sel2c_zip_sum', 'sel2c_zip_mean',\n",
    "                          'sel2c_state_sum', 'sel2c_state_mean']\n",
    "\n",
    "ordercustgeo2 = ordercustgeo.copy()\n",
    "ordercustgeo2 = ordercustgeo2.set_index('order_id')\n",
    "ordercustgeo2 = ordercustgeo2.join(productgeo_agg)\n",
    "\n",
    "\n",
    "##############################################################################\n",
    "# il faut trouver la categorie la plus achetee par commande\n",
    "# utiliser un get dummies pour facilite le calcul\n",
    "\n",
    "pd.set_option('max_columns', None)\n",
    "display(ordercustgeo2)\n",
    "display(productgeo3)\n",
    "pd.set_option('max_columns', 10)\n",
    "\n",
    "# recherche de la categorie majoritaire en CA dans la commande\n",
    "\n",
    "# one hot encoder\n",
    "productgeo4 = pd.get_dummies(data=productgeo3, columns=['prod_cat'])\n",
    "liscat = productgeo4.filter(regex='^prod_cat_', axis=1).columns.tolist()\n",
    "liscat2 = [s.replace(\"prod_cat_\", \"\") for s in liscat]\n",
    "# display(liscat2)\n",
    "\n",
    "productgeo5 = productgeo4[['order_id', 'ord_item_price']+liscat]\n",
    "productgeo6 = pd.concat([productgeo5['order_id'], productgeo5.filter(\n",
    "    regex='^prod_cat_', axis=1).multiply(productgeo5['ord_item_price'],\n",
    "                                         axis=\"index\")], axis=1)\n",
    "# aggregation\n",
    "productgeo6_agg = productgeo6.groupby(by=\"order_id\").agg(['sum'])\n",
    "productgeo6_agg.columns = productgeo6_agg.columns.droplevel(0)\n",
    "productgeo6_agg.columns = liscat2\n",
    "# categorie max\n",
    "productgeo6_agg['cat'] = productgeo6_agg.idxmax(axis=1)\n",
    "\n",
    "# fusion avec ordercustgeo2\n",
    "ordercustgeo3 = ordercustgeo2.join(productgeo6_agg['cat'])\n",
    "\n",
    "ordercustgeo3 = ordercustgeo3[['customer_id', 'customer_unique_id',\n",
    "                               'order_purchase_timestamp',\n",
    "                               'tim2approv', 'tim2carrier',\n",
    "                               'tim2deliv', 'tim2estim', 'tim', 'item_nb',\n",
    "                               'item_max', 'price', 'fdp', 'w', 'vol', 'cat',\n",
    "                               'pay_seq', 'pay_ins', 'pay_val',\n",
    "                               'pay_val_voucher', 'pay_val_pct_voucher',\n",
    "                               'pay_type',\n",
    "                               'rev_score', 'rev_com_flag', 'desc_l',\n",
    "                               'foto', 'sdist_zip', 'sdist_state',\n",
    "                               'sel2s_zip_sum',\n",
    "                               'sel2s_zip_mean', 'sel2s_state_sum',\n",
    "                               'sel2s_state_mean', 'sel2c_zip_sum',\n",
    "                               'sel2c_zip_mean',\n",
    "                               'sel2c_state_sum', 'sel2c_state_mean']]\n",
    "ordercustgeo3 = ordercustgeo3.sort_values('order_purchase_timestamp')\n",
    "orderfull = ordercustgeo3.copy()\n",
    "\n",
    "catfull = productgeo3[['order_id', 'order_item_id', 'ord_item_price',\n",
    "                       'ord_item_fdp',\n",
    "                       'prod_desc_l', 'prod_foto_nb', 'prod_w', 'prod_cat',\n",
    "                       'prod_vol']].copy()\n",
    "catfull = catfull.set_index('order_id')\n",
    "catfull = catfull.join(orderfull['order_purchase_timestamp'])\n",
    "catfull = catfull.loc[catfull.order_purchase_timestamp.notna(),\n",
    "                      :].sort_values(\n",
    "    ['order_purchase_timestamp', 'order_item_id'])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "70c9cc7f",
   "metadata": {},
   "source": [
    "# Check données manquantes et imputer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "44eb2870",
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "    Check des données manquantes\n",
    "\"\"\"\n",
    "orderfull = orderfull.loc[orderfull.pay_seq.notna(), :]\n",
    "orderfull = orderfull.loc[orderfull.pay_type.notna(), :]\n",
    "orderfull['rev_score'] = orderfull['rev_score'].fillna(2.5)\n",
    "orderfull['rev_com_flag'] = orderfull['rev_com_flag'].fillna(0)\n",
    "orderfull['desc_l'] = orderfull['desc_l'].fillna(0)\n",
    "orderfull['foto'] = orderfull['foto'].fillna(0)\n",
    "\n",
    "orderfull['temp'] = orderfull.tim-orderfull.tim2deliv-1\n",
    "orderfull.loc[orderfull.tim2carrier.isna(), ['tim2carrier']\n",
    "              ] = orderfull['temp']\n",
    "orderfull.loc[orderfull.tim2approv.isna(), ['tim2approv']] = 1\n",
    "\n",
    "orderfull.loc[(orderfull.sel2c_zip_mean.isna() &\n",
    "               orderfull.sel2c_zip_sum == 0), 'sel2c_zip_mean'] = 0\n",
    "orderfull.loc[(orderfull.sel2c_state_mean.isna() &\n",
    "               orderfull.sel2c_state_sum == 0), 'sel2c_state_mean'] = 0\n",
    "\n",
    "orderfull.loc[(orderfull.sel2s_zip_mean.isna() &\n",
    "               orderfull.sel2s_zip_sum == 0), 'sel2c_zip_mean'] = 0\n",
    "orderfull.loc[(orderfull.sel2s_state_mean.isna() &\n",
    "               orderfull.sel2s_state_sum == 0), 'sel2c_state_mean'] = 0\n",
    "\n",
    "orderfull = orderfull.drop(columns=['temp'])\n",
    "\n",
    "catfull = catfull.loc[catfull.prod_cat.notna()]\n",
    "catfull = catfull.loc[catfull.prod_w.notna()]\n",
    "\n",
    "plt.rcParams[\"figure.figsize\"] = [16, 6]\n",
    "plt.rcParams[\"figure.autolayout\"] = True\n",
    "f, axes = plt.subplots(1, 2)\n",
    "\n",
    "ordernull = 1-(orderfull.isnull().sum()/orderfull.shape[0])\n",
    "ordernull.plot.barh(color=\"#8d19a9\", title='Pourcentage de données',\n",
    "                    xlabel='pourcentage de données', ax=axes[0])\n",
    "axes[0].grid(zorder=0)\n",
    "\n",
    "catnull = 1-(catfull.isnull().sum()/catfull.shape[0])\n",
    "catnull.plot.barh(color=\"#8d19a9\", title='Pourcentage de données',\n",
    "                  xlabel='pourcentage de données', ax=axes[1])\n",
    "axes[1].grid(zorder=0)\n",
    "\n",
    "plt.show()\n",
    "\n",
    "pd.set_option('max_rows', None)\n",
    "display(ordernull)\n",
    "display(catnull)\n",
    "pd.set_option('max_rows', 10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4dd5f3b5",
   "metadata": {},
   "outputs": [],
   "source": [
    "# KNN imputer\n",
    "from sklearn.impute import KNNImputer\n",
    "import timeit\n",
    "\n",
    "start = timeit.default_timer()\n",
    "\n",
    "imputer = KNNImputer(n_neighbors=5)\n",
    "\n",
    "orderfull2 = orderfull.copy()\n",
    "orderfull2 = orderfull2.reset_index()\n",
    "orderfull3 = orderfull2.drop(columns=[\n",
    "                             'order_id', 'customer_id', 'customer_unique_id',\n",
    "                             'order_purchase_timestamp', 'cat', 'pay_type'])\n",
    "\n",
    "# wesh=orderfull2.head(1000)\n",
    "imputer.fit(orderfull3)\n",
    "orderfull4 = pd.DataFrame(imputer.transform(\n",
    "    orderfull3), columns=orderfull3.columns)\n",
    "\n",
    "stop1 = timeit.default_timer()\n",
    "print('Time: ', stop1 - start)\n",
    "\n",
    "orderfull = pd.concat([orderfull2[['order_id', 'customer_id',\n",
    "                                   'customer_unique_id',\n",
    "                                   'order_purchase_timestamp', 'cat',\n",
    "                                   'pay_type']], orderfull4], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ad058165",
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.set_option('max_rows', None)\n",
    "display(orderfull.dtypes)\n",
    "pd.set_option('max_rows', 10)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3bb06395",
   "metadata": {},
   "source": [
    "# Fonction IQR filter et Correlation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "06c7ff6c",
   "metadata": {},
   "outputs": [],
   "source": [
    "# fonction IQR filter\n",
    "# filtre IQR car les minmax ou standard scaler sont sensibles aux outliers\n",
    "from scipy.stats import normaltest\n",
    "from scipy.stats import shapiro\n",
    "from scipy.stats import spearmanr\n",
    "from scipy.stats import pearsonr\n",
    "\n",
    "\n",
    "def iqr_outlier(df, coef=1.5, reset=True, cutupQ=True):\n",
    "    \"\"\"Fonction qui traite les outliers\n",
    "    \n",
    "        dans notre cas ne traite que les données positives\n",
    "    \n",
    "        prend en entrée le dataframe\n",
    "        reset=true: reset index\n",
    "        cutupQ=True: cap à q99\n",
    "        coef=1.5 coef de l'iqr pour capper et floorer\n",
    "    \"\"\"\n",
    "    if reset == True:\n",
    "        df = df.reset_index()\n",
    "    df1 = df.copy()\n",
    "\n",
    "    df = df._get_numeric_data()\n",
    "\n",
    "    pd.set_option('max_columns', None)\n",
    "    # display(df)\n",
    "    pd.set_option('max_columns', 10)\n",
    "\n",
    "    q1 = df.quantile(0.25)\n",
    "    q3 = df.quantile(0.75)\n",
    "    q99 = df.quantile(0.99)\n",
    "    print('q1')\n",
    "    pd.set_option('max_rows', None)\n",
    "    display(q1)\n",
    "    pd.set_option('max_rows', 10)\n",
    "    print('q3')\n",
    "    pd.set_option('max_rows', None)\n",
    "    display(q3)\n",
    "    pd.set_option('max_rows', 10)\n",
    "\n",
    "    iqr = q3 - q1\n",
    "\n",
    "    print('IQR')\n",
    "    pd.set_option('max_rows', None)\n",
    "    display(iqr)\n",
    "    pd.set_option('max_rows', 10)\n",
    "\n",
    "    print('Q99')\n",
    "    pd.set_option('max_rows', None)\n",
    "    display(q99)\n",
    "    pd.set_option('max_rows', 10)\n",
    "\n",
    "    lower_bound = q1 - (coef * iqr)\n",
    "    upper_bound = q3 + (coef * iqr)\n",
    "\n",
    "    for col in df.columns:\n",
    "        for i in range(0, df.shape[0]):\n",
    "            # evitons les inferieurs à 0 dans notre cas\n",
    "            if (cutupQ == False):\n",
    "                if df.iloc[i, df.columns.get_loc(col)] < lower_bound[col]:\n",
    "                    df.iloc[i, df.columns.get_loc(col)] = max(\n",
    "                        0, lower_bound[col])\n",
    "\n",
    "                if df.iloc[i, df.columns.get_loc(col)] > upper_bound[col]:\n",
    "                    df.iloc[i, df.columns.get_loc(col)] = upper_bound[col]\n",
    "\n",
    "            if (cutupQ == True):\n",
    "                if df.iloc[i, df.columns.get_loc(col)] > q99[col]:\n",
    "                    df.iloc[i, df.columns.get_loc(col)] = q99[col]\n",
    "\n",
    "    # display(df1)\n",
    "    # display(df)\n",
    "    for col in df.columns:\n",
    "        df1.loc[:, col] = df.loc[:, col]\n",
    "\n",
    "    if reset == True:\n",
    "        return(df1.set_index(df1.columns[0]))\n",
    "    else:\n",
    "        return(df1)\n",
    "\n",
    "\n",
    "def calculate_corr(df):\n",
    "    \"\"\"Fonction qui calcul des correlations.\"\"\"\n",
    "    \n",
    "    df = df.dropna()._get_numeric_data()\n",
    "    dfcols = pd.DataFrame(columns=df.columns)\n",
    "    pearscoeff = dfcols.transpose().join(dfcols, how='outer')\n",
    "    pearspvalues = pearscoeff.copy()\n",
    "    shappvalues = pearscoeff.copy()\n",
    "    normpvalues = pearscoeff.copy()\n",
    "    spearcoeff = pearscoeff.copy()\n",
    "    spearpvalues = pearscoeff.copy()\n",
    "    for r in df.columns:\n",
    "        for c in df.columns:\n",
    "            pearscoeff[r][c] = round(pearsonr(df[r], df[c])[0], 4)\n",
    "            pearspvalues[r][c] = round(pearsonr(df[r], df[c])[1], 4)\n",
    "            spearcoeff[r][c] = round(spearmanr(df[r], df[c])[0], 4)\n",
    "            spearpvalues[r][c] = round(spearmanr(df[r], df[c])[1], 4)\n",
    "            if r == c:\n",
    "                # attention pvalue shapiro wilk seulement pour N<5000\n",
    "                shappvalues[r][c] = round(shapiro(df[r])[1], 4)\n",
    "                # d'agostino basé sur omnibus\n",
    "                # plus adapté pour les grands echantillons\n",
    "                normpvalues[r][c] = round(normaltest(df[r])[1], 4)\n",
    "    pearscoeff[df.columns] = pearscoeff[df.columns].astype(float)\n",
    "    pearspvalues[df.columns] = pearspvalues[df.columns].astype(float)\n",
    "    shappvalues[df.columns] = shappvalues[df.columns].astype(float)\n",
    "    normpvalues[df.columns] = normpvalues[df.columns].astype(float)\n",
    "    spearcoeff[df.columns] = spearcoeff[df.columns].astype(float)\n",
    "    spearpvalues[df.columns] = spearpvalues[df.columns].astype(float)\n",
    "    return pearscoeff, pearspvalues, shappvalues,\\\n",
    "normpvalues, spearcoeff, spearpvalues"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e399f9dd",
   "metadata": {},
   "source": [
    "# SEGMENTATION\n",
    "* k-means algorithm is very fast\n",
    "* but it falls in local minima. That’s why it can be useful to restart it several times\n",
    "* If the algorithm stops before fully converging,the cluster_centers_ will not be the means of the points in each cluster\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "346be57b",
   "metadata": {},
   "outputs": [],
   "source": [
    "def cutdata(cutdate='2017-10-01', correl=False):\n",
    "    \"\"\"Fonction qui prepare les données avant segmentation.\n",
    "    \n",
    "        selection de la periode via cutdate\n",
    "        ajout des données de catégories par features\n",
    "        nombre de commande LTM\n",
    "        recence\n",
    "        affichage des correlations si besoin\n",
    "    \"\"\"\n",
    "    \n",
    "    \n",
    "    \n",
    "\n",
    "    # les categories\n",
    "    # selection date\n",
    "    cat2seg = catfull.loc[(catfull['order_purchase_timestamp'] < cutdate)]\n",
    "\n",
    "    # aggregation par categorie\n",
    "    cat2seg_agg = cat2seg.groupby(by=\"prod_cat\").agg({\n",
    "        'prod_vol': ['median'],\n",
    "        'prod_w': ['median'],\n",
    "        'order_item_id': ['count'],\n",
    "        'ord_item_price': ['sum', 'median'],\n",
    "        'ord_item_fdp': ['sum', 'median'],\n",
    "        'prod_desc_l': ['median'],\n",
    "        'prod_foto_nb': ['median']})\n",
    "    cat2seg_agg.columns = cat2seg_agg.columns.droplevel(0)\n",
    "    cat2seg_agg.columns = ['cat_vol', 'cat_w', 'cat_nb', 'cat_ca',\n",
    "                           'cat_pr', 'cat_fdp_ca',\n",
    "                           'cat_fdp', 'cat_des', 'cat_fot']\n",
    "\n",
    "    cat2seg2 = cat2seg_agg\n",
    "    # on reduit le nombre de variables\n",
    "    # en regardant les correlations precedentes\n",
    "    cat2seg2b = cat2seg2.drop(\n",
    "        columns=['cat_vol', 'cat_fdp', 'cat_nb', 'cat_fdp_ca', 'cat_des'])\n",
    "\n",
    "    #################################\n",
    "    #################################\n",
    "    #################################\n",
    "    if correl == True:\n",
    "        cocorr, pvcorr, shap, dagos, cospcorr, pvspcorr = calculate_corr(\n",
    "            cat2seg2)\n",
    "\n",
    "        plt.rcParams[\"figure.figsize\"] = [16, 6]\n",
    "        f, axes = plt.subplots(1, 2)\n",
    "\n",
    "        sns.heatmap(\n",
    "            cocorr,\n",
    "            vmin=-1, vmax=1, center=0,\n",
    "            cmap=sns.diverging_palette(20, 220, n=200),\n",
    "            square=True,\n",
    "            annot=True,\n",
    "            fmt='.2f',\n",
    "            linewidths=.5, ax=axes[0])\n",
    "        axes[0].title.set_text('Coefficient correlation de Pearson')\n",
    "\n",
    "        sns.heatmap(\n",
    "            pvcorr,\n",
    "            vmin=0, vmax=1, center=0.5,\n",
    "            cmap=sns.diverging_palette(200, 20, n=200),\n",
    "            square=True,\n",
    "            annot=True,\n",
    "            fmt='.2f',\n",
    "            linewidths=.5, ax=axes[1])\n",
    "        axes[1].title.set_text('Pvalue correlation de Pearson')\n",
    "\n",
    "        plt.show()\n",
    "\n",
    "        ########################################\n",
    "\n",
    "        cocorr, pvcorr, shap, dagos, cospcorr, pvspcorr = calculate_corr(\n",
    "            cat2seg2b)\n",
    "\n",
    "        plt.rcParams[\"figure.figsize\"] = [16, 6]\n",
    "        f, axes = plt.subplots(1, 2)\n",
    "\n",
    "        sns.heatmap(\n",
    "            cocorr,\n",
    "            vmin=-1, vmax=1, center=0,\n",
    "            cmap=sns.diverging_palette(20, 220, n=200),\n",
    "            square=True,\n",
    "            annot=True,\n",
    "            fmt='.2f',\n",
    "            linewidths=.5, ax=axes[0])\n",
    "        axes[0].title.set_text('Coefficient correlation de Pearson')\n",
    "\n",
    "        sns.heatmap(\n",
    "            pvcorr,\n",
    "            vmin=0, vmax=1, center=0.5,\n",
    "            cmap=sns.diverging_palette(200, 20, n=200),\n",
    "            square=True,\n",
    "            annot=True,\n",
    "            fmt='.2f',\n",
    "            linewidths=.5, ax=axes[1])\n",
    "        axes[1].title.set_text('Pvalue correlation de Pearson')\n",
    "\n",
    "        plt.show()\n",
    "    #################################\n",
    "    #################################\n",
    "    #################################\n",
    "\n",
    "    # frequence de commande last 12 months\n",
    "    # table last 12 months pour la frequ\n",
    "    freq = orderfull.loc[((orderfull['order_purchase_timestamp'] < cutdate) & (\n",
    "        orderfull['order_purchase_timestamp'] >\n",
    "        pd.to_datetime(cutdate)-pd.DateOffset(years=1))),\n",
    "        ['customer_unique_id', 'item_nb']]\n",
    "\n",
    "    # calcul frequence de commande last 12 months\n",
    "    freq_agg = freq.groupby(by=\"customer_unique_id\").agg(\n",
    "        {'item_nb': ['count']})\n",
    "    freq_agg.columns = freq_agg.columns.droplevel(0)\n",
    "    freq_agg.columns = ['nb_com_ltm']\n",
    "\n",
    "    # le gros des données\n",
    "    # selection date\n",
    "    order2seg = orderfull.loc[(\n",
    "        orderfull['order_purchase_timestamp'] < cutdate), :].copy()\n",
    "    order2seg['init'] = dt.datetime.strptime(cutdate, '%Y-%m-%d')\n",
    "    order2seg['rec'] = (order2seg['order_purchase_timestamp'] -\n",
    "                        order2seg['init']).dt.total_seconds()/3600/24\n",
    "\n",
    "    order2segb = order2seg.reset_index()\n",
    "    # jointure avec les categories\n",
    "    order2segb = order2segb.set_index('cat').join(cat2seg2b)\n",
    "    order2segb = order2segb.reset_index()\n",
    "    order2segb = order2segb.sort_values('index')\n",
    "    order2segb = order2segb.set_index('index')\n",
    "\n",
    "    # aggregation sur client unique\n",
    "    # item_nb : nombre d 'article' -> count= nombre de commandes\n",
    "    # last = nombre d'objet de la derniere commande\n",
    "    # sum = nombre d'objet commandés\n",
    "    # prix : prix commande sans fdp' -> #sum = somme totale depensée\n",
    "    # last = prix derniere commande\n",
    "    # mean = prix moyen par commande\n",
    "    # fdp : prix commande sans fdp' -> #sum = fdp totaux depensés\n",
    "    # last = fdp derniere commande\n",
    "    # mean = fdp moyen par commande\n",
    "    # w : poids'                    -> #sum = poids de toutes les commandes\n",
    "    # last = poids derniere commande\n",
    "    # mean = poids moyen par commande\n",
    "    # vol : volume'        -> #sum = volume total de toutes les commandes\n",
    "    # last = volume derniere commande\n",
    "    # mean = volume moyen par commande\n",
    "    # pay_inst : paiements credit ' -> #sum = nombre total de paiements\n",
    "    # last = nombre de paiements pour la derniere commande\n",
    "    # mean = nombre de paiements moyen par commande\n",
    "    # rev_score : satisfaction ' -> #last = dernier score\n",
    "    # mean = score moyen\n",
    "    # rev_comm_flag : commentaire ' -> #last = derniere commande\n",
    "    #mean = moyenne\n",
    "    # rsdist_zip : distance zip ' -> #last = derniere commande\n",
    "    # sdist_state : distance etat ' -> #last = derniere commande\n",
    "\n",
    "    # cat_w: poids categorie  ->#last ->\n",
    "    # poids categorie de la derniere commande\n",
    "    # cat_ca: poids categorie  ->#last ->\n",
    "    # ca moyen categorie de la derniere commande\n",
    "    # cat_pr: poids categorie  ->#last ->\n",
    "    # prix moyen article categorie de la derniere commande\n",
    "    # cat_fot: poids categorie  ->#last ->\n",
    "    # nb photos categorie de la derniere commande\n",
    "\n",
    "    # recence -> #last derniere commande\n",
    "    order2seg_agg = order2segb.groupby(by=\"customer_unique_id\").agg({\n",
    "        'item_nb': ['count', 'last', 'sum'],\n",
    "        'price': ['sum', 'last', 'mean'],\n",
    "        'fdp': ['sum', 'last', 'mean'],\n",
    "        'w': ['sum', 'last', 'mean'],\n",
    "        'vol': ['sum', 'last', 'mean'],\n",
    "        'pay_ins': ['sum', 'last', 'mean'],\n",
    "        'pay_val_pct_voucher': ['mean'],\n",
    "        'rev_score': ['last', 'mean'],\n",
    "        'rev_com_flag': ['last', 'mean'],\n",
    "        'sdist_zip': ['last'],\n",
    "        'sdist_state': ['last'],\n",
    "        'cat_w': ['last'],\n",
    "        'cat_ca': ['last'],\n",
    "        'cat_pr': ['last'],\n",
    "        'cat_fot': ['last'],\n",
    "        'rec': ['last']})\n",
    "    order2seg_agg.columns = order2seg_agg.columns.droplevel(0)\n",
    "    order2seg_agg.columns = ['nb_com', 'item_nb_last_com', 'item_nb_total',\n",
    "                             'pr_sum', 'pr_last', 'pr_mean',\n",
    "                             'fdp_sum', 'fdp_last', 'fdp_mean',\n",
    "                             'w_sum', 'w_last', 'w_mean',\n",
    "                             'vol_sum', 'vol_last', 'vol_mean',\n",
    "                             'pay_ins_sum', 'pay_ins_last', 'pay_ins_mean',\n",
    "                             'pct_voucher', 'note_last', 'note_mean',\n",
    "                             'com_last', 'com_mean',\n",
    "                             'dist_zip', 'dist_state',\n",
    "                             'cat_w_last', 'cat_ca_last',\n",
    "                             'cat_pr_last', 'cat_fot_last',\n",
    "                             'rec']\n",
    "    # jointure avec freq last 12m\n",
    "    order2seg_agg = order2seg_agg.join(freq_agg)\n",
    "    order2seg_agg['nb_com_ltm'] = order2seg_agg['nb_com_ltm'].fillna(0)\n",
    "\n",
    "    #####################\n",
    "    # correlation\n",
    "    if correl == True:\n",
    "        cocorr, pvcorr, shap, dagos, cospcorr, pvspcorr = calculate_corr(\n",
    "            order2seg2)\n",
    "\n",
    "        plt.rcParams[\"figure.figsize\"] = [16, 12]\n",
    "        f, axes = plt.subplots(1, 1)\n",
    "\n",
    "        sns.heatmap(\n",
    "            cocorr,\n",
    "            vmin=-1, vmax=1, center=0,\n",
    "            cmap=sns.diverging_palette(20, 220, n=200),\n",
    "            square=True,\n",
    "            annot=True,\n",
    "            annot_kws={\"size\": 50 / np.sqrt(len(cocorr))},\n",
    "            fmt='.2f',\n",
    "            linewidths=.5, ax=axes)\n",
    "        axes.title.set_text('Coefficient correlation de Pearson')\n",
    "    return(order2seg_agg)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c32bf894",
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# kmeans (yellowbrick- machine learning vizualization)\n",
    "from plotly.subplots import make_subplots\n",
    "import plotly.express as px\n",
    "import plotly.graph_objects as go\n",
    "import scipy.stats as st\n",
    "import warnings\n",
    "import datetime as dt\n",
    "from sklearn.preprocessing import MinMaxScaler, StandardScaler\n",
    "from sklearn.cluster import KMeans\n",
    "from yellowbrick.cluster import KElbowVisualizer, SilhouetteVisualizer,\\\n",
    "InterclusterDistance\n",
    "\n",
    "\n",
    "def seg_kmeans(\n",
    "        cutD='2017-10-01',\n",
    "        scalerfitD='2017-10-01',\n",
    "        cutO=False,\n",
    "        Standardization=True,\n",
    "        scal='Standard',\n",
    "        lisfac=['rec', 'nb_com_ltm', 'pr_mean'],\n",
    "        ElbowVisualizer=True,\n",
    "        Sil_ICD_Visualizer=True,\n",
    "        Sil_ICD_K=[3, 4, 5, 6]):\n",
    "    \n",
    "    \"\"\"Segmentation kmeans.\n",
    "    \n",
    "        cutD: selection de la periode\n",
    "        scalerfitD: selection de la période de normalisation\n",
    "        cutO: traitement des outliers ou non\n",
    "        Standardization: normalisation ou non\n",
    "        scal: type de normalisation 'Standard' ou 'minmax'\n",
    "        lisfac: liste des features à selectionner\n",
    "        ElbowVisualizer: Visu ou non des scores de distorsion, silhouette\n",
    "        Sil_ICD_Visualizer: Visu ou non des clusters et de leurs caracteristiques\n",
    "        Sil_ICD_K: liste des nombres de clusters à visualiser\n",
    "    \"\"\"\n",
    "\n",
    "    # warnings.filterwarnings('ignore')\n",
    "\n",
    "    # selection date pour segmentation\n",
    "    data2s = cutdata(cutdate=cutD, correl=False)\n",
    "    # selection date pour scaler fit\n",
    "    data2scalerfit = cutdata(cutdate=scalerfitD, correl=False)\n",
    "\n",
    "    if cutO == True:\n",
    "        # filtre IQR / complexe à utiliser\n",
    "        # car les clients à plusieurs commandes sont rares\n",
    "        # mieux vaut faire un cut up sur Q99 ou plus: cutupQ\n",
    "        data2s = iqr_outlier(data2s, coef=1.5, cutupQ=True)\n",
    "        data2scalerfit = iqr_outlier(data2scalerfit, coef=1.5, cutupQ=True)\n",
    "\n",
    "    # selection des facteurs\n",
    "    data2s_selec = data2s.loc[:, lisfac].copy()\n",
    "    data2scalerfit_selec = data2scalerfit.loc[:, lisfac].copy()\n",
    "\n",
    "    # scaler init\n",
    "    # pour le clustering Standard est souvent prefere\n",
    "    if Standardization == True:\n",
    "        if scal == 'MinMax':\n",
    "            scaler = MinMaxScaler()\n",
    "        if scal == 'Standard':\n",
    "            scaler = StandardScaler()\n",
    "        # scaler fit\n",
    "        scaler.fit(data2scalerfit_selec)\n",
    "        # scaler transform\n",
    "        data2s2_selec = pd.DataFrame(scaler.transform(\n",
    "            data2s_selec), columns=data2s_selec.columns,\n",
    "            index=data2s_selec.index)\n",
    "    else:\n",
    "        data2s2_selec = data2s_selec\n",
    "\n",
    "    # kmeans avec yellowbrick\n",
    "    # implements the “elbow” method of selecting the optimal number of clusters\n",
    "    # by fitting the K-Means model with a range of values for K\n",
    "    # the “elbow” (the point of inflection on the curve) is a good indication\n",
    "    # that the underlying model fits best at that point\n",
    "\n",
    "    # the elbow method doesn't always work well; especially\n",
    "    # if the data is not very clustered.\n",
    "    # No clear elbow. Instead, we see a fairly smooth curve,\n",
    "    # unclear what is the best value of k to choose\n",
    "\n",
    "    # attention def legerment differente entre scikitlearn et yellowbrick\n",
    "    # DISTORSION to minimize: sum of squared distances\n",
    "    # from each point to its assigned center.\n",
    "    # CALINSKI to minimize ,dispersion within clusters /\n",
    "    # dispersion between clusters /\n",
    "    # silhouette to minimize mean ratio of intra-cluster\n",
    "    # and nearest-cluster distance\n",
    "    # init\n",
    "    if ElbowVisualizer == True:\n",
    "\n",
    "        model = KMeans()\n",
    "\n",
    "        plt.rcParams[\"figure.figsize\"] = [16, 5]\n",
    "        plt.rcParams[\"figure.autolayout\"] = True\n",
    "        f, axes = plt.subplots(1, 3)\n",
    "\n",
    "        # distorsion\n",
    "        visualizer1 = KElbowVisualizer(model, k=(3, 12), ax=axes[0])\n",
    "        visualizer1.fit(data2s2_selec)    # Fit the data to the visualizer\n",
    "        visualizer1.finalize()  # prepare the chart : title /label\n",
    "        # CALINSKI\n",
    "        visualizer2 = KElbowVisualizer(\n",
    "            model, k=(3, 12), metric='calinski_harabasz', ax=axes[2])\n",
    "        # , timings=False)\n",
    "        visualizer2.fit(data2s2_selec)    # Fit the data to the visualizer\n",
    "        visualizer2.finalize()  # prepare the chart : title /label\n",
    "        # CALINSKI\n",
    "        visualizer3 = KElbowVisualizer(\n",
    "            model, k=(3, 12), metric='silhouette', ax=axes[1])\n",
    "        # , timings=False)\n",
    "        visualizer3.fit(data2s2_selec)    # Fit the data to the visualizer\n",
    "        visualizer3.finalize()  # prepare the chart : title /label\n",
    "\n",
    "        plt.show()\n",
    "\n",
    "    # evaluate the density and separation between clusters\n",
    "    # calculated by averaging the silhouette coefficient for each sample:\n",
    "        # computed as the difference between the average intra-cluster\n",
    "        # distance and the mean nearest-cluster distance for each sample\n",
    "        # normalized by the maximum value\n",
    "\n",
    "        # scores near +1 indicate high separation\n",
    "        # scores near -1 indicate that the samples may have been\n",
    "        # assigned to the wrong cluster.\n",
    "\n",
    "    # The SilhouetteVisualizer displays the silhouette coefficient\n",
    "    # for each sample on a per-cluster basis,\n",
    "    # allowing users to visualize the density and separation of the clusters\n",
    "\n",
    "    # Intercluster distance maps :cluster centers in 2 dimensions\n",
    "    # with the distance to other centers preserved.\n",
    "    # E.g. the closer to centers are in the visualization,\n",
    "    # the closer they are in the original feature space.\n",
    "    # The clusters are sized according to a scoring metric.\n",
    "        # By default, they are sized by membership,\n",
    "        # e.g. the number of instances that belong to each center.\n",
    "        # This gives a sense of the relative importance of clusters.\n",
    "    # Note however, that because two clusters overlap in the 2D space,\n",
    "    # it does not imply that they overlap in the original\n",
    "    plt.rcParams[\"figure.figsize\"] = [16, 5]\n",
    "    plt.rcParams[\"figure.autolayout\"] = False\n",
    "    if Sil_ICD_Visualizer == True:\n",
    "\n",
    "        for i in Sil_ICD_K:\n",
    "            f, axes = plt.subplots(1, 2)\n",
    "\n",
    "            model = KMeans(i)\n",
    "            model.fit(data2s2_selec.values)\n",
    "\n",
    "            visualizer = SilhouetteVisualizer(\n",
    "                model, ax=axes[0], is_fitted=True)\n",
    "            # deja fité juste au dessus\n",
    "            # Fit the data to the visualizer\n",
    "            visualizer.fit(data2s2_selec.values)\n",
    "            visualizer.finalize()\n",
    "            # prepare the chart : title /label\n",
    "\n",
    "            visualizer = InterclusterDistance(\n",
    "                model, ax=axes[1], is_fitted=True)\n",
    "            # deja fité juste au dessus\n",
    "            # Fit the data to the visualizer\n",
    "            visualizer.fit(data2s2_selec.values)\n",
    "            visualizer.finalize()  # prepare the chart : title /label\n",
    "\n",
    "            plt.show()\n",
    "\n",
    "            model.fit(data2s2_selec)\n",
    "            kmeans_labels = model.labels_\n",
    "            kmeans_centers = pd.DataFrame(\n",
    "                model.cluster_centers_,\n",
    "                columns=model.feature_names_in_.tolist())\n",
    "\n",
    "            # display(model.labels_)\n",
    "            # display(scaler.inverse_transform(data2s2_selec))\n",
    "\n",
    "            # display(pd.value_counts(kmeans_labels,normalize=True))\n",
    "            # display(kmeans_centers)\n",
    "\n",
    "            # on percentilise le zcore #st.norm.ppf(.95) #st.norm.cdf(1.64)\n",
    "            kmeans_centers = kmeans_centers.applymap(st.norm.cdf)\n",
    "            # display(kmeans_centers)\n",
    "\n",
    "            h = kmeans_centers.T\n",
    "            h = h.reset_index()\n",
    "\n",
    "            f, axes = plt.subplots(1, 2)\n",
    "\n",
    "            axes[0].pie(pd.value_counts(kmeans_labels),\n",
    "                        labels=pd.value_counts(\n",
    "                kmeans_labels).index, normalize=True, autopct='%1.1f%%',\n",
    "                shadow=True, startangle=90)\n",
    "            # Equal aspect ratio ensures that pie is drawn as a circle.\n",
    "            axes[0].axis('equal')\n",
    "            axes[0].set_title('distribution des clusters')\n",
    "\n",
    "            h.plot(x=\"index\", y=kmeans_centers.index, kind=\"bar\",\n",
    "                   fontsize=12, width=0.7, ax=axes[1])\n",
    "            axes[1].set_title('features des centroides des clusters')\n",
    "\n",
    "            plt.show()\n",
    "\n",
    "            # boxplot des clusters\n",
    "            # on aurait pu prendre la table avant inversion directement\n",
    "            temp1 = pd.concat([pd.DataFrame(\n",
    "                scaler.inverse_transform(data2s2_selec),\n",
    "                columns=lisfac, index=data2s2_selec.index),\n",
    "                pd.DataFrame(model.labels_,\n",
    "                             columns=['clus'],\n",
    "                             index=data2s2_selec.index)],\n",
    "                axis=1)\n",
    "            temp2 = pd.concat([data2s2_selec,\n",
    "                               pd.DataFrame(model.labels_,\n",
    "                                            columns=['clus'],\n",
    "                                            index=data2s2_selec.index)],\n",
    "                              axis=1)\n",
    "\n",
    "            plt.rcParams[\"figure.figsize\"] = [16, 5]\n",
    "            plt.rcParams[\"figure.autolayout\"] = True\n",
    "\n",
    "            # non standardisé\n",
    "            # attention à ne pas mettre trop de facteurs\n",
    "            f, axes = plt.subplots(1, len(lisfac))\n",
    "            counter = 0\n",
    "            for i in lisfac:\n",
    "                sns.boxplot(x='clus', y=i, data=temp1, showmeans=True,\n",
    "                            meanprops={\n",
    "                                \"marker\": \"s\", \"markerfacecolor\": \"pink\",\n",
    "                                \"markeredgecolor\": \"blue\"},\n",
    "                            showfliers=False, ax=axes[counter])\n",
    "                axes[counter].set_title(\n",
    "                    'boxplot feature par cluster', fontsize=15)\n",
    "                counter = counter+1\n",
    "\n",
    "            plt.show()\n",
    "\n",
    "            # plt.show()\n",
    "\n",
    "            # scatterplot des clusters\n",
    "            if len(lisfac) == 3:\n",
    "                f, axes = plt.subplots(1, 3)\n",
    "                sns.scatterplot(\n",
    "                    data=temp1,\n",
    "                    x=lisfac[0],\n",
    "                    y=lisfac[1],\n",
    "                    hue='clus',\n",
    "                    ax=axes[0])\n",
    "                sns.scatterplot(\n",
    "                    data=temp1,\n",
    "                    x=lisfac[0],\n",
    "                    y=lisfac[2],\n",
    "                    hue='clus',\n",
    "                    ax=axes[1])\n",
    "                sns.scatterplot(\n",
    "                    data=temp1,\n",
    "                    x=lisfac[1],\n",
    "                    y=lisfac[2],\n",
    "                    hue='clus',\n",
    "                    ax=axes[2])\n",
    "                axes[0].set_title(\n",
    "                    'scatterplot feature par cluster', fontsize=15)\n",
    "                axes[1].set_title(\n",
    "                    'scatterplot feature par cluster', fontsize=15)\n",
    "                axes[2].set_title(\n",
    "                    'scatterplot feature par cluster', fontsize=15)\n",
    "                plt.show()\n",
    "\n",
    "                # 3D scatter\n",
    "                dat3d = [go.Scatter3d(x=temp1[lisfac[0]],\n",
    "                                      y=temp1[lisfac[1]],\n",
    "                                      z=temp1[lisfac[2]],\n",
    "                                      mode='markers',\n",
    "                                      marker=dict(size=2,\n",
    "                                                  color=temp1['clus'],\n",
    "                                                  colorbar=dict(title=\"cluster\")))]\n",
    "                fig = go.Figure(dat3d)\n",
    "\n",
    "                fig.update_layout(\n",
    "                    title=\"cluster-scatter\",\n",
    "                    scene=dict(\n",
    "                        xaxis_title=lisfac[0],\n",
    "                        yaxis_title=lisfac[1],\n",
    "                        zaxis_title=lisfac[2]),)\n",
    "\n",
    "                fig.show()\n",
    "\n",
    "            if len(lisfac) == 4:\n",
    "                f, axes = plt.subplots(1, 3)\n",
    "                sns.scatterplot(\n",
    "                    data=temp1, x=lisfac[0], y=lisfac[1],\n",
    "                    hue='clus', ax=axes[0])\n",
    "                sns.scatterplot(\n",
    "                    data=temp1, x=lisfac[0], y=lisfac[2],\n",
    "                    hue='clus', ax=axes[1])\n",
    "                sns.scatterplot(\n",
    "                    data=temp1, x=lisfac[0], y=lisfac[3],\n",
    "                    hue='clus', ax=axes[2])\n",
    "                axes[0].set_title(\n",
    "                    'scatterplot feature par cluster', fontsize=15)\n",
    "                axes[1].set_title(\n",
    "                    'scatterplot feature par cluster', fontsize=15)\n",
    "                axes[2].set_title(\n",
    "                    'scatterplot feature par cluster', fontsize=15)\n",
    "                plt.show()\n",
    "\n",
    "                f, axes = plt.subplots(1, 3)\n",
    "                sns.scatterplot(\n",
    "                    data=temp1, x=lisfac[1], y=lisfac[2],\n",
    "                    hue='clus', ax=axes[0])\n",
    "                sns.scatterplot(\n",
    "                    data=temp1, x=lisfac[1], y=lisfac[3],\n",
    "                    hue='clus', ax=axes[1])\n",
    "                sns.scatterplot(\n",
    "                    data=temp1, x=lisfac[2], y=lisfac[3],\n",
    "                    hue='clus', ax=axes[2])\n",
    "                axes[0].set_title(\n",
    "                    'scatterplot feature par cluster', fontsize=15)\n",
    "                axes[1].set_title(\n",
    "                    'scatterplot feature par cluster', fontsize=15)\n",
    "                axes[2].set_title(\n",
    "                    'scatterplot feature par cluster', fontsize=15)\n",
    "                plt.show()\n",
    "\n",
    "                # 3D scatter\n",
    "                dat3d = [go.Scatter3d(x=temp1[lisfac[0]],\n",
    "                                      y=temp1[lisfac[1]], z=temp1[lisfac[2]],\n",
    "                                      mode='markers',\n",
    "                                      marker=dict(size=2,\n",
    "                                                  color=temp1['clus'],\n",
    "                                                  colorbar=dict(title=\"cluster\")))]\n",
    "                fig = go.Figure(dat3d)\n",
    "\n",
    "                fig.update_layout(\n",
    "                    title=\"cluster-scatter\",\n",
    "                    scene=dict(\n",
    "                        xaxis_title=lisfac[0],\n",
    "                        yaxis_title=lisfac[1],\n",
    "                        zaxis_title=lisfac[2]),)\n",
    "\n",
    "                fig.show()\n",
    "\n",
    "                dat3d = [go.Scatter3d(x=temp1[lisfac[0]],\n",
    "                                      y=temp1[lisfac[1]],\n",
    "                                      z=temp1[lisfac[3]],\n",
    "                                      mode='markers',\n",
    "                                      marker=dict(size=2, color=temp1['clus'],\n",
    "                                                  colorbar=dict(title=\"cluster\")))]\n",
    "                fig = go.Figure(dat3d)\n",
    "\n",
    "                fig.update_layout(\n",
    "                    title=\"cluster-scatter\",\n",
    "                    scene=dict(\n",
    "                        xaxis_title=lisfac[0],\n",
    "                        yaxis_title=lisfac[1],\n",
    "                        zaxis_title=lisfac[3]),)\n",
    "\n",
    "                fig.show()\n",
    "\n",
    "                dat3d = [go.Scatter3d(x=temp1[lisfac[0]],\n",
    "                                      y=temp1[lisfac[2]],\n",
    "                                      z=temp1[lisfac[3]],\n",
    "                                      mode='markers',\n",
    "                                      marker=dict(size=2,\n",
    "                                                  color=temp1['clus'],\n",
    "                                                  colorbar=dict(title=\"cluster\")))]\n",
    "                fig = go.Figure(dat3d)\n",
    "\n",
    "                fig.update_layout(\n",
    "                    title=\"cluster-scatter\",\n",
    "                    scene=dict(\n",
    "                        xaxis_title=lisfac[0],\n",
    "                        yaxis_title=lisfac[2],\n",
    "                        zaxis_title=lisfac[3]),)\n",
    "\n",
    "                fig.show()\n",
    "\n",
    "                dat3d = [go.Scatter3d(x=temp1[lisfac[1]],\n",
    "                                      y=temp1[lisfac[2]],\n",
    "                                      z=temp1[lisfac[3]],\n",
    "                                      mode='markers',\n",
    "                                      marker=dict(size=2,\n",
    "                                                  color=temp1['clus'],\n",
    "                                                  colorbar=dict(title=\"cluster\")))]\n",
    "                fig = go.Figure(dat3d)\n",
    "\n",
    "                fig.update_layout(\n",
    "                    title=\"cluster-scatter\",\n",
    "                    scene=dict(\n",
    "                        xaxis_title=lisfac[1],\n",
    "                        yaxis_title=lisfac[2],\n",
    "                        zaxis_title=lisfac[3]),)\n",
    "\n",
    "                fig.show()\n",
    "\n",
    "            if len(lisfac) == 5:\n",
    "                f, axes = plt.subplots(1, 4)\n",
    "                sns.scatterplot(\n",
    "                    data=temp1, x=lisfac[0], y=lisfac[1],\n",
    "                    hue='clus', ax=axes[0])\n",
    "                sns.scatterplot(\n",
    "                    data=temp1, x=lisfac[0], y=lisfac[2],\n",
    "                    hue='clus', ax=axes[1])\n",
    "                sns.scatterplot(\n",
    "                    data=temp1, x=lisfac[0], y=lisfac[3],\n",
    "                    hue='clus', ax=axes[2])\n",
    "                sns.scatterplot(\n",
    "                    data=temp1, x=lisfac[0], y=lisfac[4],\n",
    "                    hue='clus', ax=axes[3])\n",
    "                axes[0].set_title(\n",
    "                    'scatterplot feature par cluster', fontsize=15)\n",
    "                axes[1].set_title(\n",
    "                    'scatterplot feature par cluster', fontsize=15)\n",
    "                axes[2].set_title(\n",
    "                    'scatterplot feature par cluster', fontsize=15)\n",
    "                axes[3].set_title(\n",
    "                    'scatterplot feature par cluster', fontsize=15)\n",
    "                plt.show()\n",
    "\n",
    "                f, axes = plt.subplots(1, 3)\n",
    "                sns.scatterplot(\n",
    "                    data=temp1, x=lisfac[1], y=lisfac[2],\n",
    "                    hue='clus', ax=axes[0])\n",
    "                sns.scatterplot(\n",
    "                    data=temp1, x=lisfac[1], y=lisfac[3],\n",
    "                    hue='clus', ax=axes[1])\n",
    "                sns.scatterplot(\n",
    "                    data=temp1, x=lisfac[1], y=lisfac[4],\n",
    "                    hue='clus', ax=axes[2])\n",
    "                axes[0].set_title(\n",
    "                    'scatterplot feature par cluster', fontsize=15)\n",
    "                axes[1].set_title(\n",
    "                    'scatterplot feature par cluster', fontsize=15)\n",
    "                axes[2].set_title(\n",
    "                    'scatterplot feature par cluster', fontsize=15)\n",
    "                plt.show()\n",
    "\n",
    "                f, axes = plt.subplots(1, 3)\n",
    "                sns.scatterplot(\n",
    "                    data=temp1, x=lisfac[2], y=lisfac[3],\n",
    "                    hue='clus', ax=axes[0])\n",
    "                sns.scatterplot(\n",
    "                    data=temp1, x=lisfac[2], y=lisfac[4],\n",
    "                    hue='clus', ax=axes[1])\n",
    "                sns.scatterplot(\n",
    "                    data=temp1, x=lisfac[3], y=lisfac[4],\n",
    "                    hue='clus', ax=axes[2])\n",
    "                axes[0].set_title(\n",
    "                    'scatterplot feature par cluster', fontsize=15)\n",
    "                axes[1].set_title(\n",
    "                    'scatterplot feature par cluster', fontsize=15)\n",
    "                axes[2].set_title(\n",
    "                    'scatterplot feature par cluster', fontsize=15)\n",
    "                plt.show()\n",
    "\n",
    "                # 3D scatter\n",
    "                dat3d = [go.Scatter3d(x=temp1[lisfac[0]],\n",
    "                                      y=temp1[lisfac[1]],\n",
    "                                      z=temp1[lisfac[2]],\n",
    "                                      mode='markers',\n",
    "                                      marker=dict(size=2,\n",
    "                                                  color=temp1['clus'],\n",
    "                                                  colorbar=dict(title=\"cluster\")))]\n",
    "                fig = go.Figure(dat3d)\n",
    "\n",
    "                fig.update_layout(\n",
    "                    title=\"cluster-scatter\",\n",
    "                    scene=dict(\n",
    "                        xaxis_title=lisfac[0],\n",
    "                        yaxis_title=lisfac[1],\n",
    "                        zaxis_title=lisfac[2]),)\n",
    "\n",
    "                fig.show()\n",
    "\n",
    "                dat3d = [go.Scatter3d(x=temp1[lisfac[0]],\n",
    "                                      y=temp1[lisfac[1]],\n",
    "                                      z=temp1[lisfac[3]],\n",
    "                                      mode='markers',\n",
    "                                      marker=dict(size=2,\n",
    "                                                  color=temp1['clus'],\n",
    "                                                  colorbar=dict(title=\"cluster\")))]\n",
    "                fig = go.Figure(dat3d)\n",
    "\n",
    "                fig.update_layout(\n",
    "                    title=\"cluster-scatter\",\n",
    "                    scene=dict(\n",
    "                        xaxis_title=lisfac[0],\n",
    "                        yaxis_title=lisfac[1],\n",
    "                        zaxis_title=lisfac[3]),)\n",
    "\n",
    "                fig.show()\n",
    "\n",
    "                dat3d = [go.Scatter3d(x=temp1[lisfac[0]],\n",
    "                                      y=temp1[lisfac[1]],\n",
    "                                      z=temp1[lisfac[4]],\n",
    "                                      mode='markers',\n",
    "                                      marker=dict(size=2,\n",
    "                                                  color=temp1['clus'],\n",
    "                                                  colorbar=dict(title=\"cluster\")))]\n",
    "                fig = go.Figure(dat3d)\n",
    "\n",
    "                fig.update_layout(\n",
    "                    title=\"cluster-scatter\",\n",
    "                    scene=dict(\n",
    "                        xaxis_title=lisfac[0],\n",
    "                        yaxis_title=lisfac[1],\n",
    "                        zaxis_title=lisfac[4]),)\n",
    "\n",
    "                fig.show()\n",
    "\n",
    "                dat3d = [go.Scatter3d(x=temp1[lisfac[0]],\n",
    "                                      y=temp1[lisfac[2]],\n",
    "                                      z=temp1[lisfac[3]],\n",
    "                                      mode='markers',\n",
    "                                      marker=dict(size=2,\n",
    "                                                  color=temp1['clus'],\n",
    "                                                  colorbar=dict(title=\"cluster\")))]\n",
    "                fig = go.Figure(dat3d)\n",
    "\n",
    "                fig.update_layout(\n",
    "                    title=\"cluster-scatter\",\n",
    "                    scene=dict(\n",
    "                        xaxis_title=lisfac[0],\n",
    "                        yaxis_title=lisfac[2],\n",
    "                        zaxis_title=lisfac[3]),)\n",
    "\n",
    "                fig.show()\n",
    "\n",
    "                dat3d = [go.Scatter3d(x=temp1[lisfac[0]],\n",
    "                                      y=temp1[lisfac[2]],\n",
    "                                      z=temp1[lisfac[4]],\n",
    "                                      mode='markers',\n",
    "                                      marker=dict(size=2, color=temp1['clus'],\n",
    "                                                  colorbar=dict(title=\"cluster\")))]\n",
    "                fig = go.Figure(dat3d)\n",
    "\n",
    "                fig.update_layout(\n",
    "                    title=\"cluster-scatter\",\n",
    "                    scene=dict(\n",
    "                        xaxis_title=lisfac[0],\n",
    "                        yaxis_title=lisfac[2],\n",
    "                        zaxis_title=lisfac[4]),)\n",
    "\n",
    "                fig.show()\n",
    "\n",
    "                dat3d = [go.Scatter3d(x=temp1[lisfac[0]],\n",
    "                                      y=temp1[lisfac[3]],\n",
    "                                      z=temp1[lisfac[4]],\n",
    "                                      mode='markers',\n",
    "                                      marker=dict(size=2, color=temp1['clus'],\n",
    "                                                  colorbar=dict(title=\"cluster\")))]\n",
    "                fig = go.Figure(dat3d)\n",
    "\n",
    "                fig.update_layout(\n",
    "                    title=\"cluster-scatter\",\n",
    "                    scene=dict(\n",
    "                        xaxis_title=lisfac[0],\n",
    "                        yaxis_title=lisfac[3],\n",
    "                        zaxis_title=lisfac[4]),)\n",
    "\n",
    "                fig.show()\n",
    "\n",
    "                dat3d = [go.Scatter3d(x=temp1[lisfac[1]],\n",
    "                                      y=temp1[lisfac[2]],\n",
    "                                      z=temp1[lisfac[3]],\n",
    "                                      mode='markers',\n",
    "                                      marker=dict(size=2,\n",
    "                                                  color=temp1['clus'],\n",
    "                                                  colorbar=dict(title=\"cluster\")))]\n",
    "                fig = go.Figure(dat3d)\n",
    "\n",
    "                fig.update_layout(\n",
    "                    title=\"cluster-scatter\",\n",
    "                    scene=dict(\n",
    "                        xaxis_title=lisfac[1],\n",
    "                        yaxis_title=lisfac[2],\n",
    "                        zaxis_title=lisfac[3]),)\n",
    "\n",
    "                fig.show()\n",
    "\n",
    "                dat3d = [go.Scatter3d(x=temp1[lisfac[1]],\n",
    "                                      y=temp1[lisfac[2]],\n",
    "                                      z=temp1[lisfac[4]],\n",
    "                                      mode='markers',\n",
    "                                      marker=dict(size=2,\n",
    "                                                  color=temp1['clus'],\n",
    "                                                  colorbar=dict(title=\"cluster\")))]\n",
    "                fig = go.Figure(dat3d)\n",
    "\n",
    "                fig.update_layout(\n",
    "                    title=\"cluster-scatter\",\n",
    "                    scene=dict(\n",
    "                        xaxis_title=lisfac[1],\n",
    "                        yaxis_title=lisfac[2],\n",
    "                        zaxis_title=lisfac[4]),)\n",
    "\n",
    "                fig.show()\n",
    "\n",
    "                dat3d = [go.Scatter3d(x=temp1[lisfac[2]],\n",
    "                                      y=temp1[lisfac[3]],\n",
    "                                      z=temp1[lisfac[4]],\n",
    "                                      mode='markers',\n",
    "                                      marker=dict(size=2,\n",
    "                                                  color=temp1['clus'],\n",
    "                                                  colorbar=dict(title=\"cluster\")))]\n",
    "                fig = go.Figure(dat3d)\n",
    "\n",
    "                fig.update_layout(\n",
    "                    title=\"cluster-scatter\",\n",
    "                    scene=dict(\n",
    "                        xaxis_title=lisfac[2],\n",
    "                        yaxis_title=lisfac[3],\n",
    "                        zaxis_title=lisfac[4]),)\n",
    "\n",
    "                fig.show()\n",
    "\n",
    "            # radar\n",
    "\n",
    "            categories = kmeans_centers.columns.to_list()\n",
    "            fig = go.Figure()\n",
    "\n",
    "            for j in kmeans_centers.index:\n",
    "                fig.add_trace(go.Scatterpolar(\n",
    "                    r=kmeans_centers.iloc[j],\n",
    "                    theta=categories,\n",
    "                    fill='toself',\n",
    "                    name='cluster_'+str(j)))\n",
    "\n",
    "            fig.update_layout(\n",
    "                title=\"cluster: centroides radar\",\n",
    "                polar=dict(radialaxis=dict(visible=True, range=[0, 1])),\n",
    "                showlegend=True)\n",
    "            fig.show()\n",
    "\n",
    "    # warnings.filterwarnings('default')\n",
    "    return(data2s2_selec)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "39639642",
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "data2inter = seg_kmeans(\n",
    "    cutD='2018-08-01',\n",
    "    scalerfitD='2018-08-01',\n",
    "    cutO=False,\n",
    "    Standardization=True,\n",
    "    scal='Standard',\n",
    "    lisfac=['rec', 'nb_com_ltm', 'pr_mean'],\n",
    "    ElbowVisualizer=True,\n",
    "    Sil_ICD_Visualizer=False,\n",
    "    Sil_ICD_K=[3, 4, 5, 6])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fe622478",
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "data2inter = seg_kmeans(\n",
    "    cutD='2018-08-01',\n",
    "    scalerfitD='2018-08-01',\n",
    "    cutO=False,\n",
    "    Standardization=True,\n",
    "    scal='Standard',\n",
    "    lisfac=['rec', 'nb_com_ltm', 'pr_mean'],\n",
    "    ElbowVisualizer=True,\n",
    "    Sil_ICD_Visualizer=True,\n",
    "    Sil_ICD_K=[3, 4, 5, 6])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0e9345fb",
   "metadata": {},
   "outputs": [],
   "source": [
    "data2inter = seg_kmeans(\n",
    "    cutD='2018-08-01',\n",
    "    scalerfitD='2018-08-01',\n",
    "    cutO=False,\n",
    "    Standardization=True,\n",
    "    scal='Standard',\n",
    "    lisfac=['rec', 'nb_com_ltm', 'pr_mean', 'dist_zip'],\n",
    "    ElbowVisualizer=True,\n",
    "    Sil_ICD_Visualizer=False,\n",
    "    Sil_ICD_K=[3, 4, 5, 6])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e4504e8f",
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "data2inter = seg_kmeans(\n",
    "    cutD='2018-08-01',\n",
    "    scalerfitD='2018-08-01',\n",
    "    cutO=False,\n",
    "    Standardization=True,\n",
    "    scal='Standard',\n",
    "    lisfac=['rec', 'nb_com_ltm', 'pr_mean', 'dist_zip'],\n",
    "    ElbowVisualizer=True,\n",
    "    Sil_ICD_Visualizer=True,\n",
    "    Sil_ICD_K=[3, 4, 5, 6])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "369f43fd",
   "metadata": {},
   "outputs": [],
   "source": [
    "data2inter = seg_kmeans(\n",
    "    cutD='2018-08-01',\n",
    "    scalerfitD='2018-08-01',\n",
    "    cutO=False,\n",
    "    Standardization=True,\n",
    "    scal='Standard',\n",
    "    lisfac=['rec', 'nb_com_ltm', 'pr_mean', 'dist_zip', 'note_last'],\n",
    "    ElbowVisualizer=True,\n",
    "    Sil_ICD_Visualizer=False,\n",
    "    Sil_ICD_K=[3, 4, 5, 6])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "34ff6217",
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "data2inter = seg_kmeans(\n",
    "    cutD='2018-08-01',\n",
    "    scalerfitD='2018-08-01',\n",
    "    cutO=False,\n",
    "    Standardization=True,\n",
    "    scal='Standard',\n",
    "    lisfac=['rec', 'nb_com_ltm', 'pr_mean', 'dist_zip', 'note_last'],\n",
    "    ElbowVisualizer=True,\n",
    "    Sil_ICD_Visualizer=True,\n",
    "    Sil_ICD_K=[3, 4, 5, 6])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d8a369cc",
   "metadata": {},
   "source": [
    "# DBSCAN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4720c94c",
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# DBSCAN algorithm views clusters as areas of HIGH DENSITY\n",
    "# separated by areas of LOW DENSITY\n",
    "# clusters found by DBSCAN can be ANY SHAPE, as opposed to k-means\n",
    "# which assumes that clusters are CONVEX SHAPED\n",
    "# A cluster is therefore a set of CORE SAMPLES (HIGH DENNITY),\n",
    "# each close to each other (measured by some distance measure)\n",
    "# and a set of non-core samples (LOW DESNITY) that are close to a core sample\n",
    "# (but are not themselves core samples)\n",
    "\n",
    "# Higher MIN_SAMPLES or lower EPS indicate HIGHER DENSITY necessary\n",
    "# to form a cluster\n",
    "# when EPS too SMALL: most data will not be clustered (-1)\n",
    "# when EPS too large : only one cluster\n",
    "\n",
    "# MIN_SAMPLES need to be larger when big sample and noise\n",
    "\n",
    "# CORE SAMPLE = sample in the dataset such that there exist MIN_SAMPLES\n",
    "# other samples within a distance of EPS, which are NEIGHBOORHOOD\n",
    "# of the core sample.\n",
    "# cluster is a set of core samples that can be built by recursively\n",
    "# taking a core sample, finding all of its neighbors that are core samples,\n",
    "# finding all of their neighbors that are core samples, and so on\n",
    "# A cluster also has a set of non-core samples (FRINGES=MARGES)\n",
    "\n",
    "# implementation:\n",
    "# the core samples will always be assigned to the same clusters\n",
    "# the labels of those clusters will depend on the order in which\n",
    "# those samples are encountered\n",
    "# the clusters to which non-core samples are assigned can differ\n",
    "# depending on the data order.\n",
    "# This would happen when a non-core sample has a distance lower\n",
    "# than eps to two core samples in different clusters\n",
    "\n",
    "# OPTICS more memory efficient\n",
    "from sklearn import metrics\n",
    "from sklearn.cluster import DBSCAN\n",
    "res = [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]\n",
    "\n",
    "\n",
    "data2mod1 = seg_kmeans(\n",
    "    cutD='2017-10-01',\n",
    "    scalerfitD='2017-10-01',\n",
    "    cutO=False,\n",
    "    Standardization=True,\n",
    "    scal='Standard',\n",
    "    lisfac=['rec', 'nb_com_ltm', 'pr_mean'],\n",
    "    ElbowVisualizer=False,\n",
    "    Sil_ICD_Visualizer=False)\n",
    "\n",
    "for eps_v in np.arange(0.1, 2.1, 0.1).tolist():\n",
    "    for min_samples_v in [10]+np.arange(100, 600, 100).tolist():\n",
    "        # eps:The maximum distance between two samples for one\n",
    "        # to be considered as in the neighborhood of the other\n",
    "        # eps defaut=0.5\n",
    "        clustering = DBSCAN(\n",
    "            eps=eps_v, min_samples=min_samples_v, n_jobs=-1).fit(data2mod1)\n",
    "        temp = pd.DataFrame(pd.DataFrame(clustering.labels_, columns=[\n",
    "                            'clus']).value_counts(normalize=True),\n",
    "                            columns=['pct'])\n",
    "        temp = temp.reset_index()\n",
    "\n",
    "        # pour calculer mediane des facteurs standardisés pour le no cluster -1\n",
    "        tempp = pd.concat([data2mod1, pd.DataFrame(\n",
    "            clustering.labels_,\n",
    "            columns=['clus'],\n",
    "            index=data2mod1.index)],\n",
    "            axis=1)\n",
    "\n",
    "        res = res+[[eps_v,\n",
    "                    min_samples_v,\n",
    "                    temp.loc[(temp.clus != -1), :].shape[0],\n",
    "                    temp.loc[(temp.clus == -1), 'pct'].values[0],\n",
    "                    temp.pct.max(),\n",
    "                    metrics.silhouette_score(\n",
    "                        data2mod1, clustering.labels_, metric=\"euclidean\"),\n",
    "                    metrics.calinski_harabasz_score(\n",
    "                        data2mod1, clustering.labels_),\n",
    "                    tempp.loc[tempp.clus == -1].mean()[0],\n",
    "                    tempp.loc[tempp.clus == -1].mean()[1],\n",
    "                    tempp.loc[tempp.clus == -1].mean()[2]]]\n",
    "\n",
    "res = pd.DataFrame(res, columns=['eps', 'min_samples',\n",
    "                                 'nb_clus', 'pct_no_clus',\n",
    "                                 'clus_pct_max', 'sil_score', 'cal_score',\n",
    "                                 'rec', 'com', 'pr'])\n",
    "res = res.loc[(res.eps > 0), :]\n",
    "\n",
    "plt.rcParams[\"figure.figsize\"] = [16, 12]\n",
    "plt.rcParams[\"figure.autolayout\"] = True\n",
    "f, axes = plt.subplots(3, 3)\n",
    "sns.scatterplot(data=res, x='eps', y='nb_clus',\n",
    "                hue='min_samples', ax=axes[0, 0])\n",
    "sns.scatterplot(data=res, x='eps', y='nb_clus',\n",
    "                hue='min_samples', ax=axes[0, 1])\n",
    "sns.scatterplot(data=res, x='eps', y='pct_no_clus',\n",
    "                hue='min_samples', ax=axes[0, 2])\n",
    "sns.scatterplot(data=res, x='eps', y='sil_score',\n",
    "                hue='min_samples', ax=axes[1, 0])\n",
    "sns.scatterplot(data=res, x='eps', y='cal_score',\n",
    "                hue='min_samples', ax=axes[1, 1])\n",
    "sns.scatterplot(data=res, x='eps', y='clus_pct_max',\n",
    "                hue='min_samples', ax=axes[1, 2])\n",
    "sns.scatterplot(data=res, x='eps', y='rec',\n",
    "                hue='min_samples', ax=axes[2, 0])\n",
    "sns.scatterplot(data=res, x='eps', y='com',\n",
    "                hue='min_samples', ax=axes[2, 1])\n",
    "sns.scatterplot(data=res, x='eps', y='pr',\n",
    "                hue='min_samples', ax=axes[2, 2])\n",
    "\n",
    "axes[0, 0].set_title(\n",
    "    'scatterplot-hyperparamètres: nombre de clusters', fontsize=10)\n",
    "axes[0, 1].set_title(\n",
    "    'scatterplot-hyperparamètres: nombre de clusters (zoom)', fontsize=10)\n",
    "axes[0, 1].set_ylim(0, 5)\n",
    "axes[0, 2].set_title(\n",
    "    'scatterplot-hyperparamètres: PCT data sans cluster', fontsize=10)\n",
    "\n",
    "axes[1, 0].set_title(\n",
    "    'scatterplot-hyperparamètres: silhouette score', fontsize=10)\n",
    "axes[1, 1].set_title(\n",
    "    'scatterplot-hyperparamètres: calinski score', fontsize=10)\n",
    "axes[1, 2].set_title(\n",
    "    'scatterplot-hyperparamètres: PCT MAX data dans 1 cluster', fontsize=10)\n",
    "\n",
    "axes[2, 0].set_title(\n",
    "    'hyperparamètres: recence standard median pour clus -1 ', fontsize=10)\n",
    "axes[2, 1].set_title(\n",
    "    'hyperparamètres: nb com standard median pour clus -1', fontsize=10)\n",
    "axes[2, 2].set_title(\n",
    "    'hyperparamètres: prix standard median pour clus -1', fontsize=10)\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5f3e3ab3",
   "metadata": {},
   "source": [
    "# AGGLOMERATIVE VS KMEANS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "75d40907",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Hierarchical clustering\n",
    "# general family of clustering algorithms that build nested clusters\n",
    "# by merging or splitting them successively\n",
    "# This hierarchy of clusters is represented as a tree (or dendrogram)\n",
    "\n",
    "#  AgglomerativeClustering object performs a hierarchical clustering\n",
    "# using a bottom up approach.\n",
    "# possibility of adding connectivity constraints\n",
    "# computationally expensive\n",
    "# metric used for the merge strategy:\n",
    "# Ward : minimize squared differences within clusters ->\n",
    "# variance minimising approcah (like k means)\n",
    "# gives the most regular sizes\n",
    "# but no variety of distance\n",
    "# complete linkage -> minimize distance max between 2 clusters\n",
    "# average linkage -> minimize distance mean between 2 clusters\n",
    "# (good choice for non euclidian metric)\n",
    "# single linkage ->minimize distance min between 2 clusters\n",
    "# (worst startegy but time efficient)\n",
    "\n",
    "# Varying the metric (only for complete/average/single)\n",
    "# Euclidean distance (l2),\n",
    "# Manhattan distance (or Cityblock, or l1:\n",
    "# l1 good for sparse feature (many 0)),\n",
    "# cosine distance (invariant to global scalings)\n",
    "# any precomputed affinity matrix\n",
    "\n",
    "# guidelines for choosing a metric is to use one that maximizes\n",
    "# the distance between samples in different classes,\n",
    "# and minimizes that within each class.\n",
    "\n",
    "import time\n",
    "from sklearn.metrics.cluster import calinski_harabasz_score,\\\n",
    "    davies_bouldin_score, silhouette_score\n",
    "from sklearn.cluster import AgglomerativeClustering\n",
    "from sklearn.cluster import KMeans\n",
    "from sklearn.metrics.cluster import adjusted_rand_score,\\\n",
    "completeness_score, normalized_mutual_info_score, fowlkes_mallows_score\n",
    "\n",
    "data2mod = seg_kmeans(\n",
    "    cutD='2017-10-01',\n",
    "    scalerfitD='2017-10-01',\n",
    "    cutO=False,\n",
    "    Standardization=True,\n",
    "    scal='Standard',\n",
    "    lisfac=['rec', 'nb_com_ltm', 'pr_mean'],\n",
    "    ElbowVisualizer=False,\n",
    "    Sil_ICD_Visualizer=False)\n",
    "\n",
    "# mod_asc=AgglomerativeClustering(\n",
    "#    n_clusters=4,\n",
    "#    affinity='euclidean', # “euclidean”, “l1”, “l2”, “manhattan”,\n",
    "# “cosine”, or “precomputed” #if ward only euclidean\n",
    "#    linkage='ward', #{‘ward’, ‘complete’, ‘average’, ‘single’},\n",
    "# default=’ward’\n",
    "#    compute_distances='False') #usefull for dendrogram\n",
    "\n",
    "j = 0\n",
    "for i in [2, 3, 4, 5, 6, 7]:\n",
    "    j = j+1\n",
    "    # init\n",
    "    mod_asc_1 = AgglomerativeClustering(n_clusters=i, linkage='ward')\n",
    "    mod_asc_2 = AgglomerativeClustering(n_clusters=i, linkage='complete')\n",
    "    mod_asc_3 = AgglomerativeClustering(n_clusters=i, linkage='average')\n",
    "    mod_asc_4 = AgglomerativeClustering(n_clusters=i, linkage='single')\n",
    "    mod_km = KMeans(i)\n",
    "    # fit\n",
    "    start = time.time()\n",
    "    mod_asc_1.fit(data2mod)\n",
    "    end1 = time.time()-start\n",
    "    start = time.time()\n",
    "    mod_asc_2.fit(data2mod)\n",
    "    end2 = time.time()-start\n",
    "    start = time.time()\n",
    "    mod_asc_3.fit(data2mod)\n",
    "    end3 = time.time()-start\n",
    "    start = time.time()\n",
    "    mod_asc_4.fit(data2mod)\n",
    "    end4 = time.time()-start\n",
    "    start = time.time()\n",
    "    mod_km.fit(data2mod)\n",
    "    end0 = time.time()-start\n",
    "\n",
    "    # similitarity score\n",
    "    ari_1 = adjusted_rand_score(mod_asc_1.labels_, mod_km.labels_)\n",
    "    ari_2 = adjusted_rand_score(mod_asc_2.labels_, mod_km.labels_)\n",
    "    ari_3 = adjusted_rand_score(mod_asc_3.labels_, mod_km.labels_)\n",
    "    ari_4 = adjusted_rand_score(mod_asc_4.labels_, mod_km.labels_)\n",
    "\n",
    "    nmi_1 = normalized_mutual_info_score(mod_asc_1.labels_, mod_km.labels_)\n",
    "    nmi_2 = normalized_mutual_info_score(mod_asc_2.labels_, mod_km.labels_)\n",
    "    nmi_3 = normalized_mutual_info_score(mod_asc_3.labels_, mod_km.labels_)\n",
    "    nmi_4 = normalized_mutual_info_score(mod_asc_4.labels_, mod_km.labels_)\n",
    "\n",
    "    fms_1 = fowlkes_mallows_score(mod_asc_1.labels_, mod_km.labels_)\n",
    "    fms_2 = fowlkes_mallows_score(mod_asc_2.labels_, mod_km.labels_)\n",
    "    fms_3 = fowlkes_mallows_score(mod_asc_3.labels_, mod_km.labels_)\n",
    "    fms_4 = fowlkes_mallows_score(mod_asc_4.labels_, mod_km.labels_)\n",
    "    # dispersion score\n",
    "    # The best value is 1 and the worst value is -1\n",
    "    sil_1 = metrics.silhouette_score(data2mod, mod_asc_1.labels_)\n",
    "    sil_2 = metrics.silhouette_score(data2mod, mod_asc_2.labels_)\n",
    "    sil_3 = metrics.silhouette_score(data2mod, mod_asc_3.labels_)\n",
    "    sil_4 = metrics.silhouette_score(data2mod, mod_asc_4.labels_)\n",
    "    sil_0 = metrics.silhouette_score(data2mod, mod_km.labels_)\n",
    "    # The score is defined as ratio between the\n",
    "    # within-cluster dispersion and the between-cluster dispersion\n",
    "    cal_1 = metrics.calinski_harabasz_score(data2mod, mod_asc_1.labels_)\n",
    "    cal_2 = metrics.calinski_harabasz_score(data2mod, mod_asc_2.labels_)\n",
    "    cal_3 = metrics.calinski_harabasz_score(data2mod, mod_asc_3.labels_)\n",
    "    cal_4 = metrics.calinski_harabasz_score(data2mod, mod_asc_4.labels_)\n",
    "    cal_0 = metrics.calinski_harabasz_score(data2mod, mod_km.labels_)\n",
    "\n",
    "    # The minimum score is zero,\n",
    "    # with lower values indicating better clustering.\n",
    "    db_1 = metrics.davies_bouldin_score(data2mod, mod_asc_1.labels_)\n",
    "    db_2 = metrics.davies_bouldin_score(data2mod, mod_asc_2.labels_)\n",
    "    db_3 = metrics.davies_bouldin_score(data2mod, mod_asc_3.labels_)\n",
    "    db_4 = metrics.davies_bouldin_score(data2mod, mod_asc_4.labels_)\n",
    "    db_0 = metrics.davies_bouldin_score(data2mod, mod_km.labels_)\n",
    "\n",
    "    res = [[i,\n",
    "            end0, end1, end2, end3, end4,\n",
    "            sil_0, sil_1, sil_2, sil_3, sil_4,\n",
    "            cal_0, cal_1, cal_2, cal_3, cal_4,\n",
    "            db_0, db_1, db_2, db_3, db_4,\n",
    "            ari_1, ari_2, ari_3, ari_4,\n",
    "            nmi_1, nmi_2, nmi_3, nmi_4,\n",
    "            fms_1, fms_2, fms_3, fms_4]]\n",
    "\n",
    "    if j == 1:\n",
    "        resul = res\n",
    "    else:\n",
    "        resul = resul+res\n",
    "\n",
    "# graphiques de comparaison\n",
    "lisnam = ['nb_clus',\n",
    "          't_km', 't_ward', 't_comp', 't_ave', 't_sing',\n",
    "          'sil_km', 'sil_ward', 'sil_comp', 'sil_ave', 'sil_sing',\n",
    "          'cal_km', 'cal_ward', 'cal_comp', 'cal_ave', 'cal_sing',\n",
    "          'db_km', 'db_ward', 'db_comp', 'db_ave', 'db_sing',\n",
    "          'ari_ward', 'ari_comp', 'ari_ave', 'ari_sing',\n",
    "          'nmi_ward', 'nmi_comp', 'nmi_ave', 'nmi_sing',\n",
    "          'fms_ward', 'fms_comp', 'fms_ave', 'fms_sing']\n",
    "resultat = pd.DataFrame(resul, columns=lisnam)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7a6049ed",
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.rcParams[\"figure.figsize\"] = [16, 10]\n",
    "plt.rcParams[\"figure.autolayout\"] = True\n",
    "fig, axes = plt.subplots(2, 3)\n",
    "\n",
    "axes[0, 0].plot(resultat.nb_clus, resultat.t_km, marker='o', color='b')\n",
    "axes[0, 0].plot(resultat.nb_clus, resultat.t_ward, marker='+', color='g')\n",
    "axes[0, 0].plot(resultat.nb_clus, resultat.t_comp, marker='x', color='r')\n",
    "axes[0, 0].plot(resultat.nb_clus, resultat.t_ave, marker='s', color='c')\n",
    "axes[0, 0].plot(resultat.nb_clus, resultat.t_sing, marker=\"*\", color='m')\n",
    "axes[0, 0].set_title('Time comparison')\n",
    "axes[0, 0].set_xlabel('cluster nb')\n",
    "axes[0, 0].set_ylabel('time')\n",
    "axes[0, 0].legend(['kmeans', 'agglo_ward', 'agglo_comp',\n",
    "                  'agglo_mean', 'agglo_single'])\n",
    "\n",
    "axes[0, 1].plot(resultat.nb_clus, resultat.sil_km, marker='o', color='b')\n",
    "axes[0, 1].plot(resultat.nb_clus, resultat.sil_ward, marker='+', color='g')\n",
    "axes[0, 1].plot(resultat.nb_clus, resultat.sil_comp, marker='x', color='r')\n",
    "axes[0, 1].plot(resultat.nb_clus, resultat.sil_ave, marker='s', color='c')\n",
    "axes[0, 1].plot(resultat.nb_clus, resultat.sil_sing, marker=\"*\", color='m')\n",
    "axes[0, 1].set_title('Silhouette score comparison')\n",
    "axes[0, 1].set_xlabel('cluster nb')\n",
    "axes[0, 1].set_ylabel('score')\n",
    "axes[0, 1].legend(['kmeans', 'agglo_ward', 'agglo_comp',\n",
    "                  'agglo_mean', 'agglo_single'])\n",
    "\n",
    "axes[0, 2].plot(resultat.nb_clus, resultat.cal_km, marker='o', color='b')\n",
    "axes[0, 2].plot(resultat.nb_clus, resultat.cal_ward, marker='+', color='g')\n",
    "axes[0, 2].plot(resultat.nb_clus, resultat.cal_comp, marker='x', color='r')\n",
    "axes[0, 2].plot(resultat.nb_clus, resultat.cal_ave, marker='s', color='c')\n",
    "axes[0, 2].plot(resultat.nb_clus, resultat.cal_sing, marker=\"*\", color='m')\n",
    "axes[0, 2].set_title('Calinski score comparison')\n",
    "axes[0, 2].set_xlabel('cluster nb')\n",
    "axes[0, 2].set_ylabel('score')\n",
    "axes[0, 2].legend(['kmeans', 'agglo_ward', 'agglo_comp',\n",
    "                  'agglo_mean', 'agglo_single'])\n",
    "\n",
    "\n",
    "axes[1, 0].plot(resultat.nb_clus, resultat.ari_ward, marker='+', color='g')\n",
    "axes[1, 0].plot(resultat.nb_clus, resultat.ari_comp, marker='x', color='r')\n",
    "axes[1, 0].plot(resultat.nb_clus, resultat.ari_ave, marker='s', color='c')\n",
    "axes[1, 0].plot(resultat.nb_clus, resultat.ari_sing, marker=\"*\", color='m')\n",
    "axes[1, 0].set_title('adjusted rand index vs kmeans')\n",
    "axes[1, 0].set_xlabel('cluster nb')\n",
    "axes[1, 0].set_ylabel('score')\n",
    "axes[1, 0].legend(['agglo_ward', 'agglo_comp', 'agglo_mean', 'agglo_single'])\n",
    "\n",
    "\n",
    "axes[1, 1].plot(resultat.nb_clus, resultat.nmi_ward, marker='+', color='g')\n",
    "axes[1, 1].plot(resultat.nb_clus, resultat.nmi_comp, marker='x', color='r')\n",
    "axes[1, 1].plot(resultat.nb_clus, resultat.nmi_ave, marker='s', color='c')\n",
    "axes[1, 1].plot(resultat.nb_clus, resultat.nmi_sing, marker=\"*\", color='m')\n",
    "axes[1, 1].set_title('normalized mutual info vs kmeans ')\n",
    "axes[1, 1].set_xlabel('cluster nb')\n",
    "axes[1, 1].set_ylabel('score')\n",
    "axes[1, 1].legend(['agglo_ward', 'agglo_comp', 'agglo_mean', 'agglo_single'])\n",
    "\n",
    "\n",
    "axes[1, 2].plot(resultat.nb_clus, resultat.fms_ward, marker='+', color='g')\n",
    "axes[1, 2].plot(resultat.nb_clus, resultat.fms_comp, marker='x', color='r')\n",
    "axes[1, 2].plot(resultat.nb_clus, resultat.fms_ave, marker='s', color='c')\n",
    "axes[1, 2].plot(resultat.nb_clus, resultat.fms_sing, marker=\"*\", color='m')\n",
    "axes[1, 2].set_title('fowlkes mallows vs kmeans')\n",
    "axes[1, 2].set_xlabel('cluster nb')\n",
    "axes[1, 2].set_ylabel('score')\n",
    "axes[1, 2].legend(['agglo_ward', 'agglo_comp', 'agglo_mean', 'agglo_single'])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "748bd888",
   "metadata": {},
   "source": [
    "## stabilité temporelle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "53be8d8f",
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import MinMaxScaler\n",
    "from sklearn.cluster import KMeans\n",
    "from sklearn.metrics.cluster import adjusted_rand_score,\\\n",
    "    completeness_score, normalized_mutual_info_score, fowlkes_mallows_score\n",
    "from yellowbrick.cluster import KElbowVisualizer,\\\n",
    "    SilhouetteVisualizer, InterclusterDistance\n",
    "import time\n",
    "from scipy import stats\n",
    "\n",
    "\n",
    "def seg_stab(\n",
    "        tickweek=1,\n",
    "        mod=KMeans(4),\n",
    "        datinit='2017-10-01',\n",
    "        fac=['rec', 'nb_com_ltm', 'pr_mean'],\n",
    "        scaltype='Standard'):\n",
    "    \n",
    "    \"\"\"kmeans stability.\n",
    "        \n",
    "        permet de tracer L'adjusted rand index et d autres metrics entre\n",
    "        le predict du modèle initial et le fit d'un nouveau modèle\n",
    "    \n",
    "        tickweek: tick en nombre de semaines\n",
    "        mod: type de modele - kemans uniquement pour le moment\n",
    "        datinit: cut de selction de periode pour initialisation du modèle\n",
    "        fac: liste de features\n",
    "        scaltype: type de standardisation : 'Standard'\n",
    "    \"\"\"\n",
    "\n",
    "    start = time.time()\n",
    "    j = -tickweek\n",
    "    k = -1\n",
    "    # plus facile de faire un tick semaine qu'un type mois\n",
    "    # qui necessite une meilleure gestion (mois differents)\n",
    "\n",
    "    # initialisation du fichier client à T0=datinit\n",
    "    # (pour le scaler et la periode)\n",
    "    data2mod_init = seg_kmeans(\n",
    "        cutD=datinit,\n",
    "        scalerfitD=datinit,\n",
    "        cutO=False,\n",
    "        Standardization=True,\n",
    "        scal=scaltype,\n",
    "        lisfac=fac,\n",
    "        ElbowVisualizer=False,\n",
    "        Sil_ICD_Visualizer=False)\n",
    "    # modèle intial\n",
    "    model_init = mod\n",
    "    model_init.fit(data2mod_init)\n",
    "\n",
    "    for i in np.arange(np.datetime64(datinit),\n",
    "                       np.datetime64('2018-09-01'),\n",
    "                       np.timedelta64(tickweek, 'W')):\n",
    "        j = j+tickweek\n",
    "        k = k+1\n",
    "\n",
    "        # time to string\n",
    "        l = pd.to_datetime(str(i)).strftime('%Y-%m-%d')\n",
    "\n",
    "        # fichier client standardise intialement\n",
    "        data2mod1 = seg_kmeans(\n",
    "            cutD=l,\n",
    "            scalerfitD=datinit,\n",
    "            cutO=False,\n",
    "            Standardization=True,\n",
    "            scal=scaltype,\n",
    "            lisfac=fac,\n",
    "            ElbowVisualizer=False,\n",
    "            Sil_ICD_Visualizer=False)\n",
    "\n",
    "        # fichier client standardisé ulterieurement\n",
    "        data2mod2 = seg_kmeans(\n",
    "            cutD=l,\n",
    "            scalerfitD=l,\n",
    "            cutO=False,\n",
    "            Standardization=True,\n",
    "            scal=scaltype,\n",
    "            lisfac=fac,\n",
    "            ElbowVisualizer=False,\n",
    "            Sil_ICD_Visualizer=False)\n",
    "\n",
    "        # prediction à partir du modèle intial\n",
    "        mod_labels1 = pd.DataFrame(model_init.predict(data2mod1), columns=[\n",
    "                                   'A'+str(j)], index=data2mod1.index)\n",
    "\n",
    "        # fit d'un nouveau modèle uleterieur\n",
    "        model_ult = mod\n",
    "        model_ult.fit(data2mod2)\n",
    "        mod_labels2 = pd.DataFrame(model_ult.labels_, columns=[\n",
    "                                   'B'+str(j)], index=data2mod2.index)\n",
    "\n",
    "        mod_labels = mod_labels1.join(mod_labels2)\n",
    "\n",
    "        if k == 0:\n",
    "            ari = [[j, adjusted_rand_score(\n",
    "                mod_labels['A'+str(j)], mod_labels['B'+str(j)])]]\n",
    "            cs = [[j, completeness_score(\n",
    "                mod_labels['A'+str(j)], mod_labels['B'+str(j)])]]\n",
    "            nmi = [[j, normalized_mutual_info_score(\n",
    "                mod_labels['A'+str(j)], mod_labels['B'+str(j)])]]\n",
    "            fms = [[j, fowlkes_mallows_score(\n",
    "                mod_labels['A'+str(j)], mod_labels['B'+str(j)])]]\n",
    "\n",
    "            nbc = [[j, mod_labels.shape[0]]]\n",
    "        else:\n",
    "            ari = ari + \\\n",
    "                [[j, adjusted_rand_score(\n",
    "                    mod_labels['A'+str(j)], mod_labels['B'+str(j)])]]\n",
    "            cs = cs + \\\n",
    "                [[j, completeness_score(\n",
    "                    mod_labels['A'+str(j)], mod_labels['B'+str(j)])]]\n",
    "            nmi = nmi + \\\n",
    "                [[j, normalized_mutual_info_score(\n",
    "                    mod_labels['A'+str(j)], mod_labels['B'+str(j)])]]\n",
    "            fms = fms + \\\n",
    "                [[j, fowlkes_mallows_score(\n",
    "                    mod_labels['A'+str(j)], mod_labels['B'+str(j)])]]\n",
    "\n",
    "            nbc = nbc+[[j, mod_labels.shape[0]]]\n",
    "            # le rand score est une generalisation de l'accuracy binaire\n",
    "            # l'adjusted normalise en comparant a un rand score aleatoire\n",
    "\n",
    "    ari = pd.DataFrame(ari, columns=['weeks', 'ari'])\n",
    "    cs = pd.DataFrame(cs, columns=['weeks', 'cs'])\n",
    "    nmi = pd.DataFrame(nmi, columns=['weeks', 'nmi'])\n",
    "    fms = pd.DataFrame(fms, columns=['weeks', 'fms'])\n",
    "\n",
    "    nbc = pd.DataFrame(nbc, columns=['weeks', 'nbc'])\n",
    "    nbc['pcc'] = 100*((nbc['nbc']-nbc['nbc'][0])/nbc['nbc'][0])\n",
    "\n",
    "    plt.rcParams[\"figure.figsize\"] = [16, 5]\n",
    "    plt.rcParams[\"figure.autolayout\"] = True\n",
    "    fig, axes = plt.subplots(1, 2)\n",
    "\n",
    "    axes[0].plot(ari.weeks, ari.ari, marker='o', color='b')\n",
    "    axes[0].plot(ari.weeks, cs.cs, marker='+', color='g')\n",
    "    axes[0].plot(ari.weeks, nmi.nmi, marker='x', color='r')\n",
    "    axes[0].plot(ari.weeks, fms.fms, marker='s', color='c')\n",
    "\n",
    "    axes[0].set_title(\n",
    "        'Smiliarity score between label mod and n weeks lagged labels')\n",
    "    axes[0].set_xlabel('weeks')\n",
    "    axes[0].set_ylabel('Similarity score')\n",
    "\n",
    "    axes[0].legend(['Adjusted Rand Index', 'Completeness',\n",
    "                   'Normalized Mutual Info', 'Fowlkes Mallows'])\n",
    "\n",
    "    axes[1].plot(nbc.weeks, nbc.pcc, marker='o', color='b',\n",
    "                 label='croissance de la clientèle')\n",
    "    axes[1].set_title('croissance du nombre de clients')\n",
    "    axes[1].set_xlabel('weeks')\n",
    "    axes[1].set_ylabel('Pourcentage')\n",
    "    plt.legend()\n",
    "    plt.show()\n",
    "\n",
    "    end = time.time()\n",
    "    print('computation time for', k, 'iterations:', {end - start})\n",
    "\n",
    "    return\n",
    "\n",
    "\n",
    "seg_stab()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8ae4b59a",
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import MinMaxScaler\n",
    "from sklearn.cluster import KMeans\n",
    "from sklearn.metrics.cluster import adjusted_rand_score,\\\n",
    "    completeness_score, normalized_mutual_info_score, fowlkes_mallows_score\n",
    "from yellowbrick.cluster import KElbowVisualizer,\\\n",
    "    SilhouetteVisualizer, InterclusterDistance\n",
    "import time\n",
    "from scipy import stats\n",
    "\n",
    "\n",
    "def seg_stab_research(\n",
    "        tickweek=1,\n",
    "        mod=KMeans(6),\n",
    "        datinit='2017-10-01',\n",
    "        fac=['rec', 'nb_com_ltm', 'pr_mean'],\n",
    "        scaltype='Standard',\n",
    "        metric='ARI',\n",
    "        seuil=0.8):\n",
    "    \n",
    "    \"\"\"kmeans stability research.\n",
    "        \n",
    "        permet de trouver le temps nécéssaire entre 2 remodélisations.\n",
    "        Pour chaque point dans le temps , on observe à partir de quand une\n",
    "        metrique comme l'adjusted rand index ou autres passe sous un\n",
    "        certain seuil\n",
    "        la métrique compare toujours le resultat du  predict du modèle\n",
    "        initial à t0 et le fit d'un nouveau modèle de 1 à N tickweek aprés\n",
    "        t0 (temps du modèle initial)\n",
    "        t0 va varier jusqu'a la fin de la periode disponible\n",
    "    \n",
    "        tickweek: tick en nombre de semaines\n",
    "        mod: type de modele - kemans uniquement pour le moment\n",
    "        datinit: premier t0\n",
    "        fac: liste de features\n",
    "        scaltype: type de standardisation : 'Standard'\n",
    "        metric: \n",
    "            'ARI':adjusted_rand_score\n",
    "            'CS':completeness_score\n",
    "            'NMI':normalized_mutual_info_score\n",
    "            'FMS':fowlkes_mallows_score\n",
    "        seuil: seuil de la metrique sous laquelle il doit y avoir\n",
    "        remodelisation\n",
    "\n",
    "    \"\"\"\n",
    "\n",
    "    periode = np.arange(np.datetime64(datinit), np.datetime64(\n",
    "        '2018-09-01'), np.timedelta64(tickweek, 'W'))  # periode\n",
    "    s = periode.size  # taille de l'array periode initiale\n",
    "    res = [[-1, datinit, datinit, 0, 0, 0, 0, 0, 0, 0, 0, 0]]\n",
    "    print(s, 'iterations')\n",
    "\n",
    "    flagvertical = 100\n",
    "\n",
    "    # iteration de recherche\n",
    "    for iteration in range(0, s):\n",
    "        start = time.time()  # time\n",
    "        # initialisation\n",
    "        j = -tickweek\n",
    "        k = -1\n",
    "        score = 1  # initialisation du score à 1 > seuil\n",
    "        flag = False\n",
    "\n",
    "        m = pd.to_datetime(str(periode[0])).strftime(\n",
    "            '%Y-%m-%d')  # initialisation de la date de scaler\n",
    "        # toujours la premiere car on on raccourci\n",
    "        # par le debut le range periode\n",
    "\n",
    "        # initialisation du fichier client à T0=datinit\n",
    "        # (pour le scaler et la periode)\n",
    "        data2mod_init = seg_kmeans(\n",
    "            cutD=m,\n",
    "            scalerfitD=m,\n",
    "            cutO=False,\n",
    "            Standardization=True,\n",
    "            scal=scaltype,\n",
    "            lisfac=fac,\n",
    "            ElbowVisualizer=False,\n",
    "            Sil_ICD_Visualizer=False)\n",
    "        # modèle intial\n",
    "        model_init = mod\n",
    "        model_init.fit(data2mod_init)\n",
    "\n",
    "        # periode (range de date)\n",
    "        for i in periode:\n",
    "            j = j+tickweek  # compteur de temps (commence à 0)\n",
    "            k = k+1  # compteur de periodes testées (commence à 0)\n",
    "\n",
    "            # selection de la periode à cluster\n",
    "            # time to string\n",
    "            l = pd.to_datetime(str(i)).strftime('%Y-%m-%d')\n",
    "\n",
    "            # fichier client standardise intialement\n",
    "            data2mod1 = seg_kmeans(\n",
    "                cutD=l,\n",
    "                scalerfitD=m,\n",
    "                cutO=False,\n",
    "                Standardization=True,\n",
    "                scal=scaltype,\n",
    "                lisfac=fac,\n",
    "                ElbowVisualizer=False,\n",
    "                Sil_ICD_Visualizer=False)\n",
    "\n",
    "            # fichier client standardise ulterieurement\n",
    "            data2mod2 = seg_kmeans(\n",
    "                cutD=l,\n",
    "                scalerfitD=l,\n",
    "                cutO=False,\n",
    "                Standardization=True,\n",
    "                scal=scaltype,\n",
    "                lisfac=fac,\n",
    "                ElbowVisualizer=False,\n",
    "                Sil_ICD_Visualizer=False)\n",
    "\n",
    "            # prediction à partir du modèle intial\n",
    "            mod_labels1 = pd.DataFrame(model_init.predict(data2mod1),\n",
    "                                       columns=[\n",
    "                                       'A'+str(j)], index=data2mod1.index)\n",
    "\n",
    "            # fit d'un nouveau modèle uleterieur\n",
    "            model_ult = mod\n",
    "            model_ult.fit(data2mod2)\n",
    "            mod_labels2 = pd.DataFrame(model_ult.labels_,\n",
    "                                       columns=[\n",
    "                                           'B'+str(j)], index=data2mod2.index)\n",
    "\n",
    "            mod_labels = mod_labels1.join(mod_labels2)\n",
    "\n",
    "            if k == 0:\n",
    "                nbci = mod_labels.shape[0]\n",
    "\n",
    "            score1 = adjusted_rand_score(\n",
    "                mod_labels['A'+str(j)], mod_labels['B'+str(j)])\n",
    "            score2 = completeness_score(\n",
    "                mod_labels['A'+str(j)], mod_labels['B'+str(j)])\n",
    "            score3 = normalized_mutual_info_score(\n",
    "                mod_labels['A'+str(j)], mod_labels['B'+str(j)])\n",
    "            score4 = fowlkes_mallows_score(\n",
    "                mod_labels['A'+str(j)], mod_labels['B'+str(j)])\n",
    "\n",
    "            if metric == 'ARI':\n",
    "                score = score1\n",
    "            elif metric == 'CS':\n",
    "                score = score2\n",
    "            elif metric == 'NMI':\n",
    "                score = score3\n",
    "            elif metric == 'FMS':\n",
    "                score = score4\n",
    "\n",
    "            if score <= seuil:\n",
    "                res = res+[[iteration, m, l, j, k, seuil, score1,\n",
    "                            score2,\n",
    "                            score3, score4, mod_labels.shape[0], nbci]]\n",
    "                flag = True\n",
    "                break  # pour sortir de la 2ème boucle de recherche\n",
    "\n",
    "        # on cut la premiere valeur du array periode\n",
    "        periode = np.delete(periode, 0)\n",
    "        # on rempli res si seuil non franchi\n",
    "        if flag == False:\n",
    "            res = res+[[iteration, m, l, np.nan, np.nan, seuil, score1,\n",
    "                        score2, score3, score4, mod_labels.shape[0], nbci]]\n",
    "            # pour ligne verticale de seuil non atteint\n",
    "            if flagvertical == 100:\n",
    "                flagvertical = iteration-1\n",
    "\n",
    "        end = time.time()\n",
    "        if iteration == 0:\n",
    "            print('computation time pour une iteration:', {end - start})\n",
    "        # print('iterations:',iteration)\n",
    "\n",
    "    res = pd.DataFrame(res, columns=['iter', 'dat_in', 'dat_out', 'weeks',\n",
    "                       'tickweek', 'seuil_'+str(metric), 'ARI',\n",
    "                                     'CS', 'NMI', 'FMS', 'nbc', 'nbci'])\n",
    "    res['pcc'] = 100*((res['nbc']-res['nbci'])/res['nbci'])\n",
    "    # on retire la premiere ligne d'initialisation\n",
    "    res = res.loc[(res.iter > -1), :]\n",
    "\n",
    "    plt.rcParams[\"figure.figsize\"] = [16, 5]\n",
    "    plt.rcParams[\"figure.autolayout\"] = True\n",
    "    fig, axes = plt.subplots(1, 2)\n",
    "\n",
    "    slope, intercept, r_value, pv, se =\\\n",
    "        stats.linregress(res.loc[res.weeks.notna(\n",
    "        ), :]['iter'].to_numpy(),\n",
    "            res.loc[res.weeks.notna(), :]['weeks'].to_numpy())\n",
    "    sns.regplot(x=\"iter\", y=\"weeks\", data=res, ax=axes[0], ci=99,\n",
    "                label=\"y={0:.1f}x+{1:.1f}\".format(slope, intercept)).legend(\n",
    "        loc=\"best\")\n",
    "    # axes[0].plot(res.iter,res.weeks,'s')\n",
    "    axes[0].set_title(\n",
    "        'Nombre de semaines avant remodélisation: seuil_'+str(metric) +\n",
    "        ':'+str(seuil))\n",
    "    axes[0].set_xlabel('itérations')\n",
    "    axes[0].set_ylabel('weeks')\n",
    "\n",
    "    axes[1].plot(res.iter, res.nbci, marker='o', color='b')\n",
    "    axes[1].plot(res.iter, res.nbc, marker='+', color='g')\n",
    "    # , label='seuil non atteint')\n",
    "    axes[1].axvline(flagvertical, 0, 1, linestyle='--', color='r')\n",
    "    axes[1].set_title('nombre de clients')\n",
    "    axes[1].set_xlabel('itérations')\n",
    "    axes[1].set_ylabel('clientèle')\n",
    "    axes[1].legend(['nb client initial',\n",
    "                   'nb client seuil ou fin de periode',\n",
    "                    'zone de seuil non atteint'])\n",
    "    plt.legend\n",
    "    plt.show()\n",
    "\n",
    "    fig, axes = plt.subplots(1, 2)\n",
    "    axes[0].plot(res.iter, res['seuil_'+str(metric)],\n",
    "                 linestyle='--', color='m')\n",
    "    axes[0].plot(res.iter, res.ARI, marker='o', color='b')\n",
    "    axes[0].plot(res.iter, res.CS, marker='+', color='g')\n",
    "    axes[0].plot(res.iter, res.NMI, marker='x', color='r')\n",
    "    axes[0].plot(res.iter, res.FMS, marker='s', color='c')\n",
    "    # , label='seuil non atteint')\n",
    "    axes[0].axvline(flagvertical, 0, 1, linestyle='--', color='r')\n",
    "\n",
    "    axes[0].set_title('Les scores avant remodélisation')\n",
    "    axes[0].set_xlabel('iterations')\n",
    "    axes[0].set_ylabel('Similarity score')\n",
    "    axes[0].legend(['seuil_'+str(metric), 'Adjusted Rand Index',\n",
    "                    'Completeness',\n",
    "                   'Normalized Mutual Info', 'Fowlkes Mallows',\n",
    "                    'zone de seuil non atteint'])\n",
    "\n",
    "    axes[1].plot(res.iter, res.pcc, marker='o',\n",
    "                 color='b', label='croissance clientele')\n",
    "    axes[1].axvline(flagvertical, 0, 1, linestyle='--',\n",
    "                    color='r', label='zone de seuil non atteint')\n",
    "    axes[1].set_title(\n",
    "        'croissance de la clientèle correspondant au seuil_'+str(metric))\n",
    "    axes[1].set_xlabel('itérations')\n",
    "    axes[1].set_ylabel('croissance de clientèle')\n",
    "\n",
    "    plt.legend()\n",
    "    plt.show()\n",
    "\n",
    "    return(res)\n",
    "\n",
    "\n",
    "#res1 = seg_stab_research(tickweek=1, mod=KMeans(3))\n",
    "res2 = seg_stab_research(tickweek=1, mod=KMeans(4))\n",
    "res3 = seg_stab_research(tickweek=1, mod=KMeans(5))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fa5efdbc",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}