chat.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "chat.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "toc_visible": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3",
      "language": "python"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Q4qBbFmAzKWv",
        "outputId": "23b93c00-2658-411a-8966-a8a4157e57e8"
      },
      "source": [
        "\n",
        "!pip install emoji\n",
        "!pip install wordcloud\n",
        "!pip install datetime\n",
        "!pip install seaborn\n",
        "!pip install more-itertools\n",
        "!pip install numpy\n",
        "!pip install pandas\n",
        "!pip install matplotlib\n",
        "!pip install datetime\n",
        "!pip install pytest-warnings\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4ibqrmC70T97"
      },
      "source": [
        "import re\n",
        "\n",
        "\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "import matplotlib.pyplot as plt\n",
        "import seaborn as sns\n",
        "import wordcloud\n",
        "import emoji\n",
        "import itertools as it\n",
        "from collections import Counter\n",
        "import warnings\n",
        "from collections import OrderedDict \n",
        "import collections\n",
        "import datetime\n",
        "\n",
        "%matplotlib inline\n",
        "warnings.filterwarnings('ignore')\n",
        "#len('hsh')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oNfhsxhI0dqS"
      },
      "source": [
        "def rawToDf(file, key):\n",
        "    '''Converts raw .txt file into a Data Frame'''\n",
        "    \n",
        "    split_formats = {\n",
        "        '12hr' : '\\d{1,2}/\\d{1,2}/\\d{2,4},\\s\\d{1,2}:\\d{2}\\s[APap][mM]\\s-\\s',\n",
        "        '24hr' : '\\d{1,2}/\\d{1,2}/\\d{2,4},\\s\\d{1,2}:\\d{2}\\s-\\s',\n",
        "        'custom' : ''\n",
        "    }\n",
        "    datetime_formats = {\n",
        "        '12hr' : '%d/%m/%Y, %I:%M %p - ',\n",
        "        '24hr' : '%m/%d/%y, %H:%M - ',\n",
        "        'custom': ''\n",
        "    }\n",
        "    \n",
        "    with open(file, 'r', encoding='utf-8') as raw_data:\n",
        "        #print(raw_data.read())\n",
        "        raw_string = ' '.join(raw_data.read().split('\\n')) # converting the list split by newline char. as one whole string as there can be multi-line messages\n",
        "        #print(raw_string)\n",
        "        user_msg = re.split(split_formats[key], raw_string)[1:] # splits at all the date-time pattern, resulting in list of all the messages with user names\n",
        "        \n",
        "        date_time = re.findall(split_formats[key], raw_string) # finds all the date-time patterns\n",
        "        \n",
        "        df = pd.DataFrame({'date_time': date_time, 'user_msg': user_msg}) # exporting it to a df\n",
        "        \n",
        "    # converting date-time pattern which is of type String to type datetime,\n",
        "    # format is to be specified for the whole string where the placeholders are extracted by the method \n",
        "    df['date_time'] = pd.to_datetime(df['date_time'], format=datetime_formats[key])\n",
        "    \n",
        "    # split user and msg \n",
        "    usernames = []\n",
        "    msgs = []\n",
        "    for i in df['user_msg']:\n",
        "        a = re.split('([\\w\\W]+?):\\s', i) # lazy pattern match to first {user_name}: pattern and spliting it aka each msg from a user\n",
        "        if(a[1:]): # user typed messages\n",
        "            usernames.append(a[1])\n",
        "            msgs.append(a[2])\n",
        "        else: # other notifications in the group(eg: someone was added, some left ...)\n",
        "            usernames.append(\"group_notification\")\n",
        "            msgs.append(a[0])\n",
        "        #print(msgs[i])\n",
        "\n",
        "    # creating new columns  \n",
        "       \n",
        "    df['user'] = usernames\n",
        "    df['message'] = msgs\n",
        "\n",
        "    # dropping the old user_msg col.\n",
        "    df.drop('user_msg', axis=1, inplace=True)\n",
        "    \n",
        "    return df"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 476
        },
        "id": "4xKVEvKp1ZHe",
        "outputId": "03e50e87-8c14-4230-f314-b0c51647d5b1"
      },
      "source": [
        "chat=input('Enter path of file: ')\n",
        "format=input('Enter time format 12hr or 24hr: ')\n",
        "name=input('Enter group name: ')\n",
        "df= rawToDf(chat,format)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6zjpmusb1gqb"
      },
      "source": [
        "df['day']=df['date_time'].dt.strftime('%a')\n",
        "df['month']=df['date_time'].dt.strftime('%b')\n",
        "df['year']=df['date_time'].dt.year\n",
        "df['date']=df['date_time'].apply(lambda x:x.date())"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RmJX6EvI5KVv"
      },
      "source": [
        "na=[]\n",
        "for i in df.date_time:\n",
        "  timestamp = datetime.datetime.timestamp(i)\n",
        "  na.append(timestamp)\n",
        "ar=np.array(na)\n",
        "df['timestamp']=ar"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yyX4dk048cOq"
      },
      "source": [
        "\n",
        "def dayOfYear(date):\n",
        "      days = [0,31,28,31,30,31,30,31,31,30,31,30,31]\n",
        "      d = list(map(int,date.split(\"-\")))\n",
        "      if d[0] % 400 == 0:\n",
        "         days[2]+=1\n",
        "      elif d[0]%4 == 0 and d[0]%100!=0:\n",
        "         days[2]+=1\n",
        "      for i in range(1,len(days)):\n",
        "         days[i]+=days[i-1]\n",
        "      return days[d[1]-1]+d[2]\n",
        "\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YtwJuGutKb1Y",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "7d7c4978-37ca-4dc7-88b2-bf279a26518b"
      },
      "source": [
        "dataset=df.values"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "17BMOeU3KKnV",
        "outputId": "f9aaf906-2065-4ade-cd4c-c501bbd4fce9"
      },
      "source": [
        "start=dataset[0][0].strftime('%Y-%m-%d')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nemMby9sLmtY"
      },
      "source": [
        "date={}\n",
        "msgLen={}\n",
        "for i in range(39999):\n",
        "  date[dataset[i][6].strftime(\"%Y-%m-%d\")]=dataset[i][6].strftime(\"%Y-%m-%d\")\n",
        "  msgLen[dataset[i][6].strftime(\"%Y-%m-%d\")]= msgLen.get(dataset[i][6].strftime(\"%Y-%m-%d\"),0)+1\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "y_dC3hi-QcQ9"
      },
      "source": [
        "X=[]\n",
        "Y=[]\n",
        "for i in date:\n",
        "  X.append(i)\n",
        "  Y.append(msgLen[i])\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oo3wxSp8RlIS",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 294
        },
        "outputId": "9451ef3f-5986-48f7-fd73-e5417e867893"
      },
      "source": [
        "\n",
        "fig, ax = plt.subplots(1, figsize=(20, 6))\n",
        "major_ticks = np.arange(0, 50, 5)\n",
        "ax.set_xticks(major_ticks)\n",
        "ax.plot(X,Y)\n",
        "plt.title('Messages sent per day')\n",
        "plt.grid(b=True, which='major', color='#666666', linestyle='-',lw='0.5')\n",
        "plt.savefig('pic1.png')\n",
        "plt.show()\n",
        "\n",
        "\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "olKKNnB4cH15"
      },
      "source": [
        "userMsg={}\n",
        "for i in df.user:\n",
        "  userMsg[i]=userMsg.get(i,0)+1\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZgKlOf7vdYfV",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 336
        },
        "outputId": "87ee26ed-82a8-45dc-9de4-dc3b358fc143"
      },
      "source": [
        "user=userMsg.keys()\n",
        "msgCount=userMsg.values()\n",
        "fig = plt.figure(figsize = (15, 5)) \n",
        "plt.bar(user,msgCount,align='center',width=0.4)\n",
        "plt.xlabel(\"User name\") \n",
        "plt.ylabel(\"Total messages since nov 21\") \n",
        "plt.title(name) \n",
        "plt.savefig('pic2.png')\n",
        "plt.show() \n",
        "\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9d3OYgjhrPwA",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "0537f5f3-0ee8-47be-de71-adfdf3da9cd5"
      },
      "source": [
        "\n",
        "sorted_values = sorted(userMsg.values()) # Sort the values\n",
        "sorted_dict = {}\n",
        "\n",
        "for i in sorted_values:\n",
        "    for k in userMsg.keys():\n",
        "        if userMsg[k] == i:\n",
        "            sorted_dict[k] = userMsg[k]\n",
        "            break\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oJR-cvoBsr0e",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "03408510-e733-4889-97bc-541ae42150c0"
      },
      "source": [
        "out = dict(it.islice(sorted_dict.items(), 5))\n",
        "tot=len(sorted_dict) \n",
        "new_dict= dict(it.islice(sorted_dict.items(), tot-5,tot))\n",
        "new_dict['others']=sum(out.values())\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FsnP5L0vllh8",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 503
        },
        "outputId": "986bb67c-0395-4495-93ec-164595a4e474"
      },
      "source": [
        "def func(pct, allvalues): \n",
        "    absolute = int((pct / 100.0)*(39999.0)) \n",
        "    return \"{:.1f}%\\n({:d})\".format(pct, absolute) \n",
        "  \n",
        "# Creating plot \n",
        "\n",
        "wp = { 'linewidth' : 1, 'edgecolor' : \"green\" } \n",
        "\n",
        "fig, ax = plt.subplots(figsize =(10, 7)) \n",
        "wedges, texts, autotexts = ax.pie(new_dict.values(),  \n",
        "                                  autopct = lambda pct: func(pct, new_dict.values()), \n",
        "                                  labels = new_dict.keys(), \n",
        "                                  shadow = True,\n",
        "                                  startangle = 90, \n",
        "                                  wedgeprops = wp, \n",
        "                                  textprops = dict(color =\"black\")) \n",
        "ax.legend(wedges, new_dict.keys(), \n",
        "          title =\"Username\", \n",
        "          loc =\"center left\", \n",
        "          bbox_to_anchor =(1, 0.6, 0.5, 1)) \n",
        "  \n",
        "plt.setp(autotexts, size = 8, weight =\"bold\") \n",
        "ax.set_title(\"User messages chart\") \n",
        "  \n",
        "# show plot \n",
        "plt.savefig('pic3.png')\n",
        "plt.show() \n",
        "\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "u2CfWeYX9t6u"
      },
      "source": [
        "wordList={}\n",
        "for i in df.message:\n",
        "  words=i.split()\n",
        "  for word in words:\n",
        "    if(word.isalpha()==False):\n",
        "      continue\n",
        "    if(len(word)<=1):\n",
        "      continue\n",
        "    word=word.lower()\n",
        "    word = word.replace(\".\",\"\")\n",
        "    word = word.replace(\",\",\"\")\n",
        "    word = word.replace(\":\",\"\")\n",
        "    word = word.replace(\"\\\"\",\"\")\n",
        "    word = word.replace(\"!\",\"\")\n",
        "    word = word.replace(\"â€œ\",\"\")\n",
        "    word = word.replace(\"â€˜\",\"\")\n",
        "    word = word.replace(\"*\",\"\")\n",
        "    wordList[word]=wordList.get(word,0)+1\n",
        "wordList=OrderedDict(sorted(wordList.items())) \n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "LwImCuHSlotM",
        "outputId": "59b1bc76-8278-4558-8673-09c74189ec03"
      },
      "source": [
        "n_print=10\n",
        "word_counter = collections.Counter(wordList)\n",
        "top10words={}\n",
        "for word, count in word_counter.most_common(n_print):\n",
        "    \n",
        "    top10words[word]=count"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 337
        },
        "id": "NXPrGSOhlrWb",
        "outputId": "4916ecbb-0e0c-467d-a34c-af8d70ebeb1e"
      },
      "source": [
        "fig = plt.figure(figsize = (15, 5)) \n",
        "plt.bar(top10words.keys(),top10words.values(),align='center',width=0.4)\n",
        "plt.xlabel(\"Word\") \n",
        "plt.ylabel(\"Count\") \n",
        "plt.title(name)\n",
        "plt.savefig('pic4.png') \n",
        "plt.show()\n",
        "\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oaXoBj8FluHO"
      },
      "source": [
        "msgTime={}\n",
        "for i in range(39999):\n",
        "  h=dataset[i][0].strftime('%H')\n",
        "  msgTime[h]=msgTime.get(h,0)+1"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 370
        },
        "id": "VuvGBdwBlxAZ",
        "outputId": "cdc38f99-ba7f-4591-ef67-aa14b2d0fc9f"
      },
      "source": [
        "msgTime=OrderedDict(sorted(msgTime.items()))\n",
        "fig = plt.figure(figsize = (15, 5)) \n",
        "plt.bar(msgTime.keys(),msgTime.values(),align='center',width=0.5)\n",
        "plt.xlabel(\"Hour\") \n",
        "plt.ylabel(\"Message count\") \n",
        "plt.title(\"Most active hours of \"+name) \n",
        "plt.savefig('pic5.png')\n",
        "plt.show() \n",
        "\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "df8XVuw-qaDB"
      },
      "source": [
        "daywise={}\n",
        "for i in df.date:\n",
        "  daywise[i]=daywise.get(i,0)+1\n",
        "daywise=OrderedDict(sorted(daywise.items()))\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "AaCe8lrks3KK",
        "outputId": "56ef194d-027f-4fe6-adf8-40578d6160bb"
      },
      "source": [
        "n_print=10\n",
        "\n",
        "day_counter = collections.Counter(daywise)\n",
        "\n",
        "top10days={}\n",
        "n_print=10\n",
        "day_counter.most_common(n_print)\n",
        "for day,count in day_counter.most_common(n_print):\n",
        "    \n",
        "    top10days[day.strftime(\"%d\")+'/'+day.strftime(\"%m\")+'/'+day.strftime(\"%Y\")]=count\n",
        "\n",
        "\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 318
        },
        "id": "A-1aj6SCtdBx",
        "outputId": "29be042f-7bce-4c0b-8f00-1b43515330bf"
      },
      "source": [
        "x=list(top10days.keys())\n",
        "y=list(top10days.values())\n",
        "\n",
        "fig = plt.figure(figsize = (16, 5)) \n",
        "plt.bar(x,y,align='center',width=0.5)\n",
        "plt.xlabel(\"Day\") \n",
        "plt.ylabel(\"Message Count\") \n",
        "plt.title(name+\" Most active days\")\n",
        "plt.savefig('pic6.png') \n",
        "plt.show() \n",
        "\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": []
    }
  ]
}