methylation_data_cleaning.ipynb

{
 "metadata": {
  "name": "",
  "signature": "sha256:4c649d849b56b45647f5f605361c07efd5010f5099a2ba0459e0ffc2925a6aef"
 },
 "nbformat": 3,
 "nbformat_minor": 0,
 "worksheets": [
  {
   "cells": [
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "import synapseclient\n",
      "import pandas as pd\n",
      "import sys\n",
      "import os\n",
      "%pylab inline"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Populating the interactive namespace from numpy and matplotlib\n"
       ]
      }
     ],
     "prompt_number": 195
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "sys.path.append(os.environ['HOME'] + \"/dev/appys/lib/\")\n",
      "import synapseutils"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 196
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "syn = synapseclient.login()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "Welcome, Abhishek Pratap!\n"
       ]
      }
     ],
     "prompt_number": 197
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "meth_to_gene = syn.get('syn2324928')\n",
      "meth_to_gene = pd.DataFrame.from_csv(meth_to_gene.path,sep='\\t')\n",
      "\n",
      "meth_data = syn.get('syn2233188')\n",
      "meth_data = pd.DataFrame.from_csv(meth_data.path,sep='\\t')"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 203
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#only keep those probes which have a corresponding Entrez\n",
      "meth_to_gene = meth_to_gene[meth_to_gene.EntrezID.notnull()]\n",
      "meth_to_gene = meth_to_gene['EntrezID']\n",
      "\n",
      "#only keep those probes in the meth_data which have a EntrezID\n",
      "meth_data = meth_data.loc[meth_to_gene.index]\n",
      "\n",
      "#only keep those probes that have variance > .01\n",
      "meth_data = meth_data[meth_data.var(axis=1) > .01]\n",
      "\n",
      "#only keep those probes in meth_to_genes which are present in filtered meth_data\n",
      "meth_to_gene = meth_to_gene.loc[meth_data.index]"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 204
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#split the multiple entrez ids for each probe\n",
      "def  temp_func(x):\n",
      "    ids=x['EntrezID'].values[0].split(';') \n",
      "    df = pd.DataFrame({'entrezId':ids})\n",
      "    return df\n",
      "\n",
      "meth_to_gene = meth_to_gene.reset_index()\n",
      "grp = meth_to_gene.groupby(['index'])\n",
      "\n",
      "meth_to_gene = grp.apply(temp_func).reset_index()\n",
      "meth_to_gene = meth_to_gene[['index', 'entrezId']]"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 208
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#upload filtered meth_data to synapse\n",
      "meth_data.to_csv('methData_filtered.csv', sep=\"\\t\", header=True, index=True)\n",
      "_ = syn.store(synapseclient.File('methData_filtered.csv', parentId='syn2731118'), used='syn2233188', \n",
      "          executed='https://raw.githubusercontent.com/apratap/PCBC_HeatMap_app/dev/methylation_data_cleaning.ipynb')\n",
      "\n",
      "! rm methData_filtered.csv"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        ","
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        ","
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        ","
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        ","
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        ","
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        ","
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "!"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "!"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "!"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "!\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Upload completed in 1 minute, 9 seconds.\n"
       ]
      }
     ],
     "prompt_number": 210
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "meth_to_gene.columns = ['methProbeID', 'entrezID']\n",
      "#upload filtered meth_data to synapse\n",
      "meth_to_gene.to_csv('meth_to_gene.csv', sep=\"\\t\", header=True, index=False)\n",
      "_ = syn.store(synapseclient.File('meth_to_gene.csv', parentId='syn2731118'), used='syn2324928', \n",
      "          executed='https://raw.githubusercontent.com/apratap/PCBC_HeatMap_app/dev/methylation_data_cleaning.ipynb')\n",
      "\n",
      "! rm meth_to_gene.csv"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        ","
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "!"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "!"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "!\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Upload completed in 5 seconds.\n"
       ]
      }
     ],
     "prompt_number": 211
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [],
     "language": "python",
     "metadata": {},
     "outputs": []
    }
   ],
   "metadata": {}
  }
 ]
}