wiki_and_w2v_viz.ipynb

{
 "cells": [
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# wiki dump"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO: Preprocessing 'data/enwiki-20230220-pages-articles-multistream.xml.bz2' to collect template definitions: this may take some time.\n",
      "INFO: Preprocessed 100000 pages\n",
      "INFO: Preprocessed 200000 pages\n",
      "INFO: Preprocessed 300000 pages\n",
      "INFO: Preprocessed 400000 pages\n",
      "INFO: Preprocessed 500000 pages\n",
      "INFO: Preprocessed 600000 pages\n",
      "INFO: Preprocessed 700000 pages\n",
      "INFO: Preprocessed 800000 pages\n",
      "INFO: Preprocessed 900000 pages\n",
      "INFO: Preprocessed 1000000 pages\n",
      "INFO: Preprocessed 1100000 pages\n",
      "INFO: Preprocessed 1200000 pages\n",
      "INFO: Preprocessed 1300000 pages\n",
      "INFO: Preprocessed 1400000 pages\n",
      "INFO: Preprocessed 1500000 pages\n",
      "INFO: Preprocessed 1600000 pages\n",
      "INFO: Preprocessed 1700000 pages\n",
      "INFO: Preprocessed 1800000 pages\n",
      "INFO: Preprocessed 1900000 pages\n",
      "INFO: Preprocessed 2000000 pages\n",
      "INFO: Preprocessed 2100000 pages\n",
      "INFO: Preprocessed 2200000 pages\n",
      "INFO: Preprocessed 2300000 pages\n",
      "INFO: Preprocessed 2400000 pages\n",
      "INFO: Preprocessed 2500000 pages\n",
      "INFO: Preprocessed 2600000 pages\n",
      "INFO: Preprocessed 2700000 pages\n",
      "INFO: Preprocessed 2800000 pages\n",
      "INFO: Preprocessed 2900000 pages\n",
      "INFO: Preprocessed 3000000 pages\n",
      "INFO: Preprocessed 3100000 pages\n",
      "INFO: Preprocessed 3200000 pages\n",
      "INFO: Preprocessed 3300000 pages\n",
      "INFO: Preprocessed 3400000 pages\n",
      "INFO: Preprocessed 3500000 pages\n",
      "INFO: Preprocessed 3600000 pages\n",
      "INFO: Preprocessed 3700000 pages\n",
      "INFO: Preprocessed 3800000 pages\n",
      "INFO: Preprocessed 3900000 pages\n",
      "INFO: Preprocessed 4000000 pages\n",
      "INFO: Preprocessed 4100000 pages\n",
      "INFO: Preprocessed 4200000 pages\n",
      "INFO: Preprocessed 4300000 pages\n",
      "INFO: Preprocessed 4400000 pages\n",
      "INFO: Preprocessed 4500000 pages\n",
      "INFO: Preprocessed 4600000 pages\n",
      "INFO: Preprocessed 4700000 pages\n",
      "INFO: Preprocessed 4800000 pages\n",
      "INFO: Preprocessed 4900000 pages\n",
      "INFO: Preprocessed 5000000 pages\n",
      "INFO: Preprocessed 5100000 pages\n",
      "INFO: Preprocessed 5200000 pages\n",
      "INFO: Preprocessed 5300000 pages\n",
      "INFO: Preprocessed 5400000 pages\n",
      "INFO: Preprocessed 5500000 pages\n",
      "INFO: Preprocessed 5600000 pages\n",
      "INFO: Preprocessed 5700000 pages\n",
      "INFO: Preprocessed 5800000 pages\n",
      "INFO: Preprocessed 5900000 pages\n",
      "INFO: Preprocessed 6000000 pages\n",
      "INFO: Preprocessed 6100000 pages\n",
      "INFO: Preprocessed 6200000 pages\n",
      "INFO: Preprocessed 6300000 pages\n",
      "INFO: Preprocessed 6400000 pages\n",
      "INFO: Preprocessed 6500000 pages\n",
      "INFO: Preprocessed 6600000 pages\n",
      "INFO: Preprocessed 6700000 pages\n",
      "INFO: Preprocessed 6800000 pages\n",
      "INFO: Preprocessed 6900000 pages\n",
      "INFO: Preprocessed 7000000 pages\n",
      "INFO: Preprocessed 7100000 pages\n",
      "INFO: Preprocessed 7200000 pages\n",
      "INFO: Preprocessed 7300000 pages\n",
      "INFO: Preprocessed 7400000 pages\n",
      "INFO: Preprocessed 7500000 pages\n",
      "INFO: Preprocessed 7600000 pages\n",
      "INFO: Preprocessed 7700000 pages\n",
      "INFO: Preprocessed 7800000 pages\n",
      "INFO: Preprocessed 7900000 pages\n",
      "INFO: Preprocessed 8000000 pages\n",
      "INFO: Preprocessed 8100000 pages\n",
      "INFO: Preprocessed 8200000 pages\n",
      "INFO: Preprocessed 8300000 pages\n",
      "INFO: Preprocessed 8400000 pages\n",
      "INFO: Preprocessed 8500000 pages\n",
      "INFO: Preprocessed 8600000 pages\n",
      "INFO: Preprocessed 8700000 pages\n",
      "INFO: Preprocessed 8800000 pages\n",
      "INFO: Preprocessed 8900000 pages\n",
      "INFO: Preprocessed 9000000 pages\n",
      "INFO: Preprocessed 9100000 pages\n",
      "INFO: Preprocessed 9200000 pages\n",
      "INFO: Preprocessed 9300000 pages\n",
      "INFO: Preprocessed 9400000 pages\n",
      "INFO: Preprocessed 9500000 pages\n",
      "INFO: Preprocessed 9600000 pages\n",
      "INFO: Preprocessed 9700000 pages\n",
      "INFO: Preprocessed 9800000 pages\n",
      "INFO: Preprocessed 9900000 pages\n",
      "INFO: Preprocessed 10000000 pages\n",
      "INFO: Preprocessed 10100000 pages\n",
      "INFO: Preprocessed 10200000 pages\n",
      "INFO: Preprocessed 10300000 pages\n",
      "INFO: Preprocessed 10400000 pages\n",
      "INFO: Preprocessed 10500000 pages\n",
      "INFO: Preprocessed 10600000 pages\n",
      "INFO: Preprocessed 10700000 pages\n",
      "INFO: Preprocessed 10800000 pages\n",
      "INFO: Preprocessed 10900000 pages\n",
      "INFO: Preprocessed 11000000 pages\n",
      "INFO: Preprocessed 11100000 pages\n",
      "INFO: Preprocessed 11200000 pages\n",
      "INFO: Preprocessed 11300000 pages\n",
      "INFO: Preprocessed 11400000 pages\n",
      "INFO: Preprocessed 11500000 pages\n",
      "INFO: Preprocessed 11600000 pages\n",
      "INFO: Preprocessed 11700000 pages\n",
      "INFO: Preprocessed 11800000 pages\n",
      "INFO: Preprocessed 11900000 pages\n",
      "INFO: Preprocessed 12000000 pages\n",
      "INFO: Preprocessed 12100000 pages\n",
      "INFO: Preprocessed 12200000 pages\n",
      "INFO: Preprocessed 12300000 pages\n",
      "INFO: Preprocessed 12400000 pages\n",
      "INFO: Preprocessed 12500000 pages\n",
      "INFO: Preprocessed 12600000 pages\n",
      "INFO: Preprocessed 12700000 pages\n",
      "INFO: Preprocessed 12800000 pages\n",
      "INFO: Preprocessed 12900000 pages\n",
      "INFO: Preprocessed 13000000 pages\n",
      "INFO: Preprocessed 13100000 pages\n",
      "INFO: Preprocessed 13200000 pages\n",
      "INFO: Preprocessed 13300000 pages\n",
      "INFO: Preprocessed 13400000 pages\n",
      "INFO: Preprocessed 13500000 pages\n",
      "INFO: Preprocessed 13600000 pages\n",
      "INFO: Preprocessed 13700000 pages\n",
      "INFO: Preprocessed 13800000 pages\n",
      "INFO: Preprocessed 13900000 pages\n",
      "INFO: Preprocessed 14000000 pages\n",
      "INFO: Preprocessed 14100000 pages\n",
      "INFO: Preprocessed 14200000 pages\n",
      "INFO: Preprocessed 14300000 pages\n",
      "INFO: Preprocessed 14400000 pages\n",
      "INFO: Preprocessed 14500000 pages\n",
      "INFO: Preprocessed 14600000 pages\n",
      "INFO: Preprocessed 14700000 pages\n",
      "INFO: Preprocessed 14800000 pages\n",
      "INFO: Preprocessed 14900000 pages\n",
      "INFO: Preprocessed 15000000 pages\n",
      "INFO: Preprocessed 15100000 pages\n",
      "INFO: Preprocessed 15200000 pages\n",
      "INFO: Preprocessed 15300000 pages\n",
      "INFO: Preprocessed 15400000 pages\n",
      "INFO: Preprocessed 15500000 pages\n",
      "INFO: Preprocessed 15600000 pages\n",
      "INFO: Preprocessed 15700000 pages\n",
      "INFO: Preprocessed 15800000 pages\n",
      "INFO: Preprocessed 15900000 pages\n",
      "INFO: Preprocessed 16000000 pages\n",
      "INFO: Preprocessed 16100000 pages\n",
      "INFO: Preprocessed 16200000 pages\n",
      "INFO: Preprocessed 16300000 pages\n",
      "INFO: Preprocessed 16400000 pages\n",
      "INFO: Preprocessed 16500000 pages\n",
      "INFO: Preprocessed 16600000 pages\n",
      "INFO: Preprocessed 16700000 pages\n",
      "INFO: Preprocessed 16800000 pages\n",
      "INFO: Preprocessed 16900000 pages\n",
      "INFO: Preprocessed 17000000 pages\n",
      "INFO: Preprocessed 17100000 pages\n",
      "INFO: Preprocessed 17200000 pages\n",
      "INFO: Preprocessed 17300000 pages\n",
      "INFO: Preprocessed 17400000 pages\n",
      "INFO: Preprocessed 17500000 pages\n",
      "INFO: Preprocessed 17600000 pages\n",
      "INFO: Preprocessed 17700000 pages\n",
      "INFO: Preprocessed 17800000 pages\n",
      "INFO: Preprocessed 17900000 pages\n",
      "INFO: Preprocessed 18000000 pages\n",
      "INFO: Preprocessed 18100000 pages\n",
      "INFO: Preprocessed 18200000 pages\n",
      "INFO: Preprocessed 18300000 pages\n",
      "INFO: Preprocessed 18400000 pages\n",
      "INFO: Preprocessed 18500000 pages\n",
      "INFO: Preprocessed 18600000 pages\n",
      "INFO: Preprocessed 18700000 pages\n",
      "INFO: Preprocessed 18800000 pages\n",
      "INFO: Preprocessed 18900000 pages\n",
      "INFO: Preprocessed 19000000 pages\n",
      "INFO: Preprocessed 19100000 pages\n",
      "INFO: Preprocessed 19200000 pages\n",
      "INFO: Preprocessed 19300000 pages\n",
      "INFO: Preprocessed 19400000 pages\n",
      "INFO: Preprocessed 19500000 pages\n",
      "INFO: Preprocessed 19600000 pages\n",
      "INFO: Preprocessed 19700000 pages\n",
      "INFO: Preprocessed 19800000 pages\n",
      "INFO: Preprocessed 19900000 pages\n",
      "INFO: Preprocessed 20000000 pages\n",
      "INFO: Preprocessed 20100000 pages\n",
      "INFO: Preprocessed 20200000 pages\n",
      "INFO: Preprocessed 20300000 pages\n",
      "INFO: Preprocessed 20400000 pages\n",
      "INFO: Preprocessed 20500000 pages\n",
      "INFO: Preprocessed 20600000 pages\n",
      "INFO: Preprocessed 20700000 pages\n",
      "INFO: Preprocessed 20800000 pages\n",
      "INFO: Preprocessed 20900000 pages\n",
      "INFO: Preprocessed 21000000 pages\n",
      "INFO: Preprocessed 21100000 pages\n",
      "INFO: Preprocessed 21200000 pages\n",
      "INFO: Preprocessed 21300000 pages\n",
      "INFO: Preprocessed 21400000 pages\n",
      "INFO: Preprocessed 21500000 pages\n",
      "INFO: Preprocessed 21600000 pages\n",
      "INFO: Preprocessed 21700000 pages\n",
      "INFO: Preprocessed 21800000 pages\n",
      "INFO: Preprocessed 21900000 pages\n",
      "INFO: Preprocessed 22000000 pages\n",
      "INFO: Preprocessed 22100000 pages\n",
      "INFO: Preprocessed 22200000 pages\n",
      "INFO: Preprocessed 22300000 pages\n",
      "INFO: Preprocessed 22400000 pages\n",
      "INFO: Preprocessed 22500000 pages\n",
      "INFO: Preprocessed 22600000 pages\n",
      "INFO: Loaded 753131 templates in 4263.8s\n",
      "INFO: Starting page extraction from data/enwiki-20230220-pages-articles-multistream.xml.bz2.\n",
      "INFO: Using 7 extract processes.\n",
      "INFO: Extracted 100000 articles (722.8 art/s)\n",
      "INFO: Extracted 200000 articles (877.4 art/s)\n",
      "INFO: Extracted 300000 articles (1018.7 art/s)\n",
      "INFO: Extracted 400000 articles (1141.5 art/s)\n",
      "INFO: Extracted 500000 articles (1359.0 art/s)\n",
      "INFO: Extracted 600000 articles (1248.8 art/s)\n",
      "INFO: Extracted 700000 articles (1325.3 art/s)\n",
      "INFO: Extracted 800000 articles (1362.2 art/s)\n",
      "INFO: Extracted 900000 articles (1413.3 art/s)\n",
      "INFO: Extracted 1000000 articles (1452.2 art/s)\n",
      "INFO: Extracted 1100000 articles (1465.2 art/s)\n",
      "INFO: Extracted 1200000 articles (1510.0 art/s)\n",
      "INFO: Extracted 1300000 articles (1477.8 art/s)\n",
      "INFO: Extracted 1400000 articles (1510.4 art/s)\n",
      "INFO: Extracted 1500000 articles (1543.5 art/s)\n",
      "INFO: Extracted 1600000 articles (1592.6 art/s)\n",
      "INFO: Extracted 1700000 articles (1593.7 art/s)\n",
      "INFO: Extracted 1800000 articles (1586.6 art/s)\n",
      "INFO: Extracted 1900000 articles (1634.0 art/s)\n",
      "INFO: Extracted 2000000 articles (1663.7 art/s)\n",
      "INFO: Extracted 2100000 articles (1684.6 art/s)\n",
      "INFO: Extracted 2200000 articles (1666.8 art/s)\n",
      "INFO: Extracted 2300000 articles (1595.6 art/s)\n",
      "INFO: Extracted 2400000 articles (1655.8 art/s)\n",
      "INFO: Extracted 2500000 articles (1693.8 art/s)\n",
      "INFO: Extracted 2600000 articles (1727.2 art/s)\n",
      "INFO: Extracted 2700000 articles (1750.0 art/s)\n",
      "INFO: Extracted 2800000 articles (1699.7 art/s)\n",
      "INFO: Extracted 2900000 articles (1657.4 art/s)\n",
      "INFO: Extracted 3000000 articles (1758.1 art/s)\n",
      "INFO: Extracted 3100000 articles (1730.9 art/s)\n",
      "INFO: Extracted 3200000 articles (1740.9 art/s)\n",
      "INFO: Extracted 3300000 articles (1722.7 art/s)\n",
      "INFO: Extracted 3400000 articles (1787.3 art/s)\n",
      "INFO: Extracted 3500000 articles (1854.2 art/s)\n",
      "INFO: Extracted 3600000 articles (1737.4 art/s)\n",
      "INFO: Extracted 3700000 articles (1876.3 art/s)\n",
      "INFO: Extracted 3800000 articles (1736.8 art/s)\n",
      "INFO: Extracted 3900000 articles (1844.7 art/s)\n",
      "INFO: Extracted 4000000 articles (1845.0 art/s)\n",
      "INFO: Extracted 4100000 articles (1918.1 art/s)\n",
      "INFO: Extracted 4200000 articles (1890.3 art/s)\n",
      "INFO: Extracted 4300000 articles (1926.9 art/s)\n",
      "INFO: Extracted 4400000 articles (1916.6 art/s)\n",
      "INFO: Extracted 4500000 articles (1929.7 art/s)\n",
      "INFO: Extracted 4600000 articles (2163.9 art/s)\n",
      "INFO: Extracted 4700000 articles (2116.1 art/s)\n",
      "INFO: Extracted 4800000 articles (1873.7 art/s)\n",
      "INFO: Extracted 4900000 articles (1876.6 art/s)\n",
      "INFO: Extracted 5000000 articles (1927.3 art/s)\n",
      "INFO: Extracted 5100000 articles (1895.3 art/s)\n",
      "INFO: Extracted 5200000 articles (1878.5 art/s)\n",
      "INFO: Extracted 5300000 articles (1881.8 art/s)\n",
      "INFO: Extracted 5400000 articles (1942.6 art/s)\n",
      "INFO: Extracted 5500000 articles (1883.3 art/s)\n",
      "INFO: Extracted 5600000 articles (1796.0 art/s)\n",
      "INFO: Extracted 5700000 articles (1825.2 art/s)\n",
      "INFO: Extracted 5800000 articles (1892.5 art/s)\n",
      "INFO: Extracted 5900000 articles (1947.0 art/s)\n",
      "INFO: Extracted 6000000 articles (1899.2 art/s)\n",
      "INFO: Extracted 6100000 articles (1932.4 art/s)\n",
      "INFO: Extracted 6200000 articles (2020.1 art/s)\n",
      "INFO: Extracted 6300000 articles (1903.9 art/s)\n",
      "INFO: Extracted 6400000 articles (1823.1 art/s)\n",
      "INFO: Extracted 6500000 articles (1855.3 art/s)\n",
      "INFO: Extracted 6600000 articles (1848.5 art/s)\n",
      "INFO: Extracted 6700000 articles (1902.0 art/s)\n",
      "INFO: Extracted 6800000 articles (1832.9 art/s)\n",
      "INFO: Extracted 6900000 articles (1826.1 art/s)\n",
      "INFO: Extracted 7000000 articles (1948.7 art/s)\n",
      "INFO: Extracted 7100000 articles (1963.7 art/s)\n",
      "INFO: Extracted 7200000 articles (1938.6 art/s)\n",
      "INFO: Extracted 7300000 articles (1890.4 art/s)\n",
      "INFO: Extracted 7400000 articles (2279.8 art/s)\n",
      "INFO: Extracted 7500000 articles (1941.3 art/s)\n",
      "INFO: Extracted 7600000 articles (2014.7 art/s)\n",
      "INFO: Extracted 7700000 articles (2231.2 art/s)\n",
      "INFO: Extracted 7800000 articles (2034.7 art/s)\n",
      "INFO: Extracted 7900000 articles (1804.0 art/s)\n",
      "INFO: Extracted 8000000 articles (1884.5 art/s)\n",
      "INFO: Extracted 8100000 articles (1955.5 art/s)\n",
      "INFO: Extracted 8200000 articles (1940.3 art/s)\n",
      "INFO: Extracted 8300000 articles (1942.6 art/s)\n",
      "INFO: Extracted 8400000 articles (1927.2 art/s)\n",
      "INFO: Extracted 8500000 articles (1895.8 art/s)\n",
      "INFO: Extracted 8600000 articles (1875.0 art/s)\n",
      "INFO: Extracted 8700000 articles (1971.6 art/s)\n",
      "INFO: Extracted 8800000 articles (2120.3 art/s)\n",
      "INFO: Extracted 8900000 articles (1962.8 art/s)\n",
      "INFO: Extracted 9000000 articles (1706.5 art/s)\n",
      "INFO: Extracted 9100000 articles (1968.8 art/s)\n",
      "INFO: Extracted 9200000 articles (1988.4 art/s)\n",
      "INFO: Extracted 9300000 articles (1985.1 art/s)\n",
      "INFO: Extracted 9400000 articles (2026.0 art/s)\n",
      "INFO: Extracted 9500000 articles (1986.3 art/s)\n",
      "INFO: Extracted 9600000 articles (1988.2 art/s)\n",
      "INFO: Extracted 9700000 articles (1968.2 art/s)\n",
      "INFO: Extracted 9800000 articles (1958.0 art/s)\n",
      "INFO: Extracted 9900000 articles (1914.7 art/s)\n",
      "INFO: Extracted 10000000 articles (1962.8 art/s)\n",
      "INFO: Extracted 10100000 articles (1989.0 art/s)\n",
      "INFO: Extracted 10200000 articles (1968.0 art/s)\n",
      "INFO: Extracted 10300000 articles (1942.2 art/s)\n",
      "INFO: Extracted 10400000 articles (1981.0 art/s)\n",
      "INFO: Extracted 10500000 articles (1947.0 art/s)\n",
      "INFO: Extracted 10600000 articles (1966.5 art/s)\n",
      "INFO: Extracted 10700000 articles (1959.0 art/s)\n",
      "INFO: Extracted 10800000 articles (1994.6 art/s)\n",
      "INFO: Extracted 10900000 articles (1940.0 art/s)\n",
      "INFO: Extracted 11000000 articles (1891.5 art/s)\n",
      "INFO: Extracted 11100000 articles (1899.5 art/s)\n",
      "INFO: Extracted 11200000 articles (1903.9 art/s)\n",
      "INFO: Extracted 11300000 articles (1965.3 art/s)\n",
      "INFO: Extracted 11400000 articles (1989.7 art/s)\n",
      "INFO: Extracted 11500000 articles (2012.2 art/s)\n",
      "INFO: Extracted 11600000 articles (1940.2 art/s)\n",
      "INFO: Extracted 11700000 articles (1963.6 art/s)\n",
      "INFO: Extracted 11800000 articles (2008.6 art/s)\n",
      "INFO: Extracted 11900000 articles (1895.0 art/s)\n",
      "INFO: Extracted 12000000 articles (2169.5 art/s)\n",
      "INFO: Extracted 12100000 articles (1909.4 art/s)\n",
      "INFO: Extracted 12200000 articles (1923.4 art/s)\n",
      "INFO: Extracted 12300000 articles (1893.9 art/s)\n",
      "INFO: Extracted 12400000 articles (1978.2 art/s)\n",
      "INFO: Extracted 12500000 articles (1733.6 art/s)\n",
      "INFO: Extracted 12600000 articles (1862.9 art/s)\n",
      "INFO: Extracted 12700000 articles (1821.1 art/s)\n",
      "INFO: Extracted 12800000 articles (1780.7 art/s)\n",
      "INFO: Extracted 12900000 articles (1815.8 art/s)\n",
      "INFO: Extracted 13000000 articles (1955.4 art/s)\n",
      "INFO: Extracted 13100000 articles (2069.2 art/s)\n",
      "INFO: Extracted 13200000 articles (2057.3 art/s)\n",
      "INFO: Extracted 13300000 articles (1967.6 art/s)\n",
      "INFO: Extracted 13400000 articles (1953.8 art/s)\n",
      "INFO: Extracted 13500000 articles (1947.0 art/s)\n",
      "INFO: Extracted 13600000 articles (1829.7 art/s)\n",
      "INFO: Extracted 13700000 articles (1924.9 art/s)\n",
      "INFO: Extracted 13800000 articles (1897.0 art/s)\n",
      "INFO: Extracted 13900000 articles (1870.2 art/s)\n",
      "INFO: Extracted 14000000 articles (1739.6 art/s)\n",
      "INFO: Extracted 14100000 articles (1809.6 art/s)\n",
      "INFO: Extracted 14200000 articles (1679.5 art/s)\n",
      "INFO: Extracted 14300000 articles (1724.6 art/s)\n",
      "INFO: Extracted 14400000 articles (1862.6 art/s)\n",
      "INFO: Extracted 14500000 articles (1740.2 art/s)\n",
      "INFO: Extracted 14600000 articles (1767.2 art/s)\n",
      "INFO: Extracted 14700000 articles (1707.2 art/s)\n",
      "INFO: Extracted 14800000 articles (1757.3 art/s)\n",
      "INFO: Extracted 14900000 articles (1747.0 art/s)\n",
      "INFO: Extracted 15000000 articles (1719.7 art/s)\n",
      "INFO: Extracted 15100000 articles (1685.4 art/s)\n",
      "INFO: Extracted 15200000 articles (1818.0 art/s)\n",
      "INFO: Extracted 15300000 articles (1782.9 art/s)\n",
      "INFO: Extracted 15400000 articles (1845.1 art/s)\n",
      "INFO: Extracted 15500000 articles (1842.3 art/s)\n",
      "INFO: Extracted 15600000 articles (1849.5 art/s)\n",
      "INFO: Extracted 15700000 articles (1805.5 art/s)\n",
      "INFO: Extracted 15800000 articles (1675.9 art/s)\n",
      "INFO: Extracted 15900000 articles (1750.3 art/s)\n",
      "INFO: Extracted 16000000 articles (1792.1 art/s)\n",
      "INFO: Extracted 16100000 articles (1860.6 art/s)\n",
      "INFO: Extracted 16200000 articles (1864.7 art/s)\n",
      "INFO: Extracted 16300000 articles (1830.5 art/s)\n",
      "INFO: Extracted 16400000 articles (1830.4 art/s)\n",
      "INFO: Extracted 16500000 articles (1927.4 art/s)\n",
      "INFO: Extracted 16600000 articles (1881.1 art/s)\n",
      "INFO: Extracted 16700000 articles (1918.9 art/s)\n",
      "INFO: Extracted 16800000 articles (1927.2 art/s)\n",
      "INFO: Extracted 16900000 articles (1942.1 art/s)\n",
      "INFO: Finished 7-process extraction of 16904183 articles in 9465.8s (1785.8 art/s)\n"
     ]
    }
   ],
   "source": [
    "!python -m wikiextractor.WikiExtractor data/enwiki-20230220-pages-articles-multistream.xml.bz2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import re"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['GA',\n",
       " 'AX',\n",
       " 'BW',\n",
       " 'DN',\n",
       " 'GF',\n",
       " 'DI',\n",
       " 'BP',\n",
       " 'BY',\n",
       " 'GO',\n",
       " 'AV',\n",
       " 'DG',\n",
       " 'AQ',\n",
       " 'GH',\n",
       " 'DU',\n",
       " 'BL',\n",
       " 'AC',\n",
       " 'BK',\n",
       " 'DR',\n",
       " 'AD',\n",
       " 'AM',\n",
       " 'BB',\n",
       " 'AJ',\n",
       " 'BE',\n",
       " 'DF',\n",
       " 'AP',\n",
       " 'GI',\n",
       " 'BX',\n",
       " 'DA',\n",
       " 'GN',\n",
       " 'AW',\n",
       " 'GG',\n",
       " 'DH',\n",
       " 'BQ',\n",
       " 'AY',\n",
       " 'BV',\n",
       " 'DO',\n",
       " 'AK',\n",
       " 'BD',\n",
       " 'AL',\n",
       " 'DZ',\n",
       " 'BC',\n",
       " 'BJ',\n",
       " 'DS',\n",
       " 'AE',\n",
       " 'DT',\n",
       " 'BM',\n",
       " 'AB',\n",
       " 'EY',\n",
       " 'FV',\n",
       " 'CG',\n",
       " 'FQ',\n",
       " 'FX',\n",
       " 'EW',\n",
       " 'CN',\n",
       " 'CI',\n",
       " 'EP',\n",
       " 'FM',\n",
       " 'EB',\n",
       " 'FJ',\n",
       " 'EE',\n",
       " 'CU',\n",
       " 'EL',\n",
       " 'FC',\n",
       " 'EK',\n",
       " 'CR',\n",
       " 'FD',\n",
       " 'CH',\n",
       " 'EQ',\n",
       " 'FY',\n",
       " 'EV',\n",
       " 'CO',\n",
       " 'CF',\n",
       " 'FP',\n",
       " 'EX',\n",
       " 'CA',\n",
       " 'FW',\n",
       " 'EJ',\n",
       " 'CS',\n",
       " 'FE',\n",
       " 'CT',\n",
       " 'EM',\n",
       " 'FB',\n",
       " 'FK',\n",
       " 'ED',\n",
       " 'FL',\n",
       " 'CZ',\n",
       " 'EC',\n",
       " 'BH',\n",
       " 'DQ',\n",
       " 'AG',\n",
       " 'DV',\n",
       " 'BO',\n",
       " 'AI',\n",
       " 'BF',\n",
       " 'AN',\n",
       " 'DX',\n",
       " 'BA',\n",
       " 'GE',\n",
       " 'DJ',\n",
       " 'BS',\n",
       " 'GB',\n",
       " 'BT',\n",
       " 'DM',\n",
       " 'DD',\n",
       " 'AR',\n",
       " 'GK',\n",
       " 'BZ',\n",
       " 'DC',\n",
       " 'GL',\n",
       " 'AU',\n",
       " 'AO',\n",
       " 'DY',\n",
       " 'AH',\n",
       " 'BG',\n",
       " 'DW',\n",
       " 'BN',\n",
       " 'AA',\n",
       " 'BI',\n",
       " 'DP',\n",
       " 'AF',\n",
       " 'DB',\n",
       " 'GM',\n",
       " 'AT',\n",
       " 'DE',\n",
       " 'AS',\n",
       " 'GJ',\n",
       " 'GC',\n",
       " 'AZ',\n",
       " 'BU',\n",
       " 'DL',\n",
       " 'GD',\n",
       " 'DK',\n",
       " 'BR',\n",
       " 'FI',\n",
       " 'EF',\n",
       " 'FN',\n",
       " 'CX',\n",
       " 'EA',\n",
       " 'EH',\n",
       " 'CQ',\n",
       " 'FG',\n",
       " 'CV',\n",
       " 'EO',\n",
       " 'CD',\n",
       " 'FR',\n",
       " 'EZ',\n",
       " 'CC',\n",
       " 'FU',\n",
       " 'CJ',\n",
       " 'ES',\n",
       " 'ET',\n",
       " 'CM',\n",
       " 'CW',\n",
       " 'EN',\n",
       " 'FA',\n",
       " 'EI',\n",
       " 'CP',\n",
       " 'FF',\n",
       " 'FO',\n",
       " 'CY',\n",
       " 'FH',\n",
       " 'EG',\n",
       " 'FZ',\n",
       " 'EU',\n",
       " 'CL',\n",
       " 'CK',\n",
       " 'ER',\n",
       " 'CB',\n",
       " 'FT',\n",
       " 'CE',\n",
       " 'FS']"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "os.listdir('text')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def list_wiki(dirname):\n",
    "    filepaths = []\n",
    "    filenames = os.listdir(dirname)\n",
    "    for filename in filenames:\n",
    "        filepath = os.path.join(dirname, filename)\n",
    "\n",
    "        if os.path.isdir(filepath):\n",
    "            # 재귀 함수\n",
    "            filepaths.extend(list_wiki(filepath))\n",
    "        else:\n",
    "            find = re.findall(r\"wiki_[0-9][0-9]\", filepath)\n",
    "            if 0 < len(find):\n",
    "                filepaths.append(filepath)\n",
    "    return sorted(filepaths)\n",
    "\n",
    "filepaths = list_wiki('text')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "17065"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(filepaths)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(\"data/wiki_output_file.txt\", \"w\") as outfile:\n",
    "    for filename in filepaths:\n",
    "        with open(filename) as infile:\n",
    "            contents = infile.read()\n",
    "            outfile.write(contents)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# W2V visualization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('data/euphemism_answer_drug.txt') as euph:\n",
    "    line = euph.readlines()\n",
    "\n",
    "    euph_dict = {}\n",
    "    for i in line:\n",
    "        key = i.strip().split(':')[0]\n",
    "        values = i.strip().split(':')[1]\n",
    "        value = values.strip().split('; ')\n",
    "        euph_dict[key.lower()] = [val.lower() for val in value]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "formal drug name: fentanyl <-> euphemistic word: fent\n",
      "formal drug name: lsd <-> euphemistic word: acid\n",
      "formal drug name: promethazine <-> euphemistic word: syrup\n",
      "\n",
      "correct words: 3 & top-10 precision: 0.0375\n"
     ]
    }
   ],
   "source": [
    "# k=10\n",
    "with open('data/output/w2v/wiki_eval_w2v_embed_top10.txt') as fn:\n",
    "    line = fn.readlines()\n",
    "    result_dict = {}\n",
    "    for i in line[1:]:\n",
    "        l = i.strip().split(', ')\n",
    "        key = l[0]\n",
    "        value = l[1:]\n",
    "        result_dict[key] = value\n",
    "\n",
    "denom = len(key) * len(value)\n",
    "count_correct = 0\n",
    "for key in result_dict:\n",
    "    if key in euph_dict:\n",
    "        for val in result_dict[key]:\n",
    "            if val in euph_dict[key]:\n",
    "                print(f\"formal drug name: {key} <-> euphemistic word: {val}\")\n",
    "                count_correct += 1\n",
    "\n",
    "print(f\"\\ncorrect words: {count_correct} & top-10 precision: {count_correct / denom}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "formal drug name: molly <-> euphemistic word: rolling\n",
      "formal drug name: fentanyl <-> euphemistic word: fent\n",
      "formal drug name: heroin <-> euphemistic word: dope\n",
      "formal drug name: ketamine <-> euphemistic word: k\n",
      "formal drug name: lsd <-> euphemistic word: acid\n",
      "formal drug name: promethazine <-> euphemistic word: syrup\n",
      "formal drug name: promethazine <-> euphemistic word: juice\n",
      "\n",
      "correct words: 7 & top-20 precision: 0.04375\n"
     ]
    }
   ],
   "source": [
    "# k=20\n",
    "with open('data/output/w2v/wiki_eval_w2v_embed_top20.txt') as fn:\n",
    "    line = fn.readlines()\n",
    "    result_dict = {}\n",
    "    for i in line[1:]:\n",
    "        l = i.strip().split(', ')\n",
    "        key = l[0]\n",
    "        value = l[1:]\n",
    "        result_dict[key] = value\n",
    "\n",
    "denom = len(key) * len(value)\n",
    "count_correct = 0\n",
    "for key in result_dict:\n",
    "    if key in euph_dict:\n",
    "        for val in result_dict[key]:\n",
    "            if val in euph_dict[key]:\n",
    "                print(f\"formal drug name: {key} <-> euphemistic word: {val}\")\n",
    "                count_correct += 1\n",
    "\n",
    "print(f\"\\ncorrect words: {count_correct} & top-20 precision: {count_correct / denom}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "formal drug name: molly <-> euphemistic word: rolling\n",
      "formal drug name: fentanyl <-> euphemistic word: fent\n",
      "formal drug name: heroin <-> euphemistic word: dope\n",
      "formal drug name: ketamine <-> euphemistic word: k\n",
      "formal drug name: lsd <-> euphemistic word: acid\n",
      "formal drug name: oxycodone <-> euphemistic word: oxy\n",
      "formal drug name: promethazine <-> euphemistic word: syrup\n",
      "formal drug name: promethazine <-> euphemistic word: juice\n",
      "\n",
      "correct words: 8 & top-30 precision: 0.03333333333333333\n"
     ]
    }
   ],
   "source": [
    "# k=30\n",
    "with open('data/output/w2v/wiki_eval_w2v_embed_top30.txt') as fn:\n",
    "    line = fn.readlines()\n",
    "    result_dict = {}\n",
    "    for i in line[1:]:\n",
    "        l = i.strip().split(', ')\n",
    "        key = l[0]\n",
    "        value = l[1:]\n",
    "        result_dict[key] = value\n",
    "\n",
    "denom = len(key) * len(value)\n",
    "count_correct = 0\n",
    "for key in result_dict:\n",
    "    if key in euph_dict:\n",
    "        for val in result_dict[key]:\n",
    "            if val in euph_dict[key]:\n",
    "                print(f\"formal drug name: {key} <-> euphemistic word: {val}\")\n",
    "                count_correct += 1\n",
    "\n",
    "print(f\"\\ncorrect words: {count_correct} & top-30 precision: {count_correct / denom}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "formal drug name: molly <-> euphemistic word: rolling\n",
      "formal drug name: fentanyl <-> euphemistic word: fent\n",
      "formal drug name: heroin <-> euphemistic word: dope\n",
      "formal drug name: ketamine <-> euphemistic word: k\n",
      "formal drug name: lsd <-> euphemistic word: acid\n",
      "formal drug name: methamphetamine <-> euphemistic word: crystal\n",
      "formal drug name: oxycodone <-> euphemistic word: oxy\n",
      "formal drug name: promethazine <-> euphemistic word: syrup\n",
      "formal drug name: promethazine <-> euphemistic word: juice\n",
      "\n",
      "correct words: 9 & top-50 precision: 0.0225\n"
     ]
    }
   ],
   "source": [
    "# k=50\n",
    "with open('data/output/w2v/wiki_eval_w2v_embed_top50.txt') as fn:\n",
    "    line = fn.readlines()\n",
    "    result_dict = {}\n",
    "    for i in line[1:]:\n",
    "        l = i.strip().split(', ')\n",
    "        key = l[0]\n",
    "        value = l[1:]\n",
    "        result_dict[key] = value\n",
    "        \n",
    "denom = len(key) * len(value)\n",
    "count_correct = 0\n",
    "for key in result_dict:\n",
    "    if key in euph_dict:\n",
    "        for val in result_dict[key]:\n",
    "            if val in euph_dict[key]:\n",
    "                print(f\"formal drug name: {key} <-> euphemistic word: {val}\")\n",
    "                count_correct += 1\n",
    "\n",
    "print(f\"\\ncorrect words: {count_correct} & top-50 precision: {count_correct / denom}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "fyp-dev",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.9"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}