scraping_faq.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "import scrapy\n",
    "  \n",
    "class ExtractUrls(scrapy.Spider):\n",
    "      \n",
    "    # This name must be unique always\n",
    "    name = \"extract\"                 \n",
    "  \n",
    "    # Function which will be invoked\n",
    "    def start_requests(self):\n",
    "          \n",
    "        # enter the URL here\n",
    "        urls = [\"https://accessability.utdallas.edu/student-accommodations/frequently-asked-questions/\"]\n",
    "          \n",
    "        for url in urls:\n",
    "            yield scrapy.Request(url = url, callback = self.parse)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "def parse(self, response):\n",
    "    \n",
    "    ## Each block of text\n",
    "    \n",
    "    extract_urls = response.css(\"div.wp-block-group__inner-container\")\n",
    "    \n",
    "    total_heading = []\n",
    "    total_sub_heading = []\n",
    "    total_links = []\n",
    "    total_names_of_links = []\n",
    "    \n",
    "    \n",
    "    for extract_url in extract_urls:\n",
    "        \n",
    "        question = h3_text = extract_url.css(\"h3::text\").getall()\n",
    "        \n",
    "        sub_heading = extract_url.css(\"h4::text\").getall()\n",
    "        sub_heading = [i for i in sub_heading if i != '']\n",
    "        \n",
    "        links = extract_url.css(\"a::attr(href)\").getall()\n",
    "        names_of_links = extract_url.css(\"a::text\").getall()\n",
    "        \n",
    "        ## add to listes\n",
    "        total_heading = total_heading + heading\n",
    "        total_sub_heading = total_sub_heading + sub_heading\n",
    "        total_links = total_links + links\n",
    "        total_names_of_links = total_names_of_links + names_of_links\n",
    "\n",
    "        return{ \"heading\": total_heading, \"sub_heading\": total_sub_heading, \"links\": total_links, \"names_of_links\": total_names_of_links}\n",
    "\n",
    "ExtractUrls.parse = parse"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Use the below command to run the spide\n",
    "extract_urls = ExtractUrls()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "ename": "AttributeError",
     "evalue": "'Response' object has no attribute 'css'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
      "\u001b[0;32m/tmp/ipykernel_43732/1121250336.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mheadings\u001b[0m \u001b[0;34m,\u001b[0m \u001b[0msub_headings\u001b[0m \u001b[0;34m,\u001b[0m \u001b[0mlinks\u001b[0m \u001b[0;34m,\u001b[0m \u001b[0mnames_of_links\u001b[0m \u001b[0;34m,\u001b[0m \u001b[0murl\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mextract_urls\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mparse\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m/tmp/ipykernel_43732/1815887443.py\u001b[0m in \u001b[0;36mparse\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m      7\u001b[0m     \u001b[0;31m## Each block of text\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      8\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 9\u001b[0;31m     \u001b[0mextract_urls\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mresponse\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcss\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"div.wp-block-group__inner-container\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     10\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     11\u001b[0m     \u001b[0mtotal_heading\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mAttributeError\u001b[0m: 'Response' object has no attribute 'css'"
     ]
    }
   ],
   "source": [
    "headings , sub_headings , links , names_of_links , url = extract_urls.parse()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "base::attr(href)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.7.16 ('utd_chatbot')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.16"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "c9ff9b4d34df2539584901dcbbee8740b4a40a3636c7400fd1d2b55da32052e7"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}