04-monitoring/offline-rag-evaluation.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "82e471cb-7aec-4c00-9eed-36e6f2447eb5",
   "metadata": {},
   "source": [
    "## Load documents with IDs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "76241162-06cf-4d61-959b-1d36a8678bf3",
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests \n",
    "\n",
    "base_url = 'https://github.com/DataTalksClub/llm-zoomcamp/blob/main'\n",
    "relative_url = '03-vector-search/eval/documents-with-ids.json'\n",
    "docs_url = f'{base_url}/{relative_url}?raw=1'\n",
    "docs_response = requests.get(docs_url)\n",
    "documents = docs_response.json()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "f745e802-1cb5-4e77-afa0-a4e276270a62",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'text': 'It depends on your background and previous experience with modules. It is expected to require about 5 - 15 hours per week. [source1] [source2]\\nYou can also calculate it yourself using this data and then update this answer.',\n",
       " 'section': 'General course-related questions',\n",
       " 'question': 'Course - \\u200b\\u200bHow many hours per week am I expected to spend on this  course?',\n",
       " 'course': 'data-engineering-zoomcamp',\n",
       " 'id': 'ea739c65'}"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "documents[10]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2dcd8e36-32c1-435a-be50-56618ee7a605",
   "metadata": {},
   "source": [
    "## Load ground truth"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "79da91e5-bb37-4a7a-860b-0d3909f5de88",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "\n",
    "base_url = 'https://github.com/DataTalksClub/llm-zoomcamp/blob/main'\n",
    "relative_url = '03-vector-search/eval/ground-truth-data.csv'\n",
    "ground_truth_url = f'{base_url}/{relative_url}?raw=1'\n",
    "\n",
    "df_ground_truth = pd.read_csv(ground_truth_url)\n",
    "df_ground_truth = df_ground_truth[df_ground_truth.course == 'machine-learning-zoomcamp']\n",
    "ground_truth = df_ground_truth.to_dict(orient='records')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "4b830c7b-03a9-44be-93cd-3a0e7da12eed",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'question': 'Are sessions recorded if I miss one?',\n",
       " 'course': 'machine-learning-zoomcamp',\n",
       " 'document': '5170565b'}"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ground_truth[10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "id": "7cfd91ec-8ec8-4570-bff7-228c5eb7327b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Everything is recorded, so you won’t miss anything. You will be able to ask your questions for office hours in advance and we will cover them during the live stream. Also, you can always ask questions in Slack.'"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "doc_idx = {d['id']: d for d in documents}\n",
    "doc_idx['5170565b']['text']"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9042aa10-abed-4009-b7af-769d8b477cd0",
   "metadata": {},
   "source": [
    "## Index data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "3f61b1fe-f67a-4217-9fbe-5fd57eefd53d",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sentence_transformers import SentenceTransformer\n",
    "\n",
    "model_name = 'multi-qa-MiniLM-L6-cos-v1'\n",
    "model = SentenceTransformer(model_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "79c49a41-7693-4bf5-a086-9ad6aaaf0284",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "ObjectApiResponse({'acknowledged': True, 'shards_acknowledged': True, 'index': 'course-questions'})"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from elasticsearch import Elasticsearch\n",
    "\n",
    "es_client = Elasticsearch('http://localhost:9200') \n",
    "\n",
    "index_settings = {\n",
    "    \"settings\": {\n",
    "        \"number_of_shards\": 1,\n",
    "        \"number_of_replicas\": 0\n",
    "    },\n",
    "    \"mappings\": {\n",
    "        \"properties\": {\n",
    "            \"text\": {\"type\": \"text\"},\n",
    "            \"section\": {\"type\": \"text\"},\n",
    "            \"question\": {\"type\": \"text\"},\n",
    "            \"course\": {\"type\": \"keyword\"},\n",
    "            \"id\": {\"type\": \"keyword\"},\n",
    "            \"question_text_vector\": {\n",
    "                \"type\": \"dense_vector\",\n",
    "                \"dims\": 384,\n",
    "                \"index\": True,\n",
    "                \"similarity\": \"cosine\"\n",
    "            },\n",
    "        }\n",
    "    }\n",
    "}\n",
    "\n",
    "index_name = \"course-questions\"\n",
    "\n",
    "es_client.indices.delete(index=index_name, ignore_unavailable=True)\n",
    "es_client.indices.create(index=index_name, body=index_settings)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "8bd33453-4885-4c59-a566-3f5f405e2622",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "026d2068f93e4a8d978b33e835be5bc9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/948 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from tqdm.auto import tqdm\n",
    "\n",
    "for doc in tqdm(documents):\n",
    "    question = doc['question']\n",
    "    text = doc['text']\n",
    "    doc['question_text_vector'] = model.encode(question + ' ' + text)\n",
    "\n",
    "    es_client.index(index=index_name, document=doc)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "666609c9-93d0-4167-8287-392572eaa762",
   "metadata": {},
   "source": [
    "## Retrieval"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "7de211a5-70c5-463a-8a63-d4e63fdad821",
   "metadata": {},
   "outputs": [],
   "source": [
    "def elastic_search_knn(field, vector, course):\n",
    "    knn = {\n",
    "        \"field\": field,\n",
    "        \"query_vector\": vector,\n",
    "        \"k\": 5,\n",
    "        \"num_candidates\": 10000,\n",
    "        \"filter\": {\n",
    "            \"term\": {\n",
    "                \"course\": course\n",
    "            }\n",
    "        }\n",
    "    }\n",
    "\n",
    "    search_query = {\n",
    "        \"knn\": knn,\n",
    "        \"_source\": [\"text\", \"section\", \"question\", \"course\", \"id\"]\n",
    "    }\n",
    "\n",
    "    es_results = es_client.search(\n",
    "        index=index_name,\n",
    "        body=search_query\n",
    "    )\n",
    "    \n",
    "    result_docs = []\n",
    "    \n",
    "    for hit in es_results['hits']['hits']:\n",
    "        result_docs.append(hit['_source'])\n",
    "\n",
    "    return result_docs\n",
    "\n",
    "def question_text_vector_knn(q):\n",
    "    question = q['question']\n",
    "    course = q['course']\n",
    "\n",
    "    v_q = model.encode(question)\n",
    "\n",
    "    return elastic_search_knn('question_text_vector', v_q, course)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "96a7f632-a928-4236-abff-ff2a73d6f463",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'question': 'What if I miss a session?',\n",
       "  'course': 'machine-learning-zoomcamp',\n",
       "  'section': 'General course-related questions',\n",
       "  'text': 'Everything is recorded, so you won’t miss anything. You will be able to ask your questions for office hours in advance and we will cover them during the live stream. Also, you can always ask questions in Slack.',\n",
       "  'id': '5170565b'},\n",
       " {'question': 'Is it going to be live? When?',\n",
       "  'course': 'machine-learning-zoomcamp',\n",
       "  'section': 'General course-related questions',\n",
       "  'text': 'The course videos are pre-recorded, you can start watching the course right now.\\nWe will also occasionally have office hours - live sessions where we will answer your questions. The office hours sessions are recorded too.\\nYou can see the office hours as well as the pre-recorded course videos in the course playlist on YouTube.',\n",
       "  'id': '39fda9f0'},\n",
       " {'question': 'The same accuracy on epochs',\n",
       "  'course': 'machine-learning-zoomcamp',\n",
       "  'section': '8. Neural Networks and Deep Learning',\n",
       "  'text': \"Problem description\\nThe accuracy and the loss are both still the same or nearly the same while training.\\nSolution description\\nIn the homework, you should set class_mode='binary' while reading the data.\\nAlso, problem occurs when you choose the wrong optimizer, batch size, or learning rate\\nAdded by Ekaterina Kutovaia\",\n",
       "  'id': '7d11d5ce'},\n",
       " {'question': 'Useful Resource for Missing Data Treatment\\nhttps://www.kaggle.com/code/parulpandey/a-guide-to-handling-missing-values-in-python/notebook',\n",
       "  'course': 'machine-learning-zoomcamp',\n",
       "  'section': '2. Machine Learning for Regression',\n",
       "  'text': '(Hrithik Kumar Advani)',\n",
       "  'id': '81b8e8d0'},\n",
       " {'question': 'Will I get a certificate if I missed the midterm project?',\n",
       "  'course': 'machine-learning-zoomcamp',\n",
       "  'section': 'General course-related questions',\n",
       "  'text': \"Yes, it's possible. See the previous answer.\",\n",
       "  'id': '1d644223'}]"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "question_text_vector_knn(dict(\n",
    "    question='Are sessions recorded if I miss one?',\n",
    "    course='machine-learning-zoomcamp'\n",
    "))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "443cb32b-5018-477a-b215-647a1ee0eb9c",
   "metadata": {},
   "source": [
    "## The RAG flow"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "9a155184-278e-4ffe-b4fd-799747a6bc00",
   "metadata": {},
   "outputs": [],
   "source": [
    "def build_prompt(query, search_results):\n",
    "    prompt_template = \"\"\"\n",
    "You're a course teaching assistant. Answer the QUESTION based on the CONTEXT from the FAQ database.\n",
    "Use only the facts from the CONTEXT when answering the QUESTION.\n",
    "\n",
    "QUESTION: {question}\n",
    "\n",
    "CONTEXT: \n",
    "{context}\n",
    "\"\"\".strip()\n",
    "\n",
    "    context = \"\"\n",
    "    \n",
    "    for doc in search_results:\n",
    "        context = context + f\"section: {doc['section']}\\nquestion: {doc['question']}\\nanswer: {doc['text']}\\n\\n\"\n",
    "    \n",
    "    prompt = prompt_template.format(question=query, context=context).strip()\n",
    "    return prompt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "id": "acbc99bb-1a84-42de-82f6-b68b563b7154",
   "metadata": {},
   "outputs": [],
   "source": [
    "from openai import OpenAI\n",
    "\n",
    "client = OpenAI()\n",
    "\n",
    "def llm(prompt, model='gpt-4o'):\n",
    "    response = client.chat.completions.create(\n",
    "        model=model,\n",
    "        messages=[{\"role\": \"user\", \"content\": prompt}]\n",
    "    )\n",
    "    \n",
    "    return response.choices[0].message.content"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "id": "7c025101-f138-44fb-a7fa-2159a426422a",
   "metadata": {},
   "outputs": [],
   "source": [
    "# previously: rag(query: str) -> str\n",
    "def rag(query: dict, model='gpt-4o') -> str:\n",
    "    search_results = question_text_vector_knn(query)\n",
    "    prompt = build_prompt(query['question'], search_results)\n",
    "    answer = llm(prompt, model=model)\n",
    "    return answer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "3e57cdeb-f5e2-4481-8e86-49d70d6adb74",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'question': 'Are sessions recorded if I miss one?',\n",
       " 'course': 'machine-learning-zoomcamp',\n",
       " 'document': '5170565b'}"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ground_truth[10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "e5a6ad63-5b55-49c2-a115-cdccc4874720",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Yes, sessions are recorded if you miss one. Everything is recorded, allowing you to catch up on any missed content. Additionally, you can ask questions in advance for office hours and have them addressed during the live stream. You can also ask questions in Slack.'"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rag(ground_truth[10])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "75123a0b-1aea-467c-84d9-ae5f6f6c2256",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Everything is recorded, so you won’t miss anything. You will be able to ask your questions for office hours in advance and we will cover them during the live stream. Also, you can always ask questions in Slack.'"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "doc_idx['5170565b']['text']"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3c26035e-ea80-4817-93a9-46a6af4eaf91",
   "metadata": {},
   "source": [
    "## Cosine similarity metric"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "id": "754db44d-5e73-4784-9369-707691fb9229",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.759117"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "answer_orig = 'Yes, sessions are recorded if you miss one. Everything is recorded, allowing you to catch up on any missed content. Additionally, you can ask questions in advance for office hours and have them addressed during the live stream. You can also ask questions in Slack.'\n",
    "answer_llm = 'Everything is recorded, so you won’t miss anything. You will be able to ask your questions for office hours in advance and we will cover them during the live stream. Also, you can always ask questions in Slack.'\n",
    "\n",
    "v_llm = model.encode(answer_llm)\n",
    "v_orig = model.encode(answer_orig)\n",
    "\n",
    "v_llm.dot(v_orig)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "id": "3ffcee32-a7c0-45ea-aca2-845758ed3ee2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'question': 'Where can I sign up for the course?',\n",
       " 'course': 'machine-learning-zoomcamp',\n",
       " 'document': '0227b872'}"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ground_truth[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "id": "61f7deed-9730-467d-afc7-f053b23fe8e6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1830"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(ground_truth)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "id": "2b4b9573-f1fd-4617-9904-0c9e6999eef7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'question': 'Where can I sign up for the course?',\n",
       " 'course': 'machine-learning-zoomcamp',\n",
       " 'document': '0227b872'}"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rec"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "db605e72-f5cf-4402-ab78-4fd831a1c6de",
   "metadata": {},
   "outputs": [],
   "source": [
    "answers = {}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "id": "9e470ca3-6c27-4a62-a1d9-6ea9fc3e84f5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d1efc298fef64d548d940bd31ef5b80c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/1830 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "for i, rec in enumerate(tqdm(ground_truth)):\n",
    "    if i in answers:\n",
    "        continue\n",
    "\n",
    "    answer_llm = rag(rec)\n",
    "    doc_id = rec['document']\n",
    "    original_doc = doc_idx[doc_id]\n",
    "    answer_orig = original_doc['text']\n",
    "\n",
    "    answers[i] = {\n",
    "        'answer_llm': answer_llm,\n",
    "        'answer_orig': answer_orig,\n",
    "        'document': doc_id,\n",
    "        'question': rec['question'],\n",
    "        'course': rec['course'],\n",
    "    }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "id": "2b63fb6d-f125-42b4-a634-52c3c17e582d",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "results_gpt4o = [None] * len(ground_truth)\n",
    "\n",
    "for i, val in answers.items():\n",
    "    results_gpt4o[i] = val.copy()\n",
    "    results_gpt4o[i].update(ground_truth[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "id": "3d68aec3-c05f-447d-9795-eb6c82c439ae",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "id": "141bf0e3-b25d-474f-a633-53d4189daf77",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_gpt4o = pd.DataFrame(results_gpt4o)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "id": "3804e23f-e75d-41c1-9ab9-b2691c3c3a8e",
   "metadata": {},
   "outputs": [],
   "source": [
    "!mkdir data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "id": "7d923470-0433-4b3d-890c-813161550efb",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_gpt4o.to_csv('data/results-gpt4o.csv', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "55f32fc1-96f2-45c8-b22f-45c72c0786e0",
   "metadata": {},
   "source": [
    "## Evaluating GPT 3.5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "id": "e6678220-4686-41d2-9d6b-878498728cce",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"No, sessions are recorded so if you miss one, you won't miss anything.\""
      ]
     },
     "execution_count": 119,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rag(ground_truth[10], model='gpt-3.5-turbo')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "id": "bfdc6e3c-a9d4-4ae8-8379-c9c90efc23b6",
   "metadata": {},
   "outputs": [],
   "source": [
    "from tqdm.auto import tqdm\n",
    "\n",
    "from concurrent.futures import ThreadPoolExecutor\n",
    "\n",
    "pool = ThreadPoolExecutor(max_workers=6)\n",
    "\n",
    "def map_progress(pool, seq, f):\n",
    "    results = []\n",
    "\n",
    "    with tqdm(total=len(seq)) as progress:\n",
    "        futures = []\n",
    "\n",
    "        for el in seq:\n",
    "            future = pool.submit(f, el)\n",
    "            future.add_done_callback(lambda p: progress.update())\n",
    "            futures.append(future)\n",
    "\n",
    "        for future in futures:\n",
    "            result = future.result()\n",
    "            results.append(result)\n",
    "\n",
    "    return results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "id": "9c725d15-c355-4978-b598-81526fe298ec",
   "metadata": {},
   "outputs": [],
   "source": [
    "def process_record(rec):\n",
    "    model = 'gpt-3.5-turbo'\n",
    "    answer_llm = rag(rec, model=model)\n",
    "    \n",
    "    doc_id = rec['document']\n",
    "    original_doc = doc_idx[doc_id]\n",
    "    answer_orig = original_doc['text']\n",
    "\n",
    "    return {\n",
    "        'answer_llm': answer_llm,\n",
    "        'answer_orig': answer_orig,\n",
    "        'document': doc_id,\n",
    "        'question': rec['question'],\n",
    "        'course': rec['course'],\n",
    "    }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "id": "dfd12868-c71e-45da-9557-d22f720c04a5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'answer_llm': 'Yes, sessions are recorded if you miss one. Everything is recorded, so you won’t miss anything, and you can also ask questions for office hours in advance which will be covered during the live stream. You can always ask questions in Slack as well.',\n",
       " 'answer_orig': 'Everything is recorded, so you won’t miss anything. You will be able to ask your questions for office hours in advance and we will cover them during the live stream. Also, you can always ask questions in Slack.',\n",
       " 'document': '5170565b',\n",
       " 'question': 'Are sessions recorded if I miss one?',\n",
       " 'course': 'machine-learning-zoomcamp'}"
      ]
     },
     "execution_count": 122,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "process_record(ground_truth[10])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "id": "0a0f73f6-7542-4351-9839-ba3129d0da44",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2b793df233b741e59fe0522827af5c5e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/1830 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "results_gpt35 = map_progress(pool, ground_truth, process_record)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "id": "79eaa48a-7b55-4ff3-a4d5-16ce0e4fedd0",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_gpt35 = pd.DataFrame(results_gpt35)\n",
    "df_gpt35.to_csv('data/results-gpt35.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "id": "b8a88bd2-7553-4222-8efc-530860f0c31a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "answer_llm,answer_orig,document,question,course\n",
      "You can sign up for the course by going to the course page at http://mlzoomcamp.com/ and scrolling down to access the course materials.,\"Machine Learning Zoomcamp FAQ\n",
      "The purpose of this document is to capture frequently asked technical questions.\n",
      "We did this for our data engineering course and it worked quite well. Check this document for inspiration on how to structure your questions and answers:\n",
      "Data Engineering Zoomcamp FAQ\n",
      "In the course GitHub repository thereâ€™s a link. Here it is: https://airtable.com/shryxwLd0COOEaqXo\n",
      "work\",0227b872,Where can I sign up for the course?,machine-learning-zoomcamp\n",
      "\"I am sorry, but there is no direct link provided in the FAQ database for signing up for the course. However, you can find a link in the course GitHub repository at this address: https://airtable.com/shryxwLd0COOEaqXo.\",\"Machine Learning Zoomcamp FAQ\n",
      "The purpose of this document is to capture frequently asked technical questions.\n",
      "We did this for our data engineering course and it worked quite well. Check this document for inspiration on how to structure your questions and answers:\n"
     ]
    }
   ],
   "source": [
    "!head data/results-gpt35.csv"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "67827816-f598-41c9-9bee-b312e117a32a",
   "metadata": {},
   "source": [
    "## Cosine similarity\n",
    "\n",
    "A->Q->A' cosine similarity\n",
    "\n",
    "A -> Q -> A'\n",
    "\n",
    "cosine(A, A')\n",
    "\n",
    "### gpt-4o"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "id": "2af96a0d-627a-4f32-9965-c90ba542d65c",
   "metadata": {},
   "outputs": [],
   "source": [
    "results_gpt4o = df_gpt4o.to_dict(orient='records')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "id": "e8bfb5a3-06da-4459-bbc4-eed5ed1a84ce",
   "metadata": {},
   "outputs": [],
   "source": [
    "record = results_gpt4o[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "id": "453b5a1d-a0cc-4023-aa74-de23a65ce71c",
   "metadata": {},
   "outputs": [],
   "source": [
    "def compute_similarity(record):\n",
    "    answer_orig = record['answer_orig']\n",
    "    answer_llm = record['answer_llm']\n",
    "    \n",
    "    v_llm = model.encode(answer_llm)\n",
    "    v_orig = model.encode(answer_orig)\n",
    "    \n",
    "    return v_llm.dot(v_orig)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "id": "9b70aad3-71f4-46a4-81df-9c34ecfa6283",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "33bcfad06b244bfdb457fb3396e72e48",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/1830 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "similarity = []\n",
    "\n",
    "for record in tqdm(results_gpt4o):\n",
    "    sim = compute_similarity(record)\n",
    "    similarity.append(sim)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 142,
   "id": "39e10c57-34a0-41bf-9867-10a50fc6d9ac",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count    1830.000000\n",
       "mean        0.679129\n",
       "std         0.217995\n",
       "min        -0.153426\n",
       "25%         0.591460\n",
       "50%         0.734788\n",
       "75%         0.835390\n",
       "max         0.995339\n",
       "Name: cosine, dtype: float64"
      ]
     },
     "execution_count": 142,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_gpt4o['cosine'] = similarity\n",
    "df_gpt4o['cosine'].describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 144,
   "id": "96b69a9a-ecd6-43fa-85da-e4ca00bdc61b",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Matplotlib is building the font cache; this may take a moment.\n"
     ]
    }
   ],
   "source": [
    "import seaborn as sns"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0651cb0c-96f7-449e-93a4-66f1e6e1c31d",
   "metadata": {},
   "source": [
    "### gpt-3.5-turbo"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "id": "3f497b01-c8c6-4809-8ac8-2be8db0ca578",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "696d1c115310441aadc8eebf7aa69760",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/1830 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "results_gpt35 = df_gpt35.to_dict(orient='records')\n",
    "\n",
    "similarity_35 = []\n",
    "\n",
    "for record in tqdm(results_gpt35):\n",
    "    sim = compute_similarity(record)\n",
    "    similarity_35.append(sim)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "id": "524528a3-1651-4200-8240-badcaeee67ec",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count    1830.000000\n",
       "mean        0.657599\n",
       "std         0.226062\n",
       "min        -0.168921\n",
       "25%         0.546504\n",
       "50%         0.714783\n",
       "75%         0.817262\n",
       "max         1.000000\n",
       "Name: cosine, dtype: float64"
      ]
     },
     "execution_count": 147,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_gpt35['cosine'] = similarity_35\n",
    "df_gpt35['cosine'].describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "id": "41e8a2ab-af71-48ea-8128-724160150592",
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "588d15df-d806-40d9-938d-d4abdb3b645c",
   "metadata": {},
   "source": [
    "### gpt-4o-mini"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 151,
   "id": "17e1ff94-5cea-4d36-b80c-ba31026663d7",
   "metadata": {},
   "outputs": [],
   "source": [
    "def process_record_4o_mini(rec):\n",
    "    model = 'gpt-4o-mini'\n",
    "    answer_llm = rag(rec, model=model)\n",
    "    \n",
    "    doc_id = rec['document']\n",
    "    original_doc = doc_idx[doc_id]\n",
    "    answer_orig = original_doc['text']\n",
    "\n",
    "    return {\n",
    "        'answer_llm': answer_llm,\n",
    "        'answer_orig': answer_orig,\n",
    "        'document': doc_id,\n",
    "        'question': rec['question'],\n",
    "        'course': rec['course'],\n",
    "    }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 152,
   "id": "2e24207e-9ca8-4717-9e71-c0d3d0f62046",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'answer_llm': \"Yes, sessions are recorded, so if you miss one, you won't miss anything. You can catch up by watching the recorded sessions later. Additionally, you have the option to ask questions in advance for office hours, which will also be recorded.\",\n",
       " 'answer_orig': 'Everything is recorded, so you won’t miss anything. You will be able to ask your questions for office hours in advance and we will cover them during the live stream. Also, you can always ask questions in Slack.',\n",
       " 'document': '5170565b',\n",
       " 'question': 'Are sessions recorded if I miss one?',\n",
       " 'course': 'machine-learning-zoomcamp'}"
      ]
     },
     "execution_count": 152,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "process_record_4o_mini(ground_truth[10])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c7d54408-6264-4ecf-8f71-d637706c1be3",
   "metadata": {},
   "outputs": [],
   "source": [
    "results_gpt4omini = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "id": "6fecebc2-d7c6-4474-b17c-5732f47fe366",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "66d49b2bff16465695171cef821b2c4b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/1830 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "for record in tqdm(ground_truth):\n",
    "    result = process_record_4o_mini(record)\n",
    "    results_gpt4omini.append(result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 160,
   "id": "028defb1-9250-49c8-8c93-432e94a77763",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_gpt4o_mini = pd.DataFrame(results_gpt4omini)\n",
    "df_gpt4o_mini.to_csv('data/results-gpt4o-mini.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 161,
   "id": "d4fc4de4-0622-4bcd-b0ff-61d9954842ab",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e40ff4a2e0944333926f288ac066b45d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/1830 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "similarity_4o_mini = []\n",
    "\n",
    "for record in tqdm(results_gpt4omini):\n",
    "    sim = compute_similarity(record)\n",
    "    similarity_4o_mini.append(sim)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 162,
   "id": "22cd9af3-d3ca-4ab3-bbe1-1b8665aaeece",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count    1830.000000\n",
       "mean        0.680332\n",
       "std         0.215962\n",
       "min        -0.141910\n",
       "25%         0.585866\n",
       "50%         0.733998\n",
       "75%         0.836750\n",
       "max         0.982701\n",
       "Name: cosine, dtype: float64"
      ]
     },
     "execution_count": 162,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_gpt4o_mini['cosine'] = similarity_4o_mini\n",
    "df_gpt4o_mini['cosine'].describe()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f5478e7d-015e-4356-8e8d-f68dcf3ca156",
   "metadata": {},
   "source": [
    "gpt4o \n",
    "\n",
    "```\n",
    "count    1830.000000\n",
    "mean        0.679129\n",
    "std         0.217995\n",
    "min        -0.153426\n",
    "25%         0.591460\n",
    "50%         0.734788\n",
    "75%         0.835390\n",
    "max         0.995339\n",
    "Name: cosine, dtype: float64\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 250,
   "id": "145be2b7-d84d-47cf-b39e-6241dc20b535",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\alexe\\AppData\\Local\\Temp\\ipykernel_8108\\4043211035.py:3: UserWarning: \n",
      "\n",
      "`distplot` is a deprecated function and will be removed in seaborn v0.14.0.\n",
      "\n",
      "Please adapt your code to use either `displot` (a figure-level function with\n",
      "similar flexibility) or `histplot` (an axes-level function for histograms).\n",
      "\n",
      "For a guide to updating your code to use the new functions, please see\n",
      "https://gist.github.com/mwaskom/de44147ed2974457ad6372750bbe5751\n",
      "\n",
      "  sns.distplot(df_gpt4o['cosine'], label='4o')\n",
      "C:\\Users\\alexe\\AppData\\Local\\Temp\\ipykernel_8108\\4043211035.py:4: UserWarning: \n",
      "\n",
      "`distplot` is a deprecated function and will be removed in seaborn v0.14.0.\n",
      "\n",
      "Please adapt your code to use either `displot` (a figure-level function with\n",
      "similar flexibility) or `histplot` (an axes-level function for histograms).\n",
      "\n",
      "For a guide to updating your code to use the new functions, please see\n",
      "https://gist.github.com/mwaskom/de44147ed2974457ad6372750bbe5751\n",
      "\n",
      "  sns.distplot(df_gpt4o_mini['cosine'], label='4o-mini')\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<matplotlib.legend.Legend at 0x2d4f39b8140>"
      ]
     },
     "execution_count": 250,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAj4AAAHHCAYAAAC/R1LgAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/TGe4hAAAACXBIWXMAAA9hAAAPYQGoP6dpAAB/70lEQVR4nO3dd3hUZdrH8e+ZmWTSeychofeOIE1AUURUEAuCr5QVy4ptWRtb7CuWVdm1FwRREMuCDUWQqoJ0kA4pkADpvWfK8/4xMBISIAlJTpK5P9c1F86ZZ878ToLJzVPOoymlFEIIIYQQLsCgdwAhhBBCiMYihY8QQgghXIYUPkIIIYRwGVL4CCGEEMJlSOEjhBBCCJchhY8QQgghXIYUPkIIIYRwGVL4CCGEEMJlSOEjhBBCCJchhY8QQpxh69atDB48GG9vbzRNY9euXXpHEkLUIyl8hGgECxYsQNM058NkMtGqVSumTZvGiRMnzvm+t956C03TGDhw4HnPn5GRweOPP06PHj3w8fHBw8OD9u3bM336dH755ZcL5jt69CiapvHvf//7vO3i4uK49tprz9tm2rRpaJqGn58fpaWlVV4/cuSI8+twoc9rbBaLhZtvvpmcnBxee+01Pv74Y2JjY/WOJYSoRya9AwjhSp555hnatGlDWVkZv/32GwsWLOCXX35h7969eHh4VGm/aNEi4uLi2LJlC/Hx8bRv375Kmy1btjB27FgKCwu59dZbueeeezCbzSQlJfHVV1+xYMEC1q9fz2WXXdYYlwiAyWSipKSEb7/9lltuuaXSa4sWLcLDw4OysrJGy1NTCQkJHDt2jPfff58ZM2boHUcI0QCk8BGiEY0ZM4b+/fsDMGPGDEJCQnjxxRf55ptvqhQISUlJbNy4kaVLl3L33XezaNEinnzyyUptcnNzGT9+PCaTiV27dtG5c+dKrz/33HMsWbIET0/Phr2ws5jNZoYMGcKnn35a5boWL17M2LFj+d///teomc6nuLgYb29vMjIyAAgICKj3cwshmgYZ6hJCR8OGDQMcPQ1nW7RoEYGBgYwdO5abbrqJRYsWVWnzzjvvkJqayty5c6sUPQCapjFp0iQuueSS+g9/AZMnT+aHH34gLy/PeWzr1q0cOXKEyZMn1+gcZw7Bvfbaa8TGxuLp6cnw4cPZu3dvlfYHDx7kpptuIigoCA8PD/r3788333xTqc3pYcf169dz7733EhYWRnR0NNOmTWP48OEA3HzzzWiaxogRI5zvW7NmDcOGDcPb25uAgADGjRvHgQMHKp37qaeeQtM09u/fz+TJkwkMDGTo0KHAH8OE69ato3///nh6etKjRw/WrVsHwNKlS+nRowceHh7069ePnTt3Vjr377//zrRp02jbti0eHh5ERETwpz/9iezs7GozxMfHM23aNAICAvD392f69OmUlJRU+Zp98sknDBgwAC8vLwIDA7nssstYuXJlpTY//PCD89p9fX0ZO3Ys+/btO893ToimSwofIXR09OhRAAIDA6u8tmjRIiZMmIC7uzuTJk3iyJEjbN26tVKbb7/9Fk9PTyZMmNAYcWtlwoQJaJrG0qVLnccWL15M586d6du3b63OtXDhQv773/8yc+ZMZs+ezd69e7n88stJT093ttm3bx+XXnopBw4c4PHHH+eVV17B29ub8ePHs2zZsirnvPfee9m/fz9PPPEEjz/+OHfffTd/+9vfAHjggQf4+OOP+fvf/w7ATz/9xOjRo8nIyOCpp55i1qxZbNy4kSFDhji/h2e6+eabKSkp4fnnn+fOO+90Ho+Pj2fy5Mlcd911zJkzh9zcXK677joWLVrEX/7yF/7v//6Pp59+moSEBG655RbsdrvzvatWrSIxMZHp06fz+uuvc+utt7JkyRKuueYalFJVMtxyyy0UFhYyZ84cbrnlFhYsWMDTTz9dqc3TTz/N7bffjpubG8888wxPP/00MTExrFmzxtnm448/ZuzYsfj4+PDiiy/yz3/+k/379zN06NBqr12IJk8JIRrc/PnzFaB++uknlZmZqVJSUtSXX36pQkNDldlsVikpKZXab9u2TQFq1apVSiml7Ha7io6OVg8++GCldoGBgap3795VPq+goEBlZmY6H0VFRefNl5SUpAD18ssvn7ddbGysGjt27HnbTJ06VXl7eyullLrpppvUFVdcoZRSymazqYiICPX000/X+PNOt/P09FTHjx93Ht+8ebMC1F/+8hfnsSuuuEL16NFDlZWVOY/Z7XY1ePBg1aFDB+ex09+LoUOHKqvVWunz1q5dqwD1xRdfVDreu3dvFRYWprKzs53Hdu/erQwGg5oyZYrz2JNPPqkANWnSpCrXEhsbqwC1ceNG57Eff/zReX3Hjh1zHn/33XcVoNauXes8VlJSUuWcn376qQLUhg0bqmT405/+VKntDTfcoIKDg53Pjxw5ogwGg7rhhhuUzWar1NZutyullCosLFQBAQHqzjvvrPR6Wlqa8vf3r3JciOZAenyEaESjRo0iNDSUmJgYbrrpJry9vfnmm2+Ijo6u1G7RokWEh4czcuRIwDFkNXHiRJYsWYLNZnO2KygowMfHp8rn3H777YSGhjofjz32WMNe2DlMnjyZdevWkZaWxpo1a0hLS6vxMNeZxo8fT6tWrZzPBwwYwMCBA/n+++8ByMnJYc2aNc5ejqysLLKyssjOzmb06NEcOXKkyuq5O++8E6PReMHPTk1NZdeuXUybNo2goCDn8Z49e3LllVc6M5zpnnvuqfZcXbt2ZdCgQc7np1frXX755bRu3brK8cTEROexM+dplZWVkZWVxaWXXgrAjh07Lphh2LBhZGdnU1BQAMBXX32F3W7niSeewGCo/KtA0zTA0cuUl5fHpEmTnF/TrKwsjEYjAwcOZO3atdVepxBNmRQ+QjSiN998k1WrVvHll19yzTXXkJWVhdlsrtTGZrOxZMkSRo4cSVJSEvHx8cTHxzNw4EDS09NZvXq1s62vry9FRUVVPueZZ55h1apVrFq1qsGv6XyuueYafH19+eyzz1i0aBGXXHJJtSvTLqRDhw5VjnXs2NE51BIfH49Sin/+85+VCr7Q0FDnhPDTE5dPa9OmTY0++9ixYwB06tSpymtdunQhKyuL4uLiGp37zOIGwN/fH4CYmJhqj+fm5jqP5eTk8OCDDxIeHo6npyehoaHOz8nPz7/gZ50eTj19zoSEBAwGA127dq02KzhuPQCOwuzsr+vKlSurfE2FaA5kVZcQjWjAgAHOVV3jx49n6NChTJ48mUOHDjl7btasWUNqaipLlixhyZIlVc6xaNEirrrqKgA6d+7M7t27sVgsuLm5Odv07NmzEa7mwsxmMxMmTOCjjz4iMTGRp556qkE+5/RcmIcffpjRo0dX2+bsgqshV7qd69zn6mE613F1xtydW265hY0bN/LII4/Qu3dvfHx8sNvtXH311ZXmAtXmnBdy+rwff/wxERERVV43meRXiGh+5G+tEDoxGo3MmTOHkSNH8sYbb/D4448DjsImLCyMN998s8p7li5dyrJly3jnnXfw9PTk2muv5bfffmPZsmVVlo03FZMnT+bDDz/EYDBw66231ukcp3seznT48GHi4uIAaNu2LQBubm6MGjWqzlmrc/oGhocOHary2sGDBwkJCWnw5eq5ubmsXr2ap59+mieeeMJ5vLqvS021a9cOu93O/v376d279znbAISFhdX711UIvchQlxA6GjFiBAMGDGDu3LmUlZVRWlrK0qVLufbaa7npppuqPO677z4KCwudS7T//Oc/Ex4ezl/+8hcOHz5c5fy1+dd9Qxk5ciTPPvssb7zxRrW9BjXx1VdfVZqjs2XLFjZv3syYMWMAxy/mESNG8O6775Kamlrl/ZmZmXULD0RGRtK7d28++uijSkvz9+7dy8qVK7nmmmvqfO6aOt17c/b3c+7cuXU+5/jx4zEYDDzzzDNVeoxOf87o0aPx8/Pj+eefx2KxVDnHxXxdhdCL9PgIobNHHnmEm2++mQULFhAYGEhhYSHXX399tW0vvfRSQkNDWbRoERMnTiQoKIhly5Zx3XXX0atXL2699VYuueQS3NzcSElJ4YsvvgCqzvc4l9WrV1d7R+Xx48fTvXt3wDGf5rnnnqvSpk+fPowdO7bKcYPBwD/+8Y8aff65tG/fnqFDh/LnP/+Z8vJy5s6dS3BwMI8++qizzZtvvsnQoUPp0aMHd955J23btiU9PZ1NmzZx/Phxdu/eXefPf/nllxkzZgyDBg3ijjvuoLS0lNdffx1/f/8GG747k5+fH5dddhkvvfQSFouFVq1asXLlSpKSkup8zvbt2/P3v/+dZ599lmHDhjFhwgTMZjNbt24lKiqKOXPm4Ofnx9tvv83tt99O3759ufXWWwkNDSU5OZnly5czZMgQ3njjjXq8UiEanhQ+QuhswoQJtGvXjn//+9906dIFDw8PrrzyymrbGgwGxo4dy6JFi8jOziY4OJhBgwaxd+9eXn31VZYvX85nn32G3W6nVatWDB06lPfee895o8QLWbFiBStWrKhyPC4uzln4HDp0iH/+859V2txxxx3VFj71YcqUKRgMBubOnUtGRgYDBgzgjTfeIDIy0tmma9eubNu2jaeffpoFCxaQnZ1NWFgYffr0qTQ8VBejRo1ixYoVPPnkkzzxxBO4ubkxfPhwXnzxxRpPkr5Yixcv5v777+fNN99EKcVVV13FDz/8QFRUVJ3PeXoLlddff52///3veHl50bNnT26//XZnm8mTJxMVFcULL7zAyy+/THl5Oa1atWLYsGFMnz69Pi5NiEalqabQFy6EENU4evQobdq04eWXX+bhhx/WO44QogWQOT5CCCGEcBlS+AghhBDCZUjhI4QQQgiXIXN8hBBCCOEypMdHCCGEEC5DCh8hhBBCuAyXu4+P3W7n5MmT+Pr6OncgFkIIIUTTppSisLCQqKgoDIa699u4XOFz8uTJKjshCyGEEKJ5SElJITo6us7vd7nCx9fXF3B84fz8/HROI4QQQoiaKCgoICYmxvl7vK5crvA5Pbzl5+cnhY8QQgjRzFzsNBWZ3CyEEEIIlyGFjxBCCCFchhQ+QgghhHAZLjfHp6ZsNhsWi0XvGAJwc3PDaDTqHUMIIUQLIIXPWZRSpKWlkZeXp3cUcYaAgAAiIiLk3ktCCCEuihQ+Zzld9ISFheHl5SW/aHWmlKKkpISMjAwAIiMjdU4khBCiOZPC5ww2m81Z9AQHB+sdR5zi6ekJQEZGBmFhYTLsJYQQos5kcvMZTs/p8fLy0jmJONvp74nMuxJCCHExpPCphgxvNT3yPRFCCFEfpPARQgghhMuQwkcIIYQQLkMmN9fQ4s3Jjfp5kwe2rvN7X3jhBWbPns2DDz7I3Llz6y+UEEII0cxJj08Ls3XrVt5991169uypdxQhhBCiyZHCpwUpKiritttu4/333ycwMLDSa8nJyYwbNw4fHx/8/Py45ZZbSE9P1ympEEIIoQ8pfFqQmTNnMnbsWEaNGlXpuN1uZ9y4ceTk5LB+/XpWrVpFYmIiEydO1CmpEEIIoQ+Z49NCLFmyhB07drB169Yqr61evZo9e/aQlJRETEwMAAsXLqRbt25s3bqVSy65pLHjCiFE/do2v3bt+09vmByiyZMenxYgJSWFBx98kEWLFuHh4VHl9QMHDhATE+MsegC6du1KQEAABw4caMyoQgghhK6k8GkBtm/fTkZGBn379sVkMmEymVi/fj3//e9/MZlMKKX0jiiEEEI0CTLU1QJcccUV7Nmzp9Kx6dOn07lzZx577DFSU1NJSUkhJSXF2euzf/9+8vLy6Nq1qx6RhRBCCF1I4dMC+Pr60r1790rHvL29CQ4Opnv37nTr1o0ePXpw2223MXfuXKxWK/feey/Dhw+nf//+OqUWQgghGp8MdbkATdP4+uuvCQwM5LLLLmPUqFG0bduWzz77TO9oQgghRKOSHp8aupg7Keth3bp1lZ63bt2ar7/+Wp8wQgghRBMhPT5CCCGEcBnS4yOEEOKiNMZehs2t1100XdLjI4QQQgiXIYWPEEIIIVyGFD5CCCGEcBlS+AghhBDCZcjkZiGEEC3S5qScc76WYKs6IVsmULsGXXt85syZwyWXXIKvry9hYWGMHz+eQ4cOnfc9CxYsQNO0So/qNuYUQgghhDibroXP+vXrmTlzJr/99hurVq3CYrFw1VVXUVxcfN73+fn5kZqa6nwcO3askRILIYQQojnTdahrxYoVlZ4vWLCAsLAwtm/fzmWXXXbO92maRkREREPHE3UUFxfHQw89xEMPPVSj9uvWrWPkyJHk5uYSEBDQoNmEEEK4tiY1xyc/Px+AoKCg87YrKioiNjYWu91O3759ef755+nWrVu1bcvLyykvL3c+LygoqFu4bfPr9r666j+9zm994YUXmD17Ng8++CBz586tv0w1tHXrVry9vWvcfvDgwaSmpuLv79+AqYQQQogmtKrLbrfz0EMPMWTIkCo7jZ+pU6dOfPjhh3z99dd88skn2O12Bg8ezPHjx6ttP2fOHPz9/Z2PmJiYhrqEJmHr1q28++679OzZU7cMoaGheHl51bi9u7s7ERERaJrWgKmEEEKIJlT4zJw5k71797JkyZLzths0aBBTpkyhd+/eDB8+nKVLlxIaGsq7775bbfvZs2eTn5/vfKSkpDRE/CahqKiI2267jffff5/AwMBKryUnJzNu3Dh8fHzw8/PjlltuIT09/bznW7BgAQEBAXz33Xd06tQJLy8vbrrpJkpKSvjoo4+Ii4sjMDCQBx54AJvN5nxfXFxcpZ4mTdP44IMPuOGGG/Dy8qJDhw588803ztfXrVuHpmnk5eXVy9dBCCGEOJcmMdR133338d1337Fhwwaio6Nr9V43Nzf69OlDfHx8ta+bzWbMZnN9xGzyZs6cydixYxk1ahTPPfec87jdbncWPevXr8dqtTJz5kwmTpxYZRf3s5WUlPDf//6XJUuWUFhYyIQJE7jhhhsICAjg+++/JzExkRtvvJEhQ4YwceLEc57n6aef5qWXXuLll1/m9ddf57bbbuPYsWMXHNYUQjR/7ZK/qFX7hNY3N1ASIXQufJRS3H///Sxbtox169bRpk2bWp/DZrOxZ88errnmmgZI2HwsWbKEHTt2sHXr1iqvrV69mj179pCUlOQc6lu4cCHdunVj69atXHLJJec8r8Vi4e2336Zdu3YA3HTTTXz88cekp6fj4+ND165dGTlyJGvXrj1v4TNt2jQmTZoEwPPPP89///tftmzZwtVXX30xly2EEELUiq5DXTNnzuSTTz5h8eLF+Pr6kpaWRlpaGqWlpc42U6ZMYfbs2c7nzzzzDCtXriQxMZEdO3bwf//3fxw7dowZM2bocQlNQkpKCg8++CCLFi2q9p5GBw4cICYmptL8pq5duxIQEMCBAwcA6NatGz4+Pvj4+DBmzBhnOy8vL2fRAxAeHk5cXBw+Pj6VjmVkZJw345lzjry9vfHz87vge4QQQoj6pmuPz9tvvw3AiBEjKh2fP38+06ZNAxxzUwyGP+qz3Nxc7rzzTtLS0ggMDKRfv35s3LiRrl27NlbsJmf79u1kZGTQt29f5zGbzcaGDRt44403eOWVVy54ju+//x6LxQKAp6en87ibm1uldpqmVXvMbref9/x1eY8QQghR33Qf6rqQs+egvPbaa7z22msNlKh5uuKKK9izZ0+lY9OnT6dz58489thjpKamkpKSQkpKirPXZ//+/eTl5TkLxtjY2EbPLYQQQjS2JjG5WVwcX1/fKrcA8Pb2Jjg4mO7du9OtWzd69OjBbbfdxty5c7Fardx7770MHz6c/v3765RaCCGqKrIa2J2Sx5GMIvJLKygut2E0aPx8JJO2od5c3jmMPjGBGAxy+wtRN1L4uABN0/j666+5//77ueyyyzAYDFx99dW8/vrrekcTQggAsitMfHEyhA3Z/tioetuRE3mOuZ9vrk0gzNfMtCFxTBsch5e7/BoTtaOpmow3tSAFBQX4+/uTn5+Pn59fpdfKyspISkqiTZs2svFpEyPfGyGarsWbq+50fqbzLWe3K/g6LZj/pQZjUY75nGG+ZjpH+BHuZ8bbbMJqU7QL82Znch5rD2ZQWG4FIMTHnUdHd+bm/tFo2xdUOfd5d2evZsm87M7etJ3v93dtSKkshBBCF2U2jTeORrE1zxeAzj4l3NYqA2PX66q0nTywNdOHQLnVxvLfU5n70xGSc0p49H+/s+FIJs+30fBzc6l/x4s6ajJ3bhZCCOE6Cq0G/nkolq15vpg0O3fHpvJUx2Q6+pSd931mk5EJfaNZ/dfhPHZ1Z0wGje9+T+X61YGcKJFfaeLC5G+JEEKIRlVu13gpPprkUg/8TVae7JjM5SH51Ga7PjejgT+PaMfn9wyiVYAnR4tM3Lo+kOPF8mtNnJ/8DRFCCNFobAr+kxjF4WIvvI02njhPL4+bpZCw7C20Sl8DB5fDsU1QUVKpTd/WgXz550HE+VhJKTZy6/pA0krlV5s4N5njUw0Xm+/dLMj3RIiW4cuTIWzP98VNs/No++NEe1ZUet1oK6Nz0ke0Pf4VAUVn7MG449SfmhGiekO/6dDzFjCZifT3ZMnwPCavDyCxyMS9m/xZMiK30a5JNC9S+Jzh9N2FS0pKKt29WOivpMTxr7yz7wAthGg+DhV5siwtGIA/x6XS2eeP7Yk0ZSMyayNRWb9gtFucx4s8W1FqDiHUxx3yj0NhKpzY7nisfgaufAZ63UqEp535Q/O5dnUgO3LceP53H8b4n3tVl3BdUvicwWg0EhAQ4NxDysvLC602g86i3imlKCkpISMjg4CAAIxGo96RhBB1UGoz8GZSJAqNYUH5DAkqdL7mWZZBuxNf412WCkCeT3sOxd3G8bCRlJsdhdLkga1BKUfxs28pbH4XCk7AV/fAoeUQM5BYHx9eu6SAGRsDWBDvRUBbXwYFFlabR7guKXzOEhERASAbaDYxAQEBzu+NEKL5+fREKOkV7oS4W/hT63Tncb+iJDqmfIbRXoHV6MHRiKvZ0uMZqp3prGkQEANDHoRL74WN/4W1z8OBbx3zfwbNZFRUIPd2KuatQ97MTw6np28x3ibZF1D8QQqfs2iaRmRkJGFhYc5NO4W+3NzcpKdHiGYsudSdlZkBANwTm4qX0VGIBBYcoP3xpRiUjQKvWOKjJ2Bx862+6Dmb0Q2G/RXaXQGf3Q75ybDxdRg0k4e6wY8nzSQUmvjsZGilQksIKXzOwWg0yi9bIYS4SEopPj4ehkJjQEAhPfwc8/X8i+LpkPIlGoocvy7Et7oBZajDr6So3vCnH+Dd4VCSBZvewH3oX3m2jxuTNwSyMjOAESF5tPUqr98LE82WFD5CCCFqbtv8KofaJZ97EvHOfG9+L4jBpNm5LdoxhcCzLJP2Kf9DQ5Hl34OEVuNAu4gl6P7RMPh+R49PSRbsXMjgAXczOLCAjbl+zE8O55lOybW6T5BouaTwEUII0SDsCj45HgbAmLBcIswWTNYSOiZ/isleToFXaxKjrqtR0XPB/cBSbXhG3kS3pHkYsw5zcsv/uD36arbm+XC42Is9hV709Cs57zmEa5C7PAkhhGgQW/N8OV5mxtto44aIbFCKuNTleFjyKHMP5EjMLXUb3jqHUo8wEqPGARCVtZG4isOMCs0DYFlqSL19jmjepPARQghR75TCec+e0aG5eJvsBOfvJbjgAHYMHIm+CavJq94/N8e/K+mB/QGIS/2ecaHpGDXF/iIvDhbJ/dmEFD5CCCEawO+FXiSVeGA22BkTnoubpYC41B8AOBk6jBLPyAb77JTwy6kw+eBZkUP3gvWMCM4DYFlqcIN9pmg+pPARQghR7746NbR0RUgefiYbsWkrMdnLKPKI4mTo0Ab9bJvRg2MRVwMQlfUrk4MOYUCxq8CH5FJzg362aPqk8BFCCFGv4os92F/khVFTXBueg2/xMYIL9qPQSGx1HUpr+FuF5Ph1Ic+nPQZlo3feKvoHFAGwJsu/wT9bNG1S+AghhKhXp29WODiwgGC3CmLTfgQgI7AvpR7hjRNC00gOH4UCggsOcLPfAQA2ZPtTYZd17a5MCh8hhBD1pshqYGOOHwBXheYSmrcb77I0rAYzx8NGNGqWUo8wcvy6AnB56QpC3C0U24xsyfNt1ByiaZHCRwghRL1Zn+2PRRmI9Syjo2cRrTLWA3Ai9DKsJu9Gz3MidLij16fwIJP99wKwOlOGu1yZFD5CCCHqhV3BqsxAAK4MzSMsfzdmawEVJl/Sgy7RJVOpRyjZ/t0BuFV9j4Zif5E3WUWyhYWrksJHCCFEvdhb6EVquTueBhuXBeYQlfULACdDBtfrjQpr62TIEAAii/Yz0vc4ALtS8nTLI/QlhY8QQoh6sS7bMYQ0NLiA6MLdmC35VJi8yQjsq2uuUo9wCrxao6GY4b4KgH0n83XNJPQjhY8QQoiLVmozsDXXMWl4RFCus7cnNXgwyuCmZzQA0oMGANCvbBMeWEgvKCerUIa7XJFsUiqEEOKibcnzoUIZiDRX0N++B4+KXKxGDzKC+tX6XO2Sv6j3fLl+nagw+WK2FjLD+2feKL6cvSfzGdEprN4/SzRt0uMjhBDiom04Ncw1LDifiJzNgOO+PXaDu56xnJRmdA65TTT8BMC+kwV6RhI6kcJHCCHERcmuMLGv0LHh6DXeh/EvPopCIz1Qn5Vc55IR2BeFRozlKLFaOifySsktrtA7lmhkUvgIIYS4KL/k+KHQ6OxTQteiTQDk+HWmwr1p3S/H4uZLvncbAKb6bgVkkrMrksJHCCHERTl9p+YrA1IJyfsd+GMycVOTHdADgLHqZ0BxIK1Q30Ci0UnhI4QQos6OFhk5WuqBAcVYw68YlJVij3AKvVrrHa1aOb6dsWkmwi0p9NCSSM4uodxq0zuWaERS+AghhKizH46bAejmU0zr/B0AZAb0Ba1pbgRqN5rJ8+0EwETzJmxKkZRVrHMq0ZhkObsQQog6+/6Eo/C5wXcfXrkZ2DUjWQHddU51flkB3Qku2MdYw0ae5FbiM4roHOHH4s3JtTrP5IFNs1dLnJ/0+AghhKiTlGIDe3Ld0FBcaf8VgBy/rtiMnjonO7987/aUu/kRaM+ln3aYIxlFekcSjUgKHyGEEHVyepirj08ukYV7AMgM7K1joppRBiMnQy8D4ErjDjILy8krkWXtrkIKHyGEEHWy/LgHALd7bsRkL6fMLZACrzh9Q9XQ8bCRAFzttgNQxEuvj8uQwkcIIUStpZUa2H1qmGuYzXGn5qyAnk12UvPZUkOHYNPciFGptNNOynCXC5HCRwghRK2tTnVsRTEiMIvgkkQAsk7dI6c5sJq8SQ923GvoSsN2EjKLUErpnEo0Bil8hBBC1NpPJx3ze/7kvRENRZFnK8rdg3ROVTvHwy8HHPN8SipsZBbJbu2uQAofIYQQtVJihV8zHD0+/cscW1Rk+Tef3p7TToQNB6CP4Qgh5HMsu0TnRKIxSOEjhBCiVn5Od6fCrjHI6wSeRckoNLL9u+odq9ZKPcLJ9u+GAcUI4y6OZcuNDF2BFD5CCCFqZXWqY5jrbp+fAcj3aYvV5KNnpDpLDRkMwBDDXo5Kj49LkDs3CyGEqDG7gjWnCp8BFVsAyPZv2ndqrk675C8AMNgc83qGGvaSU1xOcMJSAtyq37srofXNjZZPNBzp8RFCCFFju3JMZJUb6OWWgldZGmhGck/tfdUcFXlGY9PcCNXy6aSlcKjIS+9IooFJ4SOEEKLG1qc5entm+DgmNRPaCZvRQ8dEF0cZTBR6xwIw1LCHg0VNe7sNcfGk8BFCCFFjG9Idq7mG2R3DXET20jFN/cj3bgM4hruk8Gn5pPARQghRI3klFezOMdFOO0FA+UnQDBDe/Jaxny3fpy0AAw0HOVlipMzWPO4+LepGCh8hhBA18kt8FnY0/s/rN8eBkI7g3vznxJSaw6gweeOlldPbEE9CifT6tGRS+AghhKiRDYczARhjOD3M1Vu/MPVJ0yjwdvT6DDHsJbG4+c5ZEhcmhY8QQogLUkqx4XAW0VomEZYUQIOI5reM/VwKvOMAGGA4SEKJFD4tmRQ+QgghLuhIRhFpBWWMNm53HAhqC+7N86aF1Sn0ag1Aby2BlBKjzmlEQ5LCRwghxAWtP+QY5rrBY5vjQHjL6e0BKHMPosLojVmzEF5xnCKr/HpsqeQ7K4QQ4oI2HMnEj2K6Wg86DrSwwgdNo8grBoABhkMkynBXiyWFjxBCiPMqrbCxOSmH4YbdGLCDTxj4hOodq96dHu7qbzhEgkxwbrF0LXzmzJnDJZdcgq+vL2FhYYwfP55Dhw5d8H1ffPEFnTt3xsPDgx49evD99983QlohhHBNvyVlU2G1c53HbseBFnDvnuoUev9R+CQVm3VOIxqKroXP+vXrmTlzJr/99hurVq3CYrFw1VVXUVxcfM73bNy4kUmTJnHHHXewc+dOxo8fz/jx49m7d28jJhdCCNex4XAmJqwMY6fjQEsb5jql2CMCi+aOv1aCVpqldxzRQHTdnX3FihWVni9YsICwsDC2b9/OZZddVu17/vOf/3D11VfzyCOPAPDss8+yatUq3njjDd55550GzyyEEC3d4s3JlZ5/tzuVAYaDeNqLsBi92JHnC/k5OqVrQJqBQs9ogkoSaW9LJN8Sh/85dmoXzVeTmuOTn58PQFBQ0DnbbNq0iVGjRlU6Nnr0aDZt2lRt+/LycgoKCio9hBBC1ExuSQWZReVcaXAsY8/17ejYqqKFKvE+PcH5oExwbqGazN9eu93OQw89xJAhQ+je/dzdqGlpaYSHh1c6Fh4eTlpaWrXt58yZg7+/v/MRExNTr7mFEKIlO5JeBCiudnMMc+X5dtI3UAMrPLWyq6/hiBQ+LVSTKXxmzpzJ3r17WbJkSb2ed/bs2eTn5zsfKSkp9Xp+IYRoyY5kFNJJSyFSZWA1mMn3aaN3pAZV7NkKOxrRWhZ5xeV6xxENQNc5Pqfdd999fPfdd2zYsIHo6Ojzto2IiCA9Pb3SsfT0dCIiIqptbzabMZtldr4QQtSWXSkSMouYcWqYKz34UuwGd51TNSyb0Uy+KZRAawb+ZScAf70jiXqma4+PUor77ruPZcuWsWbNGtq0ufC/JAYNGsTq1asrHVu1ahWDBg1qqJhCCOGSUvPKKLPYucq0A4Dj4SP0DdRISryiAGhtO0aZTdM5jahvuhY+M2fO5JNPPmHx4sX4+vqSlpZGWloapaWlzjZTpkxh9uzZzucPPvggK1as4JVXXuHgwYM89dRTbNu2jfvuu0+PSxBCiBYrIbOIMHLpqSUAcCJ0uM6JGkeFt6Pw6aUlcLxMRgxaGl0Ln7fffpv8/HxGjBhBZGSk8/HZZ5852yQnJ5Oamup8PnjwYBYvXsx7771Hr169+PLLL/nqq6/OOyFaCCFE7SVkFnG50TGpOcu/B2UeLe9uzdUp8mwFQC9DAseK3XROI+qbrnN8lFIXbLNu3boqx26++WZuvvnmBkgkhBACwGq3czS7mL8aHHdrPhk6TOdEjafUI4wK3PDXSigtKdQ7jqhnTWZVlxBCiKYjJacUbBaGGfYArlX4KM1Ippuj18ev9ITOaUR9k8JHCCFEFQmZRfQ3HMJbK6PMPYgc/656R2pUxZ6OeT5RlmRqMDghmhEpfIQQQlSRmFnE8FPDXKkhQ1r03Zqro3wct0jppiWSVdEk7vwi6olr/U0WQghxQRVWOyk5pYw07AJca5jrtFJvx1BXV+0ox0uk8GlJpPARQghRydHsYsJUFp0Mx7FjIDVksN6RGl25WwBFeOGu2SgrytU7jqhHUvgIIYSoJCGziBHGXQBkB/Sgwt0F716saaSbHL0+5pLq94IUzZMUPkIIISpJzCx26WGu04o8HPN8wqyysqslkcJHCCGEU15JBVl5BQw27APgZOhQnRPpR3mHARBnT8Fq1zmMqDdS+AghhHD6LTGbfoZD+GhllLoHk+vXRe9I+vFxFD5dtGTSymSCc0shhY8QQginjQnZjDi9jD3U9Zaxn6ncHEwp7nhp5RQXyR2cWwrX/RsthBCiil/jsxgh83scNAPHDY4JzsaSdJ3DiPoihY8QQggA0gvKKMs8SkfDiVPL2AfpHUl32ae3rihPvUBL0VxI4SOEEAKAjQlZjDA6hrmyAnthcXPBZexnKfMKByDSelznJKK+SOEjhBACgI3xZ8zvCXHd1VxnMpxa2dVOpWCzydKulkAKHyGEECil2BqfxmDDXkDm95zm7hOIRRkJ1Iqw5CTrHUfUAyl8hBBCkJxTQqvCnXhr5ZS4h5Dr11nvSE2CZjRxTHPs1O6etVfnNKI+SOEjhBCCX88c5godCpqmc6KmI83oKHy88w7rnETUByl8hBBCsDEh64xtKmR+z5kKzY4JziEl8TonEfVBCh8hhHBxdrviWPx+2htOojQjabKMvRKrZygA0ZYknZOI+iCFjxBCuLjDGYX0Kt8GgIoegMXNT+dETYu7bxAAMfaTYC3TOY24WLL5iBBCuLJt8/n1iKfzbs0G7yDaJX+hb6Ymxs/Tg1zlQ6BWhCH7MPbwnnpHEhdBenyEEMLFbU3XGGzY73gS6sKbkp6DwaBxVHPcwdk966DOacTFkh4fIYRwYVY72LMT8DKWY3Hzw82vFeTk6h2ryUkzRoLtEOGZP2NKPrXizRh04Tf2n96wwUStSY+PEEK4sD25Jgaq3wEwhneRZeznkO8eAUBAhWxW2txJ4SOEEC5sY4b7H/N7wmSY61wsp1Z2RdhO6pxEXCwpfIQQwoUdSc+nnSEVOwYI7aR3nCbL5O0Y1gomH5O1ROc04mJI4SOEEC6qzGIjMG8fAOX+bcHNU+dETVeYl8Yxu2PDUlNJhs5pxMWQwkcIIVzUjuRchmqObSo8ImRvrvPxMdlJIBoAa3G2zmnExZDCRwghXNTmwycZbHD0+GjhMr/nQtKNkQC4lWXpnERcDCl8hBDCReUfXI+nVkGJyR98o/SO0+QVuDuGunwqMnVOIi6GFD5CCOGCCsostM75FQAVKsvYa8LiEQxAmDUNlNI5jagrKXyEEMIFbU7MYbi2CwDvKBnmqgmTdyB2peFDCSZbsd5xRB1J4SOEEC5o/77dtDOkYsMIIR31jtMshHsqkpVjuMuzTIa7mispfIQQwgVpCT8BkO8jy9hrKsxcwRHl2LNLleTonEbUlRQ+QgjhYtLyy+hWvAUAz0hZxl5TJg1SDY6tK7QSWdLeXEnhI4QQLmbToRPOZeyekTK/pzZy3cIB8JaVXc2WFD5CCOFiMvasxlOroNA9FHwj9Y7TrJSYHXt2BVlls9LmSgofIYRwIUop/E6sA6Ck9UhZxl5LyjMQAF9VBOVFOqcRdSGFjxBCuJD4jCIGWHcAENhrrM5pmp9gDwPJdkevD0Vp+oYRdSKFjxBCuJDdv++knSEVK0bcO1yud5xmJ8JcwRHl2LNLFUrh0xxJ4SOEEC6k/OCPAKQH9AEPP53TND8hZgvxyrG9R2me7NLeHEnhI4QQLsJisxOT5dimwtjxSp3TNE8mDTJOLWm3FMgE5+ZICh8hhHARe46mcwmOZexhfa7VOU3zVegeAoBbifT4NEdS+AghhIs4tmMVnloFuaZQDBHd9I7TbFk8ggDwsuaCtVznNKK2pPARQggXYU5ybFORHXmZLGO/CL4e7uQoH8eTYrmRYXMjhY8QQriA4jIL3Ys3ARDQ8xqd0zRvEWYLiacmOFMsw13NjUnvAEIIIRrW4s3JZB/9nfu1DCowsaaiO9bNyQC0S5bNNmsr3FxBkj2C/obDqKIMpO+seZEeHyGEcAFBJ9YCsM+9N1aTl85pmrcws4VE5djqo6JAhrqaGyl8hBDCBXQpcCxjPx46TOckzZ+7QZFpDAPAWiiFT3MjhY8QQrRwJXkZ9FIHHf8dJ/fvqQ/Fp5a0u5dmgFI6pxG1IYWPEEK0cO5H12DUFAlaLDa/GL3jtAh2sz82peFmL4PyAr3jiFqQwkcIIVq42KwNABzyH6JzkpYj2AOOq9OblcrKruakToVPYmJifecQQgjRACrKy+lTsR2A3GjZlLS+RJgrnBOcZUl781Knwqd9+/aMHDmSTz75hLKysvrOJIQQop4c3rYSP62EbOUHrfrpHafFqHwvH5ng3JzUqfDZsWMHPXv2ZNasWURERHD33XezZcuW+s4mhBDiIhXvWQ7Abo8BaAa5dVt9CT+jx8dSID0+zUmdCp/evXvzn//8h5MnT/Lhhx+SmprK0KFD6d69O6+++iqZmTWrfjds2MB1111HVFQUmqbx1Vdfnbf9unXr0DStyiMtLa0ulyGEEC1eq4z1AJwIG65zkpbFw6jIdXMsabcXSY9Pc3JRk5tNJhMTJkzgiy++4MUXXyQ+Pp6HH36YmJgYpkyZQmpq6nnfX1xcTK9evXjzzTdr9bmHDh0iNTXV+QgLC7uYyxBCiBYp69g+ou0nqVBGbG1G6B2nxbF5hQPgVpYNdqvOaURNXVS/57Zt2/jwww9ZsmQJ3t7ePPzww9xxxx0cP36cp59+mnHjxp13CGzMmDGMGTOm1p8bFhZGQEDARSQXQoiW7/hvSwkBdhm74+4doHecFsfP15eiYg98tDIoyQafcL0jiRqoU4/Pq6++So8ePRg8eDAnT55k4cKFHDt2jOeee442bdowbNgwFixYwI4dO+o7L+AYaouMjOTKK6/k119/bZDPEEKI5s7rqGM39viAoTonaZnifOwkqQjHE1nS3mzUqcfn7bff5k9/+hPTpk0jMjKy2jZhYWHMmzfvosKdLTIyknfeeYf+/ftTXl7OBx98wIgRI9i8eTN9+/at9j3l5eWUl5c7nxcUyI2mhBAtn7Uoh7alvwOQH305/jrnaYlifWwkqih6cFQKn2akToXPqlWraN26NQZD5Q4jpRQpKSm0bt0ad3d3pk6dWi8hT+vUqROdOnVyPh88eDAJCQm89tprfPzxx9W+Z86cOTz99NP1mkMIIZqkbfOd/5lycAdtsBOvoull3Y0xebeOwVqmWB8bq0/3+Mi9fJqNOg11tWvXjqysrCrHc3JyaNOmzUWHqo0BAwYQHx9/ztdnz55Nfn6+85GSktKI6YQQQh/Wk3sAOOLVG6Omc5gWqrW3jUS7414+tkIpfJqLOvX4qHNsyFZUVISHh8dFBaqtXbt2nXO4DcBsNmM2mxsxkRBC6MxWQeuSfQAYo3rqHKbl8ndXZJkcE5qV3MSw2ahV4TNr1iwANE3jiSeewMvLy/mazWZj8+bN9O7du8bnKyoqqtRbk5SUxK5duwgKCqJ169bMnj2bEydOsHDhQgDmzp1LmzZt6NatG2VlZXzwwQesWbOGlStX1uYyhBCiRcs/cQh/KjiuQugVG87R9Gy9I7VYyjsMSsFkKYKKEnD3uvCbhK5qVfjs3LkTcPT47NmzB3d3d+dr7u7u9OrVi4cffrjG59u2bRsjR450Pj9dWE2dOpUFCxaQmppKcnKy8/WKigr++te/cuLECby8vOjZsyc//fRTpXMIIYSryz22D39gu6kv47wUR/UO1IJF+LqRVhJIhJbrmOfjHqd3JHEBtSp81q5dC8D06dP5z3/+g5+f30V9+IgRI845bAawYMGCSs8fffRRHn300Yv6TCGEaNHsNkILHPN7LGE9dA7T8sX6WElMjSTCmOtY2RUYp3ckcQF1mtw8f/78iy56hBBC1L+KzAS8VTFZyo/ObWL1jtPixfnYztilXeb5NAc17vGZMGECCxYswM/PjwkTJpy37dKlSy86mBBCiNrLOLqPaGCj1ofrAu16x2nxWnvb+O70Lu1yL59mocaFj7+/P5qmOf9bCCFEE6Ps+GY7blqYF9QLTZaxN7jW3n/0+NiLMi5uA0zRKGpc+MyfP7/a/xZCCNE0qLwU/O25FCkPYmPbAeeeQynqR7BZkWY4tUdXcSYoO2hS/jRldfrulJaWUlJS4nx+7Ngx5s6dK8vKhRBCR1lH9wLws+rFwAgpehqDpoHBK4hyZcKgrFCaq3ckcQF1KnzGjRvnvLdOXl4eAwYM4JVXXmHcuHG8/fbb9RpQCCFEzRgzHKu5jvv2xsOocxgXEuOjSFFhjifFVXc1EE1LnQqfHTt2MGzYMAC+/PJLIiIiOHbsGAsXLuS///1vvQYUQghRA5mHCLKkUa5MBLfurHcal9La285RdWq4q0QKn6auToVPSUkJvr6+AKxcuZIJEyZgMBi49NJLOXbsWL0GFEIIcWGFOx2raTfau3FZK5lj0pha+9g46tysVAqfpq5O/3e0b9+er776ipSUFH788UeuuuoqADIyMuT+PkIIoQPrHkfhc8CzLyEeMr+nMbX2PrPwkXv5NHV1KnyeeOIJHn74YeLi4hg4cCCDBg0CHL0/ffr0qdeAQgghLiDzMIGFh7EoI+6tZFPSxhbrY+OYOr1ZqfT4NHV12p39pptuYujQoaSmptKrVy/n8SuuuIIbbrih3sIJIYSobPHm5CrHuhz6kD7Az/Ye+GvFbE6qaPxgLizKy0bymXN8ZEl7k1anwgcgIiKCiIiISscGDBhw0YGEEELUglK0OvEDABu0S7jaQ4qexuZuADwCsdiNuNmtUJYPnoF6xxLnUKfCp7i4mBdeeIHVq1eTkZGB3V75tuiJiYn1Ek4IIcT5+RcdIaz8KOXKRL5/RzStUO9ILqmVD6Tkh9JWS3NMcJbCp8mqU+EzY8YM1q9fz+23305kZKRzKwshhBCNK+bkCgDW23vRK9CqcxrXFetj41heOG05VfiEdNA7kjiHOhU+P/zwA8uXL2fIkCH1nUcIIURNnTHM9ZNhCDf4lOocyHXFOFd27YYSWdnVlNVp9lVgYCBBQUH1nUUIIUQtBBYcILj8OKXKnbSIkRik8103sXIvn2ajToXPs88+yxNPPFFpvy4hhBCN6/Qw1xp7bzrERFygtWhIrb3/WNIu9/Jp2uo01PXKK6+QkJBAeHg4cXFxuLm5VXp9x44d9RJOCCHEOShFq1OFzyptCP2CveG4zplcmOMmhn/cy0dTyrGDqWhy6lT4jB8/vp5jCCGEqI3g/D0EVqRSpDzIjLwMo4xz6crfXVHkFoxVGTDZLVBeAB7+escS1ahT4fPkk0/Wdw4hhBC10PpUb89P9r50bBWucxoBEOWtcaI4hFgtwzHPRwqfJqnOt5bMy8vjgw8+YPbs2eTk5ACOIa4TJ07UWzghhBDVUHbnMNdKhtA21EfnQAJknk9zUacen99//51Ro0bh7+/P0aNHufPOOwkKCmLp0qUkJyezcOHC+s4phBDilPCcrfhZMslXXmRFDJVhribi9Gall7HHsXWFaJLq1OMza9Yspk2bxpEjR/Dw8HAev+aaa9iwYUO9hRNCCFFV3PGvAfjWNojO0SE6pxGnnblZqSxpb7rqVPhs3bqVu+++u8rxVq1akZaWdtGhhBBCVM9kLSEm7ScAvmU47WWYq8mI8baRJPfyafLqVPiYzWYKCgqqHD98+DChoaEXHUoIIUT1otNX424vJdEeQXlEP0xG2QW8qTizx0cVZ4JSOicS1anT/zHXX389zzzzDBaLBQBN00hOTuaxxx7jxhtvrNeAQggh/tDmhGOYa6ltGN1byaqhpiTC004aodiVhmYrh4oivSOJatSp8HnllVcoKioiNDSU0tJShg8fTvv27fH19eVf//pXfWcUQggBkH+ciOwtACzXhtEh3FfnQOJMRg3CvQ2cJNhxQFZ2NUl1WtXl7+/PqlWr+PXXX9m9ezdFRUX07duXUaNG1Xc+IYQQp/3+ORqKTbauBEa1x02GuZqcGG8bR3PCiTZmyTyfJqrWhY/dbmfBggUsXbqUo0ePomkabdq0ISIiAqUUmtyiWwgh6p9SqF2fogFL7UPpFROgdyJRjVgfG8eyIxjKPlnS3kTV6p8LSimuv/56ZsyYwYkTJ+jRowfdunXj2LFjTJs2jRtuuKGhcgohhGs7uQMt+zClyp2fTYNpJ6u5mqQz9+ySoa6mqVY9PgsWLGDDhg2sXr2akSNHVnptzZo1jB8/noULFzJlypR6DSmEEC5n2/zKz/d8CcAK+yX0DSyiQ8qXOoQSF9Lax8ZmWdLepNWqx+fTTz/lb3/7W5WiB+Dyyy/n8ccfZ9GiRfUWTgghBGC3ok7uAByruYYGVb2diGgaTt+9GXD0+MiS9ianVoXP77//ztVXX33O18eMGcPu3bsvOpQQQogzpO9Hs5SQpgJJdOtAW68yvROJc2jtbSNZhTmeWMugJEffQKKKWhU+OTk5hIefexfg8PBwcnNzLzqUEEKIMyRvAmCZbSiDg4qQNSRNl5cJfM1GTqogx4GcRH0DiSpqNcfHZrNhMp37LUajEavVetGhhBBCnFKSg8o8iAYssY1klgxzNRmbk6rvzQky+XCsIoIoYw4bt27h6Mk/OgwmD2zdWPHEOdSq8FFKMW3aNMxmc7Wvl5eX10soIYQQp6T8hobiZ1t3ggKDifDI0zuRuIBws4Wj5eEMYj++JSl6xxFnqVXhM3Xq1Au2kRVdQghRT+w2SP4NgE9tlzO+tcztaQ7CzRbnnl0+Jck6pxFnq1XhM3/+/As3EkIIUT/S90F5AZnKjzWqH89E55GQqncocSFh5goST63s8i2WHp+mRu53LoQQTVXyRgC+sI3g0nA7IR6yNLo5CDdbnEvapcen6ZHCRwghmqKSbFTmIcAxqXlCrAxzNRfh5gqOnVrS7mHJw82Sr3MicSYpfIQQoilKdkxq3mDrQYFbKKOjZPFIcxFgsmHT3ElXAYAMdzU1UvgIIURTY7NAymYAFtuuYELrMsxGnTOJGtM0CDtjuMtXhruaFCl8hBCiqTn0/alJzf78ZO/LrW1K9U4kainCXMExu2NllxQ+TYsUPkII0dSc2qD0c9twegYpOvrbdA4kasvR43N6SbsMdTUlUvgIIURTknkIEtdiR+NT2+XS29NMOSY4n17SLj0+TYkUPkII0ZRsfgeAVbZ+5BpDGBsjk5qbo/BKPT5S+DQlUvgIIURTUZIDuz4F4EPrGK6PKcfbJPfuaY7OvHuzZ0UOJkuRzonEaVL4CCFEU7HjI7CWsl/FsVl1ZqIMczVboe4WivEkU/kByJ5dTYgUPkII0RTYLLDlfQA+tI6ms7+NXoFWnUOJunIzKILcrH/M8yk5pnMicZoUPkII0RQc+AYKTpCrBfCtbRAT40rRNL1DiYtRebNS6fFpKqTwEUKIpuC3twH4yHI5yuTBDbJFRbMXbq7g6Ol7+cjKriZDCh8hhNDb8W1wfCtWzY1F1lFc3yuKAHeZ1NzchZktZwx1SeHTVEjhI4QQejvV2/ONbRCZBDB1UJy+eUS9CDdXOJe0S+HTdEjhI4QQeso/Afu/AmCe5Wr6tA6gR7S/vplEvQg/Y78uz/IsTNYSnRMJkMJHCCH09dtbYLeyXevGPhXHtMFxeicS9STcXEEB3mQrX0BuZNhUSOEjhBB6KcmBbR8C8N/yawnxMTOme6TOoUR98THaMZsMzpVdvsWypL0p0LXw2bBhA9dddx1RUVFomsZXX311wfesW7eOvn37Yjabad++PQsWLGjwnEII0SA2vwOWEhKM7Vhv78n/Xdoad5P8e7Sl0DQI9nYnyTnBWZa0NwW6/h9WXFxMr169ePPNN2vUPikpibFjxzJy5Eh27drFQw89xIwZM/jxxx8bOKkQQtSz8kLnvlz/Lr0Wd5OR2y+N1TmUqG+B3u4ctZ/erFR6fJoCk54fPmbMGMaMGVPj9u+88w5t2rThlVdeAaBLly788ssvvPbaa4wePbqhYgohRP3bOg/K8kl1i+HHskuY2L8VwT5mvVOJehbs7S5L2puYZtWnumnTJkaNGlXp2OjRo9m0adM531NeXk5BQUGlhxBC6MpSCpscPd3/LhmLHQN3DG2rcyjREAIrDXVJ4dMUNKvCJy0tjfDw8ErHwsPDKSgooLS0+s385syZg7+/v/MRExPTGFGFEOLcdn4CxRnkuoXztW0wl3cOo32Yj96pRAMI9jb/sUt7eZZjiFPoStehrsYwe/ZsZs2a5XxeUFAgxY8QovFtm+/4026Dtf8C4L+lV2PFxF3hh2DbXh3DiYYS5O1OAd7kKF+CtELISYTIXnrHcmnNqvCJiIggPT290rH09HT8/Pzw9PSs9j1msxmzWcbNhRBNxIntUJpLscGXxdaR9A+uYGCIRe9UooH4e7ph0CBJRUjh00Q0q6GuQYMGsXr16krHVq1axaBBg3RKJIQQtWC3QfwqAN61XEM57tzbuUR2YW/BjAaNAC935x2cyU7QN5DQt/ApKipi165d7Nq1C3AsV9+1axfJyY4JYLNnz2bKlCnO9vfccw+JiYk8+uijHDx4kLfeeovPP/+cv/zlL3rEF0KI2jm+FYozKTH4MM9yFZ39LYyMqNA7lWhgQd7uzl3ayUnUN4zQt/DZtm0bffr0oU+fPgDMmjWLPn368MQTTwCQmprqLIIA2rRpw/Lly1m1ahW9evXilVde4YMPPpCl7EKIps9mgcMrAHjTOo5iPKW3x0UEndnjI4WP7nSd4zNixAiUUud8vbq7Mo8YMYKdO3c2YCohhGgAxzZCWR6FxkA+KLuS9r5WxkaX651KNIKgM5a0y1CX/prV5GYhhGiWyoucc3tesUygHHce7JqPUYPNSTk6hxMNLcjbnV9PFz7FGVBWAB5++oZyYc1qcrMQQjRLm9+GiiJyTWF8UjGcjn7S2+NKgrzdKcSLbHWq2MlN0jeQi5PCRwghGlJJDvz6OgDPl9+EFRN/6VqMQeb2uIxgH3cAjp66kaEMd+lLCh8hhGhIv/4HyvNJd4vmS8tguvhbGN1KentcidlkxNfDdMYEZyl89CSFjxBCNJSCVNj8LgD/LJmIwsDjPaS3xxUFe7uTZD9d+MhQl56k8BFCiIay+hmwlpLk2Z2Vtr4MCavgsnC5b48rCvb5Y88uGerSlxQ+QgjREI5vh92LAfhL/i2AxmPdi+S+PS4q5Mwl7TLUpSspfIQQor7Z7fDDowCs9xzFLnt7ro0uo2eQVedgQi+OHp/TS9ozHUvahS6k8BFCiPq253M4sQ2ryYuHc2/A3WTg8R5FeqcSOgr2cSxpz+HUkna5g7NupPARQoj6VF4Eq54EYJ52I5kEctewtkR723UOJvQU7G0GINEuw116k8JHCCHq0y+vQlEa+R7RvFI4ijBfM38e0U7vVEJn7iYDfpWWtEuPj16k8BFCiPqSkwQb3wDgb8W3UoEbj13dGW+z7A4kHPN8nLu0Z0vhoxcpfIQQoj4oBStmg62cfR59WW7pw8A2QUzo20rvZKKJCPZ2l5sYNgFS+AghRH3YtwwO/4Bdc+Oh/FtxMxr41w3d0WT9ujgl2Md8xpJ26fHRixQ+QghxsUpynMvX5xtu4IiK5u7L2tE+zFfnYKIpCfZ2/+MmhrKkXTdS+AghxMX68e9QnEmmRxwvFo+ldZAX913eXu9UookJ8TFThBfZ+DsOyHCXLmTGnRBCXIyENbB7MQqNewqmUYEbl3cOY+mOE5WatUvO0SmgaCqCvB27tCfYIwg25Du2rojqo3Mq1yM9PkIIUVcVxfDtgwAs97yO7faO9GjlT8dwGeISVbmbDET6e5Bkj3QcyDqibyAXJYWPEELU1Zp/QV4yhR4RPJo7HrPJwNgekXqnEk1Y21BvEtWpvyPZUvjoQQofIYSoi2MbYfPbAMwqnkoJHlzTPRI/Tzedg4mmrG2Izx+Fj/T46EIKHyGEqK3SPFh6Fyg7azxGscrSi2EdQugfF6h3MtHEVe7xSXDc/0k0Kil8hBCiNpSC7x6C/BTyPaO5P28SvmYTL97YU+7ZIy6obagPySocGwawFENhqt6RXI4UPkIIURu7FsO+ZSiDiTsK76EYT/5xbReiAjz1TiaagbYh3lgwkazCHAdkuKvRSeEjhBA1lZ0A3z8CwCeet7HN2pbhHUO5pX+MzsFEc9EqwBOzyUCiXSY460UKHyGEqAlrBfzvDrAUc8K/H09mX4mv2cQLN/aQIS5RYwaDRpuQM+b5ZMXrG8gFSeEjhBA1seYZOLkTq9mfW7OmY8fAP6/tSqS/DHGJ2pEl7fqSOzcLIcQ5LN6cDEDr1BUM3fU6AH+z3kWKLYiukX5YbHZnGyFqqm2ID1vtUY4nMsen0UmPjxBCnEdAwSEu3fMEAF/73MLnxX3wNZu4oU8rGeISdVKpxycvGazl+gZyMVL4CCHEObhX5DNsx0OYbKXE+w7gL1nXA3Bjv2i8zdJhLuqmbagPmfhThBegICdR70guRQofIYSojt3G4N2P4Vt6nAKPKKYW3I0dA4PaBcteXOKitA31BjTi7RGOAzLc1aik8BFCiOqs/RdRWb9iNXgw2/QYJ8o9CfM1c3W3CL2TiWbOz8ONUF8zier0PJ/D+gZyMdJXK4QQZ9u1GH5+BYDF4Q+zPCkUo0Fj4iUxuBkNtEv+QueAorlrG+JNQnIUGJEen0YmPT5CCHGmxHXwzf0AbIueyrPJ3QC4qmu4LF0X9aZ9mA/xqpXjSeZBfcO4GCl8hBDitPR98NntYLdi7zaBhzKvx2JTtAv1Zkj7EL3TiRakQ5gP8eqMJe12u76BXIgMdQkhxLb5jh3Xf50L5QUQ1I7XCkdxPL8cb6ONv0bsJjhlu94pRQvSPsyXYyocK0ZMlmIoOAEBsvVJY5AeHyGEsJTB1vegLA98wtgUezdvHPYH4K7YNILdrfrmEy1Oh3AfrJhIUqdXdh3SN5ALkR4fIYRrs5TCtg+g4CSYfcnudQ/3/xKJQmNUSC6XBhbqnVA0Y1UmwhuDAAhT4OsWwhF7KzoYT8COjyEvBfpP1yGla5HCRwjhEqrbWsJgt3DZjgeIyo7HZnBnX9RE/rE1mqxyIzEeZUyNydAhqXAFmgYdfG3EF5ya51OUrm8gFyKFjxCiZdk2v9rD7ZJzKh9Qdtof/x/BBQewaSYOtZ7EkoLu/F7gg7tm58G2J3E3qEYILFxVBz8r8XmnVnZJ4dNoZI6PEML1KEXbk98SXHAAu2bkSOuJbFed+OxEKADTWqcT41mhc0jR0nXws5Jwekl7YRooKbQbgxQ+QgjXohSxaSsIzduNQiM++kZOenTgv0lR2NAYFFjA5cH5eqcULqC9n40EFYkdDSwlUFGkdySXIIWPEMJ1nCp6InK2ApDYahw5vp15LzmCzAp3wtwruCs2Ddl0XTSGDn5WyjBzXJ26R5QMdzUKKXyEEK5BKeJSvyciZysKSIy6jqyAnqzJ8ue3XD+MKB5sexIvo9xITjSOKE873iY78XaZ59OYpPARQrR8yk6bk98RnrvdUfS0GkdmYB+SS83MTwkHYFKrTNp7l+mbU7gUTYP2vjaOnDnPRzQ4KXyEEC2bssPvnxGWtxOFRkKr8WQF9KLEZuDVhFZYlIHefkWMDc+58LmEqGft/ax/7NklPT6NQgofIUTLZbfCzo8hZbOj6Im+geyAnigFbx2NJLXcnWA3CzPbpGKQeT1CBx38bBy2RzueFKbqG8ZFSOEjhGiZbBWw9QM4uRM0I/HRN5Lt3x2A79KD2Jrni1FTzGp3Aj+TTeewwlV18rNyWJ0qfMoLoThL30AuQAofIUTLU1ECv70NmQfB6A6XzCDHvysA+ws9WXz6fj0x6TKvR+iqs7+VUjw4psIcBzL26xvIBUjhI4RoWcoKYNMbkJsEbp4w8M8Q1gWAXIuR/yS2wo7GsKB8rgzJ0zercHkRnnb83ewcsp/amT1dCp+GJoWPEKLlyDoCG/8DhSfB7AeD7oegNgBY7BpzE1uRZzUR41HGjNZyvx6hP01z9PocUqcKH+nxaXBS+AghWobk32DelVCSDV7BMPgB8HNsAKkUfJAczsEiLzwNNma1O4GHUbYHEE1DlwDrHz0+Uvg0OCl8hBDN376v4KProTQXAlrDkIfAO8T58gdHPFmXHYCG4qG2J4nysOgWVYizVe7xOSB7djUw2Z1dCNF8KQWb3oSV/wAUdBoLbYc7JjSfsibVned/9wFgSnQGvf2LdQorRPU6+1tJUhFYlBG3iiLIS4bAWL1jtVjS4yOEaJ5sFvjuIVj5d0DBJXfCxI8rFT2H8408sNkPhcYVIXmMCcvVLa4Q59LRz4oNI/HKMTQrw10NSwofIUTzU5IDH98A2xcAGlz1HFzzMhiMziZZZRp3bAygyGpgYEgFf4qRycyiafIyQZyPTSY4NxIpfIQQzUvGQXh/JBz9Gdx9YdISGHw/Z1Y1BRaNqb8EkFJspLW3jXcG5WOSn3aiCevsb+WQvbXjiSxpb1BN4kfBm2++SVxcHB4eHgwcOJAtW7acs+2CBQvQNK3Sw8PDoxHTCiF0c/hHx8qt3KMQEAszVkGnqys1KbPBjF/92ZfnRrDZzvyheQSaZbKoaNocE5xP3cFZenwalO6Fz2effcasWbN48skn2bFjB7169WL06NFkZGSc8z1+fn6kpqY6H8eOHWvExEKIRme3w9o5sHgilBdA7BC4c63zxoSnWWx2Zv7mz5Ysd3xNdj4amkc7X9mOQjR9nf2tHDzd45N1GKzl+gZqwXRf1fXqq69y5513Mn36dADeeecdli9fzocffsjjjz9e7Xs0TSMiIqIxYwoh9LLxDdi16I9/BccOgW43wIFvKjWzK3h0qx+rUz0wGxQfDMmne6BVh8BC1F4XfysnCSZPeRNgL3Ysa4/qrXesFknXHp+Kigq2b9/OqFGjnMcMBgOjRo1i06ZN53xfUVERsbGxxMTEMG7cOPbt23fOtuXl5RQUFFR6CCGaidTf4ZdXHUWPwQ16T4YeN4Oh8r/ZlIKndvmwLNkDo6Z469J8BobKvXpE8xHjbcfPTbHPHuc4kPa7rnlaMl0Ln6ysLGw2G+Hh4ZWOh4eHk5aWVu17OnXqxIcffsjXX3/NJ598gt1uZ/DgwRw/frza9nPmzMHf39/5iImJqffrEEI0gF2fVr4T85AHIXpAlWZ2BX/f6cvCBC8AXrmkgCuiKho7rRAXRdOge4CVfSrOcSBVCp+Govscn9oaNGgQU6ZMoXfv3gwfPpylS5cSGhrKu+++W2372bNnk5+f73ykpKQ0cmIhRK1YK2D5X+Gre8BaBqFdYOgs8I+u2tQOj27zZXGiJxqKl/sXML61zI0QzVOPQAt7T/f4pO7WNUtLpuscn5CQEIxGI+np6ZWOp6en13gOj5ubG3369CE+Pr7a181mM2az+aKzCiEaQcFJ+HwKHN/qeD78cfAJA63qv9FKrXD/Zn9+SjVj1BSvXlLAOCl6RDPWPdDKT6d7fNL3gt1W6d5Uon7o2uPj7u5Ov379WL16tfOY3W5n9erVDBo0qEbnsNls7Nmzh8jIyIaKKYRoDEk/w7uXOYoeD3+Y/DmMnF1t0ZNTrjF5QyA/pZoxGxxzeqToEc1dj0ArSSqSEmUGSwlkJ+gdqUXSfVXXrFmzmDp1Kv3792fAgAHMnTuX4uJi5yqvKVOm0KpVK+bMmQPAM888w6WXXkr79u3Jy8vj5Zdf5tixY8yYMUPPyxBC1JVSsOkNWPUkKBuE94CJCyGobbXND+QZmbExgBMlRvzd7Mwbkk//EJnILJq/WG8b3h7uHLTH0FeLd0xwDu2od6wWR/fCZ+LEiWRmZvLEE0+QlpZG7969WbFihXPCc3JyMgbDH//iy83N5c477yQtLY3AwED69evHxo0b6dq1q16XIISoqW3zKz+3lsPuTyF1l+N5q/7Q8xZIXO94nGX5cTOPbPWlxGYgzsfKB4Pzae8n9+kRLYOmQfcof/Ylx9HXEO+Y59PjJr1jtTiaUsqlbmlaUFCAv78/+fn5+Pn56R1HCNdyZuFTnAXb5kFhqmM4q9sNEDuU6jbUKrPBc7t9+CTRsXJrWFgF0yKP4mOyN1ZyIerFwDZB5339+YxLKfj1A15w+wDajoApXzdOsGagvn5/697jI4RwQZmHYMdHjnkMZl/oN/2cQ1v78kz8dasfB/MdP67u7VTMrG7FbD8mRY9oebq38ud958qu3x1DwbK7br2SwkcI0XiUgsS1cOBbQEFAa+j3J/AMqNK0wg7vHPTivwe8sSqNYLOdVy8pYHiE3KNHtFw9WvlzWEVjUUbcSnMg/zgEyP3n6pMUPkKIxlFRAjs/hpM7HM9jBkL3m8DoVqXpb5lu/GOHL/GFjh9Ro6PK+FffQkI8XGpkXrig2CAv3D28OGSPobt21PH/ixQ+9UoKHyFE3Zw9Ufl8SnJg2weO+/ScZz7P0SIjL+/1ZvlxDwCCzXb+2auQcTHl0tsvXILBoNE7JoBdSe3objgKx7dB13F6x2pRpPARQjSsvGTY+oFjV3V3H8d8nuB2lZpklWn894A3ixM9sSoNDcWktmU81r0If3fp5RGupU/rQHYntuP/WA0ntusdp8WRwkcI0XDS9zkmMdsqwDcSBtwFnoHOl/MqNOYf8eKDI54UWx23rRgRUc5j3YvoEiDL1IVr6hcbyLP29o4nJ3eCzQpG+XVdX+QrKYSod5uTcgjL2Upc6go0FPnebTnS6mZsaQrIIbvCxPL0IH7KCqDc7ih4egZaeLxHEYPD5GaEwrX1jgkgkSgKlCd+lhLIPAARPfSO1WJI4SOEqF/KTkzaKqKyNwGQEdCbo1FjUZqRk2XufJMWxIYcf2zKMWknzrOM8ZHZPNDHDYPM4xECf0832of58XtOW4Ya9znm+UjhU2+k8BFC1B9bBexaRFS2Y2fplLCRnAwZSkKJJ1+nBbElzxeFo7rp6lPCuIhsevkVo2mw9aiOuYVoYvrFBrIruz1D2QcntkH/6XpHajGk8BFC1I+KItg6D3KTsGsGEqLGscEwkK+OBLOn0NvZrL9/IeMisunoU6ZjWCGatj6tA1m57dQ8n+Mywbk+SeEjhLh4RZmw5V0oyUKZPPkmcApvpV3C4WLHFhMGFEOCCrg+IpvWnnIDQiEupF9sIC/bHasfVeZBtLIC8JBtluqDFD5CiIuTk+RYrm4ppsQ9mFk8zIqUWADcNDuXh+RzbXgOYWaZtCxETbUN8cbiFcpxWwjRWpZjdVfb4XrHahGk8BFC1N3JXahdn6DZrRzS2nBbwaNk4Y/ZYOeq0FyuDc8hwE2WpQtRW5qm0bd1IDsT2hNtzIKUzVL41BOD3gGEEM2QUtjj18COBWh2Kytt/Rhf+k/K3Xx5oEsxb/aI5/+iM6XoEeIiDGgTxGZ7F8eTo7/oG6YFkR4fIcQFLd6c/McTm4UOO57lkqxvAJhvHc1cNYlxUfmMDsvFyyi7pgtRHwa3C+avpwoflbIFzVoBJnedUzV/UvgIIWosIyuTgTse5RLbNuxK40XbZHJCB/Cf8KN4SsEjRL3qFuVPmjmWbOVLsLUQUndBzAC9YzV7MtQlhLigglILa7fsZPTmaQyybaNMufGB1wz6d+3ITVHZUvQI0QCMBo1L24aw1d7ZcUCGu+qFFD5CiHMqs9h4c208q376gX9lPUAXQzK5WiDf9f2AXm0j8ZM5PEI0qMHtgtl8uvA5tlHfMC2EDHUJIapQSrF8Typzvj9Iz4J1fOL2Np5aBRme7dk48C0qPCMhOUHvmEK0eIPbhfDF6Xk+yb+hyYalF02+ekKISvaeyOeZb/ez7WgWD5r+x4PuywA4ETqMX3u9hNXNR+eEQjRfm5Nyzvt6gi250vNJA2LI9GxHgc0Lv4pCSPsdWvVtyIgtnhQ+QggAMgrLeHnFIb7ccRx/VchH5rcYpjn23DoUexs7Oj+MMsiPDCEak6ZpDGwfxpYDnRhl3AnHfpXC5yLJTzEhXFyZxca8X5J4a208xRU2umlJLPR9nWBLGpg84br/sL1skN4xhXBZg9uF8Nu+ro7CJ3E9DL5f70jNmkxuFsJF2e2Kr3ed4IpX1vPyj4corrDySMgmvvV62lH0BLaBGT9Br4l6RxXCpQ3vFMp6ey8A1NGfwVKqc6LmTXp8hHBBmxKyef77A+w5kQ9AV99S5gUtJDJ9vaNBx6vhhnfBM0C/kEIIAFoFeGIK78LJ3CCirDlw9FfoMErvWM2W9PgI4ULiMwqZ8dFWJr3/G3tO5ONjNvFWvxMsNz3qKHqM7nDVc3Drp1L0CNGEXNElnHU2R68P8av0DdPMSY+PEC4go6CM/645wqdbUrDZFQYNLm9t5BHtYzrt+xaAXN9ObOw1h3xjB9h6XOfEQogzXdEljHfW92Yya1FHfkIbo3ei5ksKHyFasLT8Mt5Zn8DiLclUWB13V+4W4c1fgzcxJPktzJYC7Bg40PZP7OlwL3aDm86JhRDV6RUdwEHPvlisRtxy4iEnCYLa6B2rWZLCR4gWKHXDR7x9yIslSZ5U2DUA+gdX8HTsXuISF+GdkApAsUc4RyOvocgzSooeIZowg0FjYJdYtv/ekUu1AxD/Ewy4U+9YzZIUPkI0c2funJ6SU8LGhCz2Hg/GhqPg6exTwh0hBxhZ8iMh+/cCYDWYOR42kvSg/qDJVD8hmoMruoSzbmcvLjUcQB1ZhSaFT51I4SNEM2e129l/soCNCdkk55ScOqrR1aeEaWGHGFm8kpC039FQAGQG9CI5fBRWk7d+oYUQtTa0fQivG/oBS1AJa9HKCsDDT+9YzY4UPkI0Q0op9p4o4H87jvP5thRKKhybhRo1jZ7R/sxw/5FLijcQcnI3Bhxze3J9O3I8dDglnpF6RhdC1JG32USbLv05crAVHTgBh36Q+2zVgRQ+QjQjCZlF/Lgvja92nuBwepHzuK/ZxCVxgdwQmEDfE2/R6sTPztfyfNpxPHQExV6t9IgshKhH43q3Yvn+gTxkWIratxRNCp9ak8JHiCbMZlfsSslj1f50Vu5PIzGz2Pma2WTgqm4RRJgtXG79mU4pnxN07CAACsfy9NSQwRR5xeiUXghR3y7rGMrbbkN5SC1Fxa9BK82Te27VkhQ+QjQhSimSsor5NT6LX+Oz2ZSYTX6pxfm6m1FjULsQxnQL57qQVHz2zsPy+5e42Ry3sLcaPUlsNY4S92DKzcG1+ux2yV/U67UIIeqfu8lA554DOLQzmk4ch0PfQ+/JesdqVqTwEUJHFptjYvL2Y7lsT85l+9Fc0grKKrXxNZsYEVbEVVHlDA/KxS/jc/jlNyh0LEl3A0rdg8kI7ENmQB9sJk8drkQI0VjG92nF8m2X0snwJbY9SzFK4VMrUvgI0YhyiivYcbrIOZbL78fzKLPYK7VxNxroFxvIkPbBDG4fQs8wd0wr/wYntsOe/WB3TGTG4AaRvdjv1o1Cr9agaTpckRCisfVrHchr3pdBxZdoiWuhKBN8QvWO1WxI4SNELZx5z5wLsdkVGYVlHM8p5VhOCck5xWQVVVRp5+lmpHWQF7HBXrQO8uKvV3XC06ggcR1sfwUOfAcVhX+8wTcKWg+CVv3A3YvCpJx6uDIhRHNhMGgMvnQwu9a1o7chAXYtgqEP6R2r2ZDCR4h6YFeKzMJyTuSVcjy3lBO5JaTml2G1qyptQ33NxAY5ipzWwV6E+JgxaBoGWwXhOZvx/PE9OPAtlGT98SbPQIjq6yh2/KIa8cqEEE3RrQNa8+81V9LbkED55nmYBz8ABrkZaU1I4SP0sW3+BZtsPqsnI6H1zfUaYfLA1rV+T4XVTn6phezicjIKyilNO0RKqTsnysyU26v+0PEy2mjrVUYH71I6+ZTSwbuU9LY3Ol/3KM8i4uQqojPWEpn5K262EudrZW6BJEeO5mjUNfgXHHEMZWUD2dLDI4SrC/Exo7rfQMGBhfgVJkPiWmh/hd6xmgUpfESzoZSiuMJGdlE5uSUVFJfbKLXYKKmwUVphxWJTKKVQgFKgUI6eFE3DaHA8HP8NBk3jSEYhbkYDJoOGyWjA7dSfdqUoKrdSVGalsMxCQZmVtPwyUvNLyS2xnJXK3/lfZoOdNl5ltPMqo513KW29ygg3WzCcMfXGaCvDmLGBiOzfCM/+jcDCI5XOVmIO40TYCI6HX05a8ADUqf2z/AvjG+irKoRoriYP6cz/9g5juulHyn/7ALMUPjUihY9okix2SCw2c6TYkyPFnpwoc+fE7v2UW+0XfnMNnd2jVFPuJgNBXu6E+prpouKJ9ignxrOCCHNFpSJHs1vxLMvCu/QkPqXH8Sk9gWd5JmdPQc7x68LJ0KEcD7ucHP+usneWEKJGesUE8EHoeKbn/ohb/I+QfwL85UalFyKFj2gUZ08KbpdcueiwKkgpsJOVV4CtOBcsJfhRRE+tkBEU4qmVYzcYsLsZ0IxGjEYjZUZfSk1+lJv8KXcPoNQtgCK3YAqNQRS5BVNq9MOGhk0pbHaF3a6wKcekY7tSdI7wxWpXWGx2rDaF1W7HYlMYNPAxu+HjYcLXbMLHw0S4n5moAE9+PZKNh5sB7dQKqnbHfsPNWoTZkotHfh7mihy8yjLwLM/EoyLHuT/WmQo9o0kPGUha8KWkBw+k3D2w4b7wQogWbczlI9n8RWcGGg5StmEuHte9rHekJk8KH6ELg92CVnCC8twT+JadINKWyhAt/48GF/qbaTv1qLpIysmumShzD6LUHEKZOeTUn8GUeYZQ7hbIkDatwc0T3LzA5OF4k1Kg7KCsYC2F8kLHo7AAMvMhPtFR1JRn4VmRjU/JcQzKes4MVqMHxR4RFHlGU+TViiLPaA61nVbDr5IQQpzf1d0i+OePtzGw6J8Yd3wEI/4KvhF6x2rSpPARjUMpAgsOEnryJwLTN9G6dD9u2P54/dT4T6YWRL5bOEYPb9zcPbAZvbCaPLEZ3EkPvhQNO5qyY1BW3CyFmC35uFvyMFfkYa7IxbM8C4/yLDwseRiUFa/yDLzKM6rPtLv2l9G5uktDo8LNjzL3QMrdAik1h1LiEUqpOQyLyafK/XXkDslCiPpiMGhcMfYWtn+6kH6GI5SsfQWv66XX53yk8BENx26D5N8o/v0rrvr9G0KsaZVePqGC2at1otCrNT7+gQT6+YHJfM7TpUReVeOPNtgtdEpaiJu1qNLD3eL402Qrxc9dgc0CtgrHnwDuPo45NgYDGM3g4Qdm31MPP/YVmCkzhzp6j9yDCcndQYWbP0oz1ulLJIQQF2tk53CeCZxKv/x/4LZzAYx8GHzD9Y7VZEnhI+pfdgLWHYuw7FiEZ2ka3oA3UKrcWW/vxW6PS3DzCaJDkDthHlYCanja2vaUVLj5UeHmd87XB7YJqnqw//TznnP3WXOVfIuP1iqTEELUN03TGH39JHZ8tIC+hniyvn2CkMnv6h2ryZLCR9SP8iLU/q8o+u0jfNO3YMLxlytfefGTvR/bPYdQEjOcjtFhxHi5nypizj03RgghRM1d2i6E19s8QN9jDxByeAmWxCm4tR2id6wmSQofUVVNby6oFD4lKfjl/E5YwT7MlOML2JXGz/Ye/KANRQW1Y3BwCeM8LMAmyLrQmYUQQtTF/90yia/+/TXj1Wryv7yPkFmbweSud6wmRwofUXuleYRm/EJAzu8E2f6oZJLs4SyzX0aybx96hBoY51OC4cyVWkIIIRpMoLc77mOeJXv5FkJKEjm+fA7R457UO1aTI4WPqBmbFXvaHgoSt+KXf4C2p+5PU6zMLLddylbzAELDwxgYWMQQY5nOYWum2hsYJr1y3ve0a6AsQghRH8Zc0pWPdtzPtLTnidw5l6w2lxLSc7TesZoUKXzE+eUfpzBxC6bU7Xjai50TkTfbO7OSIZQFdeLSkHJuMluAwvOcSAghREPTNI1bps/ix1e2MLriJ9yXzaA0Yj2eYW31jtZkSOEjqiovpCR5B2VHtxJUfhzfU4dTVRDfqqHkhV1KmJfGlT6laFqRrlGFEEJU5mV2o+uM99n31ii6qQSOvXcDwfeuwCcoUu9oTYIUPsKhopiKfd+R99snBKf/ghd2vIByZWKVvR/7fIfSqV1bbm9lwdNU932uhBBCNLyYsCD23rKQrM+vIdZ6lOQ3r8J29/f4h8XoHU13Uvi4MpuViiOrydr0CcHJqzCrUsJOvbTb3pZf3YfgE9uH0XEmrvW0A2fvTC6EEKKp6t61O4dv/grbFzfS2pZMyttXcmLcfLr2HqR3NF1J4eNqyoso2L+S3B1fE3RyLb62fKJOvXTUHs4KwzCyfLrQKcxMP88KII+jaXBUx8hCCCHqpmO3viS6fUvqp+OJUamUL7uO1b//hcETH8XT7KZ3PF0Y9A4A8OabbxIXF4eHhwcDBw5ky5Yt523/xRdf0LlzZzw8POjRowfff/99IyVthuw2So/tIOGbl4h/bQwVc+Lw+3o6sSlf4WvLJ1v58ilX80ToXD4ZsAyfq5/gyliN1p7n2f1TCCFEs9G2Y3d87v+Zvd6XYtYsXJH4EkdfGMjaHz6nzGK78AlaGN17fD777DNmzZrFO++8w8CBA5k7dy6jR4/m0KFDhIWFVWm/ceNGJk2axJw5c7j22mtZvHgx48ePZ8eOHXTv3l2HK2haynJTSTu8lbzE7bif3Errwl34UFxpGXayPZTtnoMpb3sVXQeNxn6yhM5nbaQphBCi5fANiqTbX39g77KXaLvnNbqoBLpsvpN9m1/gWOyNtBo6mR7t4jAYWv7vAk0ppfQMMHDgQC655BLeeOMNAOx2OzExMdx///08/vjjVdpPnDiR4uJivvvuO+exSy+9lN69e/POO+9c8PMKCgrw9/cnPz8fP79z7+PUVNmsVrKz0shNO0ZJegLWrCS0vKOYC48RVZZIMLlV3lOoPNlj7EJO6AC8uo2hZ59LCfH1cL6++Kz9p2T3cCGE0EdC65srPZ88sHW9f0ZZXhrxXz5Jp+Nf4nZq6yCb0tindeBkYH8MUX3wb9Ob0FZtiQoJxMOtaWzCXF+/v3Xt8amoqGD79u3Mnj3becxgMDBq1Cg2bdpU7Xs2bdrErFmzKh0bPXo0X331VUNGvaDc4go2JWZjsyvsSmEqzyc4Y6Njh3K7DZQNdepPnH/a0exWlLJhs1pR1goMlhIM1hKM1hKMtlJMtlJM1hK8bAX4qzz8VRFhmqJqX5iDXWkc0yJJ9ehAeVhPPDpcRtsegxkc4NOoXw8hhBBNk0dABN1nvIut8DkO/fQBPgc+p1VFIj05TM/cw5C7GPY52mYrX45qIRS6h1Lh5ofNzQfl5oVm9gGTJ5jMYHQjM6g/JX5tcTMaMBkNhPqaGd4xVN8LPQddC5+srCxsNhvh4eGVjoeHh3Pw4MFq35OWllZt+7S0tGrbl5eXU15e7nyen+/YQqGgoOBiolfxe3Iu93z4x9ykztoxvjQ/U6+fAVB86s8cfMk2RlDgEUGFdzSm4Fi8o7sT3rYnQYGBBFcaurKf93pLiivfeLC4pHnceVkIIVqas38e1/fvqsrMRF4xE66YSVZOCsd3rMCSvB2v3AOEViTjgQU3CoiiAMoTz3umJy1T+J9tuPN5z2h/+tx5ab2mPf21uNiBKt3n+DS0OXPm8PTTT1c5HhPTsPcySAH8G/QTCoGTDfoJQgghGts/Kj27U6cUtffmqYdDCuD/cMN8UmFhIf7+df8Nq2vhExISgtFoJD09vdLx9PR0IiIiqn1PRERErdrPnj270tCY3W4nJyeH4OBgtBY0obegoICYmBhSUlKa5dylunLV6wa5dle8dle9bpBrd8VrP/u6lVIUFhYSFRV14Tefh66Fj7u7O/369WP16tWMHz8ecBQmq1ev5r777qv2PYMGDWL16tU89NBDzmOrVq1i0KDqb8hkNpsxm82VjgUEBNRH/CbJz8/Ppf7HOM1Vrxvk2l3x2l31ukGu3RWv/czrvpientN0H+qaNWsWU6dOpX///gwYMIC5c+dSXFzM9OnTAZgyZQqtWrVizpw5ADz44IMMHz6cV155hbFjx7JkyRK2bdvGe++9p+dlCCGEEKIZ0L3wmThxIpmZmTzxxBOkpaXRu3dvVqxY4ZzAnJycjMHwx30WBw8ezOLFi/nHP/7B3/72Nzp06MBXX30l9/ARQgghxAXpXvgA3Hfffecc2lq3bl2VYzfffDM333xz1cYuzGw28+STT1YZ1mvpXPW6Qa7dFa/dVa8b5Npd8dob6rp1v4GhEEIIIURjaRJ7dQkhhBBCNAYpfIQQQgjhMqTwEUIIIYTLkMJHCCGEEC5DCp9mLCcnh9tuuw0/Pz8CAgK44447KCoqOm/7+++/n06dOuHp6Unr1q154IEHnPuXNVVvvvkmcXFxeHh4MHDgQLZs2XLe9l988QWdO3fGw8ODHj168P333zdS0vpXm2t///33GTZsGIGBgQQGBjJq1KgLfq2astp+309bsmQJmqY5b4ra3NT2uvPy8pg5cyaRkZGYzWY6duzYbP/O1/ba586d6/x5FhMTw1/+8hfKyprXXoMbNmzguuuuIyoqCk3TarTh9rp16+jbty9ms5n27duzYMGCBs/ZEGp77UuXLuXKK68kNDQUPz8/Bg0axI8//lj7D1ai2br66qtVr1691G+//aZ+/vln1b59ezVp0qRztt+zZ4+aMGGC+uabb1R8fLxavXq16tChg7rxxhsbMXXtLFmyRLm7u6sPP/xQ7du3T915550qICBApaenV9v+119/VUajUb300ktq//796h//+Idyc3NTe/bsaeTkF6+21z558mT15ptvqp07d6oDBw6oadOmKX9/f3X8+PFGTn7xanvtpyUlJalWrVqpYcOGqXHjxjVO2HpU2+suLy9X/fv3V9dcc4365ZdfVFJSklq3bp3atWtXIye/eLW99kWLFimz2awWLVqkkpKS1I8//qgiIyPVX/7yl0ZOfnG+//579fe//10tXbpUAWrZsmXnbZ+YmKi8vLzUrFmz1P79+9Xrr7+ujEajWrFiReMErke1vfYHH3xQvfjii2rLli3q8OHDavbs2crNzU3t2LGjVp8rhU8ztX//fgWorVu3Oo/98MMPStM0deLEiRqf5/PPP1fu7u7KYrE0RMyLNmDAADVz5kznc5vNpqKiotScOXOqbX/LLbeosWPHVjo2cOBAdffddzdozoZQ22s/m9VqVb6+vuqjjz5qqIgNpi7XbrVa1eDBg9UHH3ygpk6d2iwLn9pe99tvv63atm2rKioqGitig6nttc+cOVNdfvnllY7NmjVLDRkypEFzNqSa/PJ/9NFHVbdu3Sodmzhxoho9enQDJmt4Nbn26nTt2lU9/fTTtXqPDHU1U5s2bSIgIID+/fs7j40aNQqDwcDmzZtrfJ78/Hz8/PwwmZrEvSwrqaioYPv27YwaNcp5zGAwMGrUKDZt2lTtezZt2lSpPcDo0aPP2b6pqsu1n62kpASLxUJQUFBDxWwQdb32Z555hrCwMO64447GiFnv6nLd33zzDYMGDWLmzJmEh4fTvXt3nn/+eWw2W2PFrhd1ufbBgwezfft253BYYmIi33//Pddcc02jZNZLS/kZVx/sdjuFhYW1/hnX9H7biRpJS0sjLCys0jGTyURQUBBpaWk1OkdWVhbPPvssd911V0NEvGhZWVnYbDbn9iWnhYeHc/DgwWrfk5aWVm37mn5Nmoq6XPvZHnvsMaKioqr8kGzq6nLtv/zyC/PmzWPXrl2NkLBh1OW6ExMTWbNmDbfddhvff/898fHx3HvvvVgsFp588snGiF0v6nLtkydPJisri6FDh6KUwmq1cs899/C3v/2tMSLr5lw/4woKCigtLcXT01OnZI3v3//+N0VFRdxyyy21ep/0+DQxjz/+OJqmnfdR019851NQUMDYsWPp2rUrTz311MUHF03KCy+8wJIlS1i2bBkeHh56x2lQhYWF3H777bz//vuEhIToHadR2e12wsLCeO+99+jXrx8TJ07k73//O++8847e0RrcunXreP7553nrrbfYsWMHS5cuZfny5Tz77LN6RxONYPHixTz99NN8/vnnVToBLkR6fJqYv/71r0ybNu28bdq2bUtERAQZGRmVjlutVnJycoiIiDjv+wsLC7n66qvx9fVl2bJluLm5XWzsBhESEoLRaCQ9Pb3S8fT09HNeY0RERK3aN1V1ufbT/v3vf/PCCy/w008/0bNnz4aM2SBqe+0JCQkcPXqU6667znnMbrcDjl7QQ4cO0a5du4YNXQ/q8j2PjIzEzc0No9HoPNalSxfS0tKoqKjA3d29QTPXl7pc+z//+U9uv/12ZsyYAUCPHj0oLi7mrrvu4u9//3ulza1bknP9jPPz83OZ3p4lS5YwY8YMvvjiizr1aLfMvxnNWGhoKJ07dz7vw93dnUGDBpGXl8f27dud712zZg12u52BAwee8/wFBQVcddVVuLu788033zTp3gB3d3f69evH6tWrncfsdjurV69m0KBB1b5n0KBBldoDrFq16pztm6q6XDvASy+9xLPPPsuKFSsqzf9qTmp77Z07d2bPnj3s2rXL+bj++usZOXIku3btIiYmpjHj11ldvudDhgwhPj7eWegBHD58mMjIyGZT9EDdrr2kpKRKcXO6AFQteAvKlvIzrq4+/fRTpk+fzqeffsrYsWPrdpJaT6EWTcbVV1+t+vTpozZv3qx++eUX1aFDh0rL2Y8fP646deqkNm/erJRSKj8/Xw0cOFD16NFDxcfHq9TUVOfDarXqdRnntWTJEmU2m9WCBQvU/v371V133aUCAgJUWlqaUkqp22+/XT3++OPO9r/++qsymUzq3//+tzpw4IB68sknm/Vy9tpc+wsvvKDc3d3Vl19+Wel7W1hYqNcl1Fltr/1szXVVV22vOzk5Wfn6+qr77rtPHTp0SH333XcqLCxMPffcc3pdQp3V9tqffPJJ5evrqz799FOVmJioVq5cqdq1a6duueUWvS6hTgoLC9XOnTvVzp07FaBeffVVtXPnTnXs2DGllFKPP/64uv32253tTy9nf+SRR9SBAwfUm2++2WyXs9f22hctWqRMJpN68803K/2My8vLq9XnSuHTjGVnZ6tJkyYpHx8f5efnp6ZPn17pl1xSUpIC1Nq1a5VSSq1du1YB1T6SkpL0uYgaeP3111Xr1q2Vu7u7GjBggPrtt9+crw0fPlxNnTq1UvvPP/9cdezYUbm7u6tu3bqp5cuXN3Li+lOba4+Nja32e/vkk082fvB6UNvv+5maa+GjVO2ve+PGjWrgwIHKbDartm3bqn/9619N9h8yF1Kba7dYLOqpp55S7dq1Ux4eHiomJkbde++9Kjc3t/GDX4Rz/Vw+fa1Tp05Vw4cPr/Ke3r17K3d3d9W2bVs1f/78Rs9dH2p77cOHDz9v+5rSlGrBfYJCCCGEEGeQOT5CCCGEcBlS+AghhBDCZUjhI4QQQgiXIYWPEEIIIVyGFD5CCCGEcBlS+AghhBDCZUjhI4QQQgiXIYWPEKLFOnr0KJqmNYld2+Pi4pg7d+5FneOpp56id+/ezufTpk1j/PjxF3VOgAULFhAQEHDR5xGiOZDCR4hGsGnTJoxGY933lqmBjRs3cs011xAYGIiHhwc9evTg1VdfxWazNdhnjh49GqPRyNatW6u8Nm3aNJ566qnzvr+iooKXXnqJXr164eXlRUhICEOGDGH+/PlYLJaLzhcTE0Nqairdu3e/6HOdT0lJCbNnz6Zdu3Z4eHgQGhrK8OHD+frrr51ttm7dyl133XVRn/Pwww9X2aepPkycOJHDhw87n59dYAnRkkjhI0QjmDdvHvfffz8bNmzg5MmTF2yfmZlJWVlZjc+/bNkyhg8fTnR0NGvXruXgwYM8+OCDPPfcc9x6660X3LTx+PHjtd7YMTk5mY0bN3Lffffx4Ycf1uq94Ch6Ro8ezQsvvMBdd93Fxo0b2bJlCzNnzuT1119n3759tT7n2YxGIxEREZhMpos+1/ncc889LF26lNdff52DBw+yYsUKbrrpJrKzs51tQkND8fLyuqjP8fHxITg4+GLjVmKxWPD09CQsLKxezytEk1U/O24IIc6lsLBQ+fj4qIMHD6qJEyeqf/3rXxd8z4IFC1RAQIC6++671caNG8/btqioSAUHB6sJEyZUee2bb75RgFqyZMl5zzFt2jQVFxennnjiCZWQkHDBfEop9dRTT6lbb71VHThwQPn7+6uSkpJKr0+dOvW8+4S9+OKLymAwqB07dlR5raKiQhUVFSmllCorK1P333+/Cg0NVWazWQ0ZMkRt2bLF2TYnJ0dNnjxZhYSEKA8PD9W+fXv14YcfKqX+2K9u586dSqk/9gb66aefVL9+/ZSnp6caNGiQOnjwYKXP/+qrr1SfPn2U2WxWbdq0UU899ZSyWCznvBZ/f3+1YMGC8369YmNj1WuvveZ8Dqh33nlHjR07Vnl6eqrOnTurjRs3qiNHjqjhw4crLy8vNWjQIBUfH+98z5NPPql69erlfH72nmQ//PCDGjJkiPL391dBQUFq7Nixld5/+uuxZMkSddlllymz2azmz5+v5s+fr/z9/ZVSSs2fP7/KXkjz589X06dPV2PHjq10TRUVFSo0NFR98MEH5712IZoSKXyEaGDz5s1T/fv3V0op9e2336p27dopu91+3vdYLBb13XffqVtuuUV5eHiojh07qn/9618qOTm5StulS5cq4JwFUseOHS+4YWdBQYGaN2+eGj58uDIYDGrYsGFq3rx5qqCgoNr2drtdxcbGqu+++04ppVS/fv3UwoULK7W5UOHTs2dPddVVV503l1JKPfDAAyoqKkp9//33at++fWrq1KkqMDBQZWdnK6WUmjlzpurdu7faunWrSkpKUqtWrVLffPONUurchc/AgQPVunXr1L59+9SwYcPU4MGDnZ+3YcMG5efnpxYsWKASEhLUypUrVVxcnHrqqafOmbFTp07qlltuOefXS6nqC59WrVqpzz77TB06dEiNHz9excXFqcsvv1ytWLFC7d+/X1166aXq6quvdr7nQoXPl19+qf73v/+pI0eOqJ07d6rrrrtO9ejRQ9lstkpfj7i4OPW///1PJSYmqpMnT1YqfEpKStRf//pX1a1bN+fu1yUlJerXX39VRqNRnTx50vl5S5cuVd7e3pU2RxaiqZPCR4gGNnjwYDV37lyllKOgCQkJUWvXrq3x+/Py8tR7772nhg0bpoxGo7riiivUwoULnT0sL7zwggLOuSv19ddfr7p06VLjzzt69Kh69tlnVceOHZWXl5e67bbb1MqVKysVaytXrlShoaHOXpDXXnutyg7SF+Lp6akeeOCB87YpKipSbm5uatGiRc5jFRUVKioqSr300ktKKaWuu+46NX369Grff74en9OWL1+uAFVaWqqUUuqKK65Qzz//fKXzfPzxxyoyMvKcOdevX6+io6OVm5ub6t+/v3rooYfUL7/8UqlNdYXPP/7xD+fzTZs2KUDNmzfPeezTTz9VHh4ezucXKnzOlpmZqQC1Z8+eSl+P038fTzuz8Knuc07r2rWrevHFF53Pr7vuOjVt2rRzfr4QTZHM8RGiAR06dIgtW7YwadIkAEwmExMnTmTevHmAY56Mj4+P8/H8889XOYe/vz933nknGzZsYOPGjSQlJTFlyhR+/PHHSu3UeebouLu7A7Bo0aJKn/fzzz9XaRsbG8s//vEPDh06xFtvvcXXX3/NVVddRX5+vrPNhx9+yMSJE51zZyZNmsSvv/5KQkJCjb8258t7WkJCAhaLhSFDhjiPubm5MWDAAA4cOADAn//8Z5YsWULv3r159NFH2bhx4wXP27NnT+d/R0ZGApCRkQHA7t27eeaZZyp9ne68805SU1MpKSmp9nyXXXYZiYmJrF69mptuuol9+/YxbNgwnn322RrnCA8PB6BHjx6VjpWVlVFQUHDBawI4cuQIkyZNom3btvj5+REXFwc4/p6dqX///jU639lmzJjB/PnzAUhPT+eHH37gT3/6U53OJYReGnbGnxAubt68eVitVqKiopzHlFKYzWbeeOMNoqKiKi21DgoKqnKOsrIyvv32WxYuXMiPP/5Inz59ePjhh7niiisA6NChAwAHDhxg8ODBVd5/4MAB5wqd66+/noEDBzpfa9WqVZX2WVlZfPrpp3z88cfs2rWLMWPGMHXqVPz9/QHIyclh2bJlWCwW3n77bef7bDYbH374If/6179q9LXp2LEjBw8erFHb8xkzZgzHjh3j+++/Z9WqVVxxxRXMnDmTf//73+d8j5ubm/O/NU0DwG63A1BUVMTTTz/NhAkTqrzPw8PjvOccNmwYw4YN47HHHuO5557jmWee4bHHHnMWnjXJcb5sF3LdddcRGxvL+++/T1RUFHa7ne7du1NRUVGpnbe3d43Od7YpU6bw+OOPs2nTJjZu3EibNm0YNmxYnc4lhF6k8BGigVitVhYuXMgrr7zCVVddVem18ePH8+mnn3LPPffQvn37Ku9VSvHLL7+wcOFCvvjiC3x9ffm///s/Xn75ZTp37lyp7ejRowkKCuKVV16pUvh88803HDlyxHn/GF9fX3x9fat8Xnl5Od988w0ff/wxK1asoFu3bkybNo3ly5cTGhpaqe2iRYuIjo7mq6++qnR85cqVvPLKKzzzzDMYjcYLfn0mT57M3/72N3bu3EmfPn0qvWaxWKioqKBdu3a4u7vz66+/Ehsb63xt69atPPTQQ872oaGhTJ06lalTpzJs2DAeeeSR8xY+59O3b18OHTpU7felNrp27YrVaqWsrOychU99ys7O5tChQ7z//vvOYuSXX36p07nc3d2rvQ1CcHAw48ePZ/78+WzatInp06dfVGYh9CCFjxAN5LvvviM3N5c77rjD2Vty2o033si8efO45557qn3vJ598wt13380NN9zA559/zqhRozAYqh+Z9vb25t133+XWW2/lrrvu4r777sPPz4/Vq1fzyCOPcOedd3LNNdecN+u9997L8uXLue2223juuecqDcGcbd68edx0001V7o0TExPD7NmzWbFiRY3uV/TQQw+xfPlyrrjiCp599lmGDh2Kr68v27Zt48UXX2TevHn07t2bP//5zzzyyCMEBQXRunVrXnrpJUpKSrjjjjsAeOKJJ+jXrx/dunWjvLyc7777ji5dulzw88/liSee4Nprr6V169bcdNNNGAwGdu/ezd69e3nuueeqfc+IESOYNGkS/fv3Jzg4mP379/O3v/2NkSNH4ufnV+cstREYGEhwcDDvvfcekZGRJCcn8/jjj9fpXHFxcSQlJbFr1y6io6Px9fXFbDYDjuGua6+9FpvNxtSpU+vzEoRoHPpOMRKi5br22mvVNddcU+1rmzdvVoDavXt3ta+fOHFC5efn1+rzNmzYoEaPHq38/Pycy5DPnIh6PkeOHDnvcu3Ttm3bpoBKy8nPNGbMGHXDDTfUOHNZWZmaM2eO6tGjh/Lw8FBBQUFqyJAhasGCBc48paWl6v7771chISHVLmd/9tlnVZcuXZSnp6cKCgpS48aNU4mJiUqpc09uPnMi+M6dOxWgkpKSnMdWrFihBg8erDw9PZWfn58aMGCAeu+99855Hc8//7waNGiQCgoKUh4eHqpt27bqgQceUFlZWc421U1uXrZsmfP52Vmry3uhyc2rVq1SXbp0UWazWfXs2VOtW7eu0udU9xlKVZ3cXFZWpm688UYVEBDgXM5+2ukVfef6uy1EU6cpVcu7lgkhmryysjLGjRtHSkoK69evrzJcJURdFRUV0apVK+bPn1/tPCghmjpZ1SVEC+Th4cHXX3/NlClT2LBhg95xRAtgt9vJyMjg2WefJSAggOuvv17vSELUifT4CCGEuKCjR4/Spk0boqOjWbBggXNVoRDNjRQ+QgghhHAZMtQlhBBCCJchhY8QQgghXIYUPkIIIYRwGVL4CCGEEMJlSOEjhBBCCJchhY8QQgghXIYUPkIIIYRwGVL4CCGEEMJlSOEjhBBCCJfx/zvLXOrv6R3aAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# sns.distplot(df_gpt35['cosine'], label='3.5')\n",
    "\n",
    "sns.distplot(df_gpt4o['cosine'], label='4o')\n",
    "sns.distplot(df_gpt4o_mini['cosine'], label='4o-mini')\n",
    "\n",
    "plt.title(\"RAG LLM performance\")\n",
    "plt.xlabel(\"A->Q->A' Cosine Similarity\")\n",
    "plt.legend()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5d865c1d-3577-4812-959a-93fc8c3a598b",
   "metadata": {},
   "source": [
    "## LLM-as-a-Judge"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 234,
   "id": "607005fb-d83f-44be-a51a-6bed9f6ebdc7",
   "metadata": {},
   "outputs": [],
   "source": [
    "prompt1_template = \"\"\"\n",
    "You are an expert evaluator for a Retrieval-Augmented Generation (RAG) system.\n",
    "Your task is to analyze the relevance of the generated answer compared to the original answer provided.\n",
    "Based on the relevance and similarity of the generated answer to the original answer, you will classify\n",
    "it as \"NON_RELEVANT\", \"PARTLY_RELEVANT\", or \"RELEVANT\".\n",
    "\n",
    "Here is the data for evaluation:\n",
    "\n",
    "Original Answer: {answer_orig}\n",
    "Generated Question: {question}\n",
    "Generated Answer: {answer_llm}\n",
    "\n",
    "Please analyze the content and context of the generated answer in relation to the original\n",
    "answer and provide your evaluation in parsable JSON without using code blocks:\n",
    "\n",
    "{{\n",
    "  \"Relevance\": \"NON_RELEVANT\" | \"PARTLY_RELEVANT\" | \"RELEVANT\",\n",
    "  \"Explanation\": \"[Provide a brief explanation for your evaluation]\"\n",
    "}}\n",
    "\"\"\".strip()\n",
    "\n",
    "prompt2_template = \"\"\"\n",
    "You are an expert evaluator for a Retrieval-Augmented Generation (RAG) system.\n",
    "Your task is to analyze the relevance of the generated answer to the given question.\n",
    "Based on the relevance of the generated answer, you will classify it\n",
    "as \"NON_RELEVANT\", \"PARTLY_RELEVANT\", or \"RELEVANT\".\n",
    "\n",
    "Here is the data for evaluation:\n",
    "\n",
    "Question: {question}\n",
    "Generated Answer: {answer_llm}\n",
    "\n",
    "Please analyze the content and context of the generated answer in relation to the question\n",
    "and provide your evaluation in parsable JSON without using code blocks:\n",
    "\n",
    "{{\n",
    "  \"Relevance\": \"NON_RELEVANT\" | \"PARTLY_RELEVANT\" | \"RELEVANT\",\n",
    "  \"Explanation\": \"[Provide a brief explanation for your evaluation]\"\n",
    "}}\n",
    "\"\"\".strip()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 169,
   "id": "fe131dc5-256b-4198-96f7-4c6209ca9b68",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_sample = df_gpt4o_mini.sample(n=150, random_state=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 173,
   "id": "fd5b25b5-4244-4322-9cba-40b2dde6b7de",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "samples = df_sample.to_dict(orient='records')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 208,
   "id": "e61a635b-db6d-472f-89a1-a4d50f622124",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'answer_llm': 'The syntax for using `precision_recall_fscore_support` in Python is as follows:\\n\\n```python\\nfrom sklearn.metrics import precision_recall_fscore_support\\nprecision, recall, fscore, support = precision_recall_fscore_support(y_val, y_val_pred, zero_division=0)\\n```',\n",
       " 'answer_orig': 'Scikit-learn offers another way: precision_recall_fscore_support\\nExample:\\nfrom sklearn.metrics import precision_recall_fscore_support\\nprecision, recall, fscore, support = precision_recall_fscore_support(y_val, y_val_pred, zero_division=0)\\n(Gopakumar Gopinathan)',\n",
       " 'document': '403bbdd8',\n",
       " 'question': 'What is the syntax for using precision_recall_fscore_support in Python?',\n",
       " 'course': 'machine-learning-zoomcamp',\n",
       " 'cosine': 0.9010756015777588}"
      ]
     },
     "execution_count": 208,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "record = samples[0]\n",
    "record"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 209,
   "id": "e514f94b-8564-40ab-a367-a5a2dda0c8e3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "You are an expert evaluator for a Retrieval-Augmented Generation (RAG) system.\n",
      "Your task is to analyze the relevance of the generated answer compared to the original answer provided.\n",
      "Based on the relevance and similarity of the generated answer to the original answer, you will classify\n",
      "it as \"NON_RELEVANT\", \"PARTLY_RELEVANT\", or \"RELEVANT\".\n",
      "\n",
      "Here is the data for evaluation:\n",
      "\n",
      "Original Answer: Scikit-learn offers another way: precision_recall_fscore_support\n",
      "Example:\n",
      "from sklearn.metrics import precision_recall_fscore_support\n",
      "precision, recall, fscore, support = precision_recall_fscore_support(y_val, y_val_pred, zero_division=0)\n",
      "(Gopakumar Gopinathan)\n",
      "Generated Question: What is the syntax for using precision_recall_fscore_support in Python?\n",
      "Generated Answer: The syntax for using `precision_recall_fscore_support` in Python is as follows:\n",
      "\n",
      "```python\n",
      "from sklearn.metrics import precision_recall_fscore_support\n",
      "precision, recall, fscore, support = precision_recall_fscore_support(y_val, y_val_pred, zero_division=0)\n",
      "```\n",
      "\n",
      "Please analyze the content and context of the generated answer in relation to the original\n",
      "answer and provide your evaluation in parsable JSON without using code blocks:\n",
      "\n",
      "{\n",
      "  \"Relevance\": \"NON_RELEVANT\" | \"PARTLY_RELEVANT\" | \"RELEVANT\",\n",
      "  \"Explanation\": \"[Provide a brief explanation for your evaluation]\"\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "prompt = prompt1_template.format(**record)\n",
    "print(prompt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 210,
   "id": "57e6694f-bf49-467d-82ae-ca5bf06e94f4",
   "metadata": {},
   "outputs": [],
   "source": [
    "answer = llm(prompt, model='gpt-4o-mini')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 187,
   "id": "edbe0ddb-75ab-4663-abbd-9886bc23a416",
   "metadata": {},
   "outputs": [],
   "source": [
    "import json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 224,
   "id": "92b557cc-13a7-4832-b1c9-382cc51d08e2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0be16eace86d4f319bc83764e6088039",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/150 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "evaluations = []\n",
    "\n",
    "for record in tqdm(samples):\n",
    "    prompt = prompt1_template.format(**record)\n",
    "    evaluation = llm(prompt, model='gpt-4o-mini')\n",
    "    evaluations.append(evaluation)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 225,
   "id": "f3afb95a-53b9-474d-87fc-3d4461a7f27d",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "json_evaluations = []\n",
    "\n",
    "for i, str_eval in enumerate(evaluations):\n",
    "    json_eval = json.loads(str_eval)\n",
    "    json_evaluations.append(json_eval)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 227,
   "id": "a6419c24-1a39-4d3b-9468-b99208355035",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_evaluations = pd.DataFrame(json_evaluations)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 228,
   "id": "a1d6aa1e-8c91-4a90-82e4-e2046a0e5be0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Relevance\n",
       "RELEVANT           124\n",
       "PARTLY_RELEVANT     16\n",
       "NON_RELEVANT        10\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 228,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_evaluations.Relevance.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 231,
   "id": "3664805d-2500-4b65-9980-de162ccf5df8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Relevance</th>\n",
       "      <th>Explanation</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>NON_RELEVANT</td>\n",
       "      <td>The generated answer discusses a pip version e...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>NON_RELEVANT</td>\n",
       "      <td>The generated answer does not address the spec...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>NON_RELEVANT</td>\n",
       "      <td>The generated answer incorrectly states that t...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41</th>\n",
       "      <td>NON_RELEVANT</td>\n",
       "      <td>The generated answer provides information abou...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>87</th>\n",
       "      <td>NON_RELEVANT</td>\n",
       "      <td>The generated answer does not address the orig...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>90</th>\n",
       "      <td>NON_RELEVANT</td>\n",
       "      <td>The generated answer responds to a question ab...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>93</th>\n",
       "      <td>NON_RELEVANT</td>\n",
       "      <td>The generated answer does not address the topi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>116</th>\n",
       "      <td>NON_RELEVANT</td>\n",
       "      <td>The generated answer discusses the recommended...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>138</th>\n",
       "      <td>NON_RELEVANT</td>\n",
       "      <td>The generated answer addresses a different iss...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>139</th>\n",
       "      <td>NON_RELEVANT</td>\n",
       "      <td>The generated answer does not relate to the to...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        Relevance                                        Explanation\n",
       "4    NON_RELEVANT  The generated answer discusses a pip version e...\n",
       "11   NON_RELEVANT  The generated answer does not address the spec...\n",
       "27   NON_RELEVANT  The generated answer incorrectly states that t...\n",
       "41   NON_RELEVANT  The generated answer provides information abou...\n",
       "87   NON_RELEVANT  The generated answer does not address the orig...\n",
       "90   NON_RELEVANT  The generated answer responds to a question ab...\n",
       "93   NON_RELEVANT  The generated answer does not address the topi...\n",
       "116  NON_RELEVANT  The generated answer discusses the recommended...\n",
       "138  NON_RELEVANT  The generated answer addresses a different iss...\n",
       "139  NON_RELEVANT  The generated answer does not relate to the to..."
      ]
     },
     "execution_count": 231,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_evaluations[df_evaluations.Relevance == 'NON_RELEVANT'] #.to_dict(orient='records')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 232,
   "id": "550a962b-c807-4dcc-91c4-bb69f47f1c48",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'answer_llm': \"The cause of the pip version error in this week's serverless deep learning section could be a version conflict in Scikit-Learn. Specifically, if you are using a different version than what was used during the model training, it can lead to warnings and potential breaking code or invalid results. To resolve this, make sure to use the same version of Scikit-Learn that was used for training the model. For instance, if you trained with version 1.1.1, you should use that same version in your virtual environment.\",\n",
       " 'answer_orig': 'When running docker build -t dino-dragon-model it returns the above error\\nThe most common source of this error in this week is because Alex video shows a version of the wheel with python 8, we need to find a wheel with the version that we are working on. In this case python 9. Another common error is to copy the link, this will also produce the same error, we need to download the raw format:\\nhttps://github.com/alexeygrigorev/tflite-aws-lambda/raw/main/tflite/tflite_runtime-2.7.0-cp39-cp39-linux_x86_64.whl\\nPastor Soto',\n",
       " 'document': '42c09143',\n",
       " 'question': \"What might be the cause of the pip version error in this week's serverless deep learning section?\",\n",
       " 'course': 'machine-learning-zoomcamp',\n",
       " 'cosine': 0.309404581785202}"
      ]
     },
     "execution_count": 232,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sample[4]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 235,
   "id": "6cdb1246-4769-406c-a9a3-a602ce5d3801",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "You are an expert evaluator for a Retrieval-Augmented Generation (RAG) system.\n",
      "Your task is to analyze the relevance of the generated answer to the given question.\n",
      "Based on the relevance of the generated answer, you will classify it\n",
      "as \"NON_RELEVANT\", \"PARTLY_RELEVANT\", or \"RELEVANT\".\n",
      "\n",
      "Here is the data for evaluation:\n",
      "\n",
      "Question: What modification was made to the median_house_value target in the homework?\n",
      "Generated Answer: The modification made to the `median_house_value` target in the homework was that it was changed to binary format. The values were made discrete as either 0 or 1, instead of remaining as a continuous variable. This change was necessary for the calculation of the mutual information score, which is applicable to categorical or discrete variables rather than continuous ones.\n",
      "\n",
      "Please analyze the content and context of the generated answer in relation to the question\n",
      "and provide your evaluation in parsable JSON without using code blocks:\n",
      "\n",
      "{\n",
      "  \"Relevance\": \"NON_RELEVANT\" | \"PARTLY_RELEVANT\" | \"RELEVANT\",\n",
      "  \"Explanation\": \"[Provide a brief explanation for your evaluation]\"\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "prompt = prompt2_template.format(**record)\n",
    "print(prompt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 236,
   "id": "650e8ecb-80e2-42f1-8039-8d497e3eeca6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\n",
      "  \"Relevance\": \"RELEVANT\",\n",
      "  \"Explanation\": \"The generated answer directly addresses the modification made to the `median_house_value` target by clarifying that it was changed to a binary format. It explains the significance of the change and its relation to the calculation of the mutual information score, which is pertinent to understanding the reasons behind the modification.\"\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "evaluation = llm(prompt, model='gpt-4o-mini')\n",
    "print(evaluation)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 237,
   "id": "03afffa3-9f7e-47eb-9c17-46ae39e38424",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c5ed7fd2f01643d783c00dbf6d6e3001",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/150 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "evaluations_2 = []\n",
    "\n",
    "for record in tqdm(samples):\n",
    "    prompt = prompt2_template.format(**record)\n",
    "    evaluation = llm(prompt, model='gpt-4o-mini')\n",
    "    evaluations_2.append(evaluation)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 238,
   "id": "c00c22f5-fe31-4782-99ac-846249fc59b6",
   "metadata": {},
   "outputs": [],
   "source": [
    "json_evaluations_2 = []\n",
    "\n",
    "for i, str_eval in enumerate(evaluations_2):\n",
    "    json_eval = json.loads(str_eval)\n",
    "    json_evaluations_2.append(json_eval)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 239,
   "id": "05c6d861-9d0c-41bb-a5db-454283c4b343",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_evaluations_2 = pd.DataFrame(json_evaluations_2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 242,
   "id": "09064866-d08c-4e0d-a095-44a9e8625bfb",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Relevance</th>\n",
       "      <th>Explanation</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>45</th>\n",
       "      <td>NON_RELEVANT</td>\n",
       "      <td>The generated answer does not address the ques...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49</th>\n",
       "      <td>NON_RELEVANT</td>\n",
       "      <td>The generated answer explicitly states that th...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>139</th>\n",
       "      <td>NON_RELEVANT</td>\n",
       "      <td>The generated answer provides information abou...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        Relevance                                        Explanation\n",
       "45   NON_RELEVANT  The generated answer does not address the ques...\n",
       "49   NON_RELEVANT  The generated answer explicitly states that th...\n",
       "139  NON_RELEVANT  The generated answer provides information abou..."
      ]
     },
     "execution_count": 242,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_evaluations_2[df_evaluations_2.Relevance == 'NON_RELEVANT']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 243,
   "id": "95af0004-e25c-4328-8d22-d48443470da0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'answer_llm': 'The provided context does not include specific commands to start the Docker daemon on Linux. Therefore, I cannot provide an answer based solely on the facts from the context.',\n",
       " 'answer_orig': 'Working on getting Docker installed - when I try running hello-world I am getting the error.\\nDocker: Cannot connect to the docker daemon at unix:///var/run/docker.sock. Is the Docker daemon running ?\\nSolution description\\nIf you’re getting this error on WSL, re-install your docker: remove the docker installation from WSL and install Docker Desktop on your host machine (Windows).\\nOn Linux, start the docker daemon with either of these commands:\\nsudo dockerd\\nsudo service docker start\\nAdded by Ugochukwu Onyebuchi',\n",
       " 'document': '4b2a3181',\n",
       " 'question': 'What commands should I use to start the docker daemon on Linux?',\n",
       " 'course': 'machine-learning-zoomcamp',\n",
       " 'cosine': 0.51130211353302}"
      ]
     },
     "execution_count": 243,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "samples[45]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2e515bdc-2a05-4934-800b-7e1ccc1652b5",
   "metadata": {},
   "source": [
    "## Saving all the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 244,
   "id": "c5894357-bfc4-4c42-939b-3e8c052818a1",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_gpt4o.to_csv('data/results-gpt4o-cosine.csv', index=False)\n",
    "df_gpt35.to_csv('data/results-gpt35-cosine.csv', index=False)\n",
    "df_gpt4o_mini.to_csv('data/results-gpt4o-mini-cosine.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 245,
   "id": "255233a1-0719-4148-9281-047db698ad67",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_evaluations.to_csv('data/evaluations-aqa.csv', index=False)\n",
    "df_evaluations_2.to_csv('data/evaluations-qa.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "33923082-8634-46ea-bba7-0db99a94d713",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}