examples/knn.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# DSPy KNN few-shot example \n",
    "This noteboook shows how KNN few-shot can be implemented with DSPy using the **KNNFewShot** teleprompter. To illustrate, we use the HotPotQA dataset. Please see [intro.ipynb](../intro.ipynb) for other example use cases of DSPy.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import openai\n",
    "import dspy\n",
    "import json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(\"creds.json\", \"r\") as creds:\n",
    "    api_key = json.loads(creds.read())[\"openai_key\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "lm = dspy.OpenAI(model='gpt-4', api_key=api_key, model_type='chat', max_tokens = 500)\n",
    "dspy.settings.configure(lm=lm)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "from dspy.datasets import HotPotQA\n",
    "\n",
    "# Load the dataset.\n",
    "dataset = HotPotQA(train_seed=1, train_size=20, eval_seed=2023, dev_size=50, test_size=0)\n",
    "\n",
    "trainset = [x.with_inputs('question') for x in dataset.train]\n",
    "devset = [x.with_inputs('question') for x in dataset.dev]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Example({'question': 'At My Window was released by which American singer-songwriter?', 'answer': 'John Townes Van Zandt'}) (input_keys={'question'})\n",
      "Question: At My Window was released by which American singer-songwriter?\n",
      "Answer: John Townes Van Zandt\n"
     ]
    }
   ],
   "source": [
    "train_example = trainset[0]\n",
    "print(train_example)\n",
    "print(f\"Question: {train_example.question}\")\n",
    "print(f\"Answer: {train_example.answer}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "class BasicQA(dspy.Signature):\n",
    "    \"\"\"Answer questions with short factoid answers.\"\"\"\n",
    "\n",
    "    question = dspy.InputField()\n",
    "    answer = dspy.OutputField(desc=\"often between 1 and 5 words\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "class BasicQABot(dspy.Module):\n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "\n",
    "        self.generate = dspy.Predict(BasicQA)\n",
    "\n",
    "    def forward(self,question):\n",
    "        prediction = self.generate(question = question)\n",
    "        return dspy.Prediction(answer = prediction.answer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Alfred the Great'"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "qa_bot = BasicQABot()\n",
    "pred = qa_bot.forward(\"In the 10th Century A.D. Ealhswith had a son called Æthelweard by which English king?\")\n",
    "pred.answer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "from dspy.teleprompt import KNNFewShot\n",
    "from dspy.predict.knn import KNN\n",
    "\n",
    "knn_teleprompter = KNNFewShot(KNN, 7, trainset)\n",
    "compiled_knn = knn_teleprompter.compile(BasicQABot(), trainset=trainset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 57%|█████▋    | 4/7 [00:02<00:02,  1.44it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Bootstrapped 4 full traces after 5 examples in round 0.\n",
      "Question:  Are both Cangzhou and Qionghai in the Hebei province of China?\n",
      "Expected answer:  no\n",
      "Prediction:  No\n"
     ]
    }
   ],
   "source": [
    "example = devset[0]\n",
    "pred = compiled_knn(question = example.question)\n",
    "print(\"Question: \", example.question)\n",
    "print(\"Expected answer: \", example.answer)\n",
    "print(\"Prediction: \", pred.answer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "\n",
      "\n",
      "Answer questions with short factoid answers.\n",
      "\n",
      "---\n",
      "\n",
      "Follow the following format.\n",
      "\n",
      "Question: ${question}\n",
      "Answer: often between 1 and 5 words\n",
      "\n",
      "---\n",
      "\n",
      "Question: On the coast of what ocean is the birthplace of Diogal Sakho?\n",
      "Answer: Atlantic Ocean\n",
      "\n",
      "Question: Which is taller, the Empire State Building or the Bank of America Tower?\n",
      "Answer: Empire State Building\n",
      "\n",
      "Question: Samantha Cristoforetti and Mark Shuttleworth are both best known for being first in their field to go where?\n",
      "Answer: Space\n",
      "\n",
      "Question: Which Pakistani cricket umpire who won 3 consecutive ICC umpire of the year awards in 2009, 2010, and 2011 will be in the ICC World Twenty20?\n",
      "Answer: Aleem Dar\n",
      "\n",
      "Question: What is the code name for the German offensive that started this Second World War engagement on the Eastern Front (a few hundred kilometers from Moscow) between Soviet and German forces, which included 102nd Infantry Division?\n",
      "Answer: Operation Citadel\n",
      "\n",
      "Question: Which of these publications was most recently published, Who Put the Bomp or Self?\n",
      "Answer: Self\n",
      "\n",
      "Question: Which magazine has published articles by Scott Shaw, Tae Kwon Do Times or Southwest Art?\n",
      "Answer: Tae Kwon Do Times\n",
      "\n",
      "---\n",
      "\n",
      "Question: Are both Cangzhou and Qionghai in the Hebei province of China?\n",
      "Answer:\u001b[32m No\u001b[0m\n",
      "\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "lm.inspect_history(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 57%|█████▋    | 4/7 [00:03<00:02,  1.28it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Bootstrapped 4 full traces after 5 examples in round 0.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 57%|█████▋    | 4/7 [00:03<00:02,  1.06it/s]  | 1/50 [00:03<03:01,  3.70s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Bootstrapped 4 full traces after 5 examples in round 0.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 57%|█████▋    | 4/7 [00:02<00:01,  1.57it/s]  | 2/50 [00:08<03:14,  4.05s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Bootstrapped 4 full traces after 5 examples in round 0.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 57%|█████▋    | 4/7 [00:03<00:02,  1.16it/s]  | 3/50 [00:11<02:58,  3.79s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Bootstrapped 4 full traces after 5 examples in round 0.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 57%|█████▋    | 4/7 [00:03<00:02,  1.30it/s] | 4/50 [00:15<02:56,  3.84s/it] \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Bootstrapped 4 full traces after 5 examples in round 0.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 57%|█████▋    | 4/7 [00:02<00:01,  1.52it/s] | 5/50 [00:19<02:49,  3.78s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Bootstrapped 4 full traces after 5 examples in round 0.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 57%|█████▋    | 4/7 [00:03<00:02,  1.10it/s] | 6/50 [00:22<02:43,  3.72s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Bootstrapped 4 full traces after 5 examples in round 0.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 57%|█████▋    | 4/7 [00:02<00:01,  1.51it/s] | 7/50 [00:26<02:46,  3.88s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Bootstrapped 4 full traces after 5 examples in round 0.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 57%|█████▋    | 4/7 [00:04<00:03,  1.02s/it] | 8/50 [00:30<02:34,  3.68s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Bootstrapped 4 full traces after 5 examples in round 0.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 57%|█████▋    | 4/7 [00:03<00:02,  1.18it/s] | 9/50 [00:35<02:54,  4.26s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Bootstrapped 4 full traces after 5 examples in round 0.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 57%|█████▋    | 4/7 [00:03<00:02,  1.31it/s]  | 10/50 [00:39<02:46,  4.15s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Bootstrapped 4 full traces after 5 examples in round 0.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 57%|█████▋    | 4/7 [00:03<00:02,  1.26it/s]  | 11/50 [00:43<02:33,  3.94s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Bootstrapped 4 full traces after 5 examples in round 0.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 57%|█████▋    | 4/7 [00:03<00:02,  1.16it/s]  | 12/50 [00:47<02:35,  4.09s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Bootstrapped 4 full traces after 5 examples in round 0.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 57%|█████▋    | 4/7 [00:03<00:02,  1.32it/s]  | 13/50 [00:51<02:29,  4.04s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Bootstrapped 4 full traces after 5 examples in round 0.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 57%|█████▋    | 4/7 [00:02<00:01,  1.62it/s]  | 14/50 [00:55<02:23,  3.97s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Bootstrapped 4 full traces after 5 examples in round 0.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 57%|█████▋    | 4/7 [00:02<00:02,  1.37it/s]  | 15/50 [00:58<02:11,  3.75s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Bootstrapped 4 full traces after 5 examples in round 0.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 57%|█████▋    | 4/7 [00:03<00:02,  1.28it/s]  | 16/50 [01:02<02:05,  3.70s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Bootstrapped 4 full traces after 5 examples in round 0.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 57%|█████▋    | 4/7 [00:02<00:02,  1.39it/s]  | 17/50 [01:05<02:04,  3.78s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Bootstrapped 4 full traces after 5 examples in round 0.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 57%|█████▋    | 4/7 [00:05<00:04,  1.36s/it]  | 18/50 [01:09<01:58,  3.69s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Bootstrapped 4 full traces after 5 examples in round 0.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 57%|█████▋    | 4/7 [00:02<00:01,  1.52it/s]  | 19/50 [01:15<02:16,  4.39s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Bootstrapped 4 full traces after 5 examples in round 0.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 57%|█████▋    | 4/7 [00:03<00:02,  1.21it/s]  | 20/50 [01:19<02:03,  4.13s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Bootstrapped 4 full traces after 5 examples in round 0.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 57%|█████▋    | 4/7 [00:03<00:02,  1.27it/s]  | 21/50 [01:22<01:57,  4.06s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Bootstrapped 4 full traces after 5 examples in round 0.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 57%|█████▋    | 4/7 [00:03<00:02,  1.10it/s]  | 22/50 [01:26<01:52,  4.01s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Bootstrapped 4 full traces after 5 examples in round 0.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 57%|█████▋    | 4/7 [00:03<00:02,  1.25it/s]  | 23/50 [01:31<01:50,  4.08s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Bootstrapped 4 full traces after 5 examples in round 0.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 57%|█████▋    | 4/7 [00:04<00:03,  1.04s/it]   | 24/50 [01:34<01:44,  4.04s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Bootstrapped 4 full traces after 5 examples in round 0.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 57%|█████▋    | 4/7 [00:03<00:02,  1.08it/s]   | 25/50 [01:39<01:47,  4.32s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Bootstrapped 4 full traces after 5 examples in round 0.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 57%|█████▋    | 4/7 [00:03<00:02,  1.14it/s]   | 26/50 [01:44<01:44,  4.36s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Bootstrapped 4 full traces after 5 examples in round 0.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 57%|█████▋    | 4/7 [00:04<00:03,  1.22s/it]   | 27/50 [01:49<01:43,  4.50s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Bootstrapped 4 full traces after 5 examples in round 0.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 57%|█████▋    | 4/7 [00:04<00:03,  1.00s/it]   | 28/50 [01:54<01:45,  4.77s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Bootstrapped 4 full traces after 5 examples in round 0.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 57%|█████▋    | 4/7 [00:03<00:02,  1.21it/s]   | 29/50 [01:59<01:40,  4.78s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Bootstrapped 4 full traces after 5 examples in round 0.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 57%|█████▋    | 4/7 [00:02<00:01,  1.67it/s]   | 30/50 [02:03<01:29,  4.45s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Bootstrapped 4 full traces after 5 examples in round 0.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 57%|█████▋    | 4/7 [00:02<00:01,  1.54it/s]   | 31/50 [02:06<01:19,  4.17s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Bootstrapped 4 full traces after 5 examples in round 0.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 57%|█████▋    | 4/7 [00:02<00:01,  1.75it/s]   | 32/50 [02:09<01:09,  3.84s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Bootstrapped 4 full traces after 5 examples in round 0.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 57%|█████▋    | 4/7 [00:02<00:01,  1.53it/s]   | 33/50 [02:12<01:01,  3.65s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Bootstrapped 4 full traces after 5 examples in round 0.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 57%|█████▋    | 4/7 [00:02<00:01,  1.51it/s]   | 34/50 [02:16<00:55,  3.50s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Bootstrapped 4 full traces after 5 examples in round 0.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 57%|█████▋    | 4/7 [00:03<00:02,  1.06it/s]   | 35/50 [02:20<00:54,  3.65s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Bootstrapped 4 full traces after 5 examples in round 0.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 57%|█████▋    | 4/7 [00:04<00:03,  1.15s/it]▏  | 36/50 [02:25<00:57,  4.09s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Bootstrapped 4 full traces after 5 examples in round 0.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 57%|█████▋    | 4/7 [00:02<00:01,  1.57it/s]▍  | 37/50 [02:30<00:57,  4.41s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Bootstrapped 4 full traces after 5 examples in round 0.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 57%|█████▋    | 4/7 [00:04<00:03,  1.08s/it]▌  | 38/50 [02:33<00:47,  4.00s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Bootstrapped 4 full traces after 5 examples in round 0.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 57%|█████▋    | 4/7 [00:02<00:02,  1.35it/s]▊  | 39/50 [02:38<00:47,  4.33s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Bootstrapped 4 full traces after 5 examples in round 0.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 57%|█████▋    | 4/7 [00:03<00:02,  1.13it/s]█  | 40/50 [02:42<00:41,  4.20s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Bootstrapped 4 full traces after 5 examples in round 0.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 57%|█████▋    | 4/7 [00:03<00:02,  1.30it/s]█▏ | 41/50 [02:46<00:38,  4.29s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Bootstrapped 4 full traces after 5 examples in round 0.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 57%|█████▋    | 4/7 [00:04<00:03,  1.04s/it]█▍ | 42/50 [02:50<00:32,  4.07s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Bootstrapped 4 full traces after 5 examples in round 0.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 57%|█████▋    | 4/7 [00:03<00:02,  1.27it/s]█▌ | 43/50 [02:56<00:31,  4.53s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Bootstrapped 4 full traces after 5 examples in round 0.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 57%|█████▋    | 4/7 [00:03<00:02,  1.33it/s]█▊ | 44/50 [02:59<00:25,  4.28s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Bootstrapped 4 full traces after 5 examples in round 0.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 57%|█████▋    | 4/7 [00:04<00:03,  1.04s/it]██ | 45/50 [03:03<00:20,  4.20s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Bootstrapped 4 full traces after 5 examples in round 0.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 57%|█████▋    | 4/7 [00:04<00:03,  1.02s/it]██▏| 46/50 [03:09<00:18,  4.61s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Bootstrapped 4 full traces after 5 examples in round 0.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 57%|█████▋    | 4/7 [00:02<00:02,  1.35it/s]██▍| 47/50 [03:14<00:14,  4.74s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Bootstrapped 4 full traces after 5 examples in round 0.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 57%|█████▋    | 4/7 [00:03<00:02,  1.17it/s]██▌| 48/50 [03:17<00:08,  4.38s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Bootstrapped 4 full traces after 5 examples in round 0.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 57%|█████▋    | 4/7 [00:04<00:03,  1.02s/it]██▊| 49/50 [03:21<00:04,  4.28s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Bootstrapped 4 full traces after 5 examples in round 0.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Average Metric: 23 / 50  (46.0): 100%|██████████| 50/50 [03:26<00:00,  4.13s/it]\n",
      "/home/jovyan/scdc/project-vaqa-autosuggest/Query-Generation-exploratory/dspy/dspy/evaluate/evaluate.py:126: FutureWarning: DataFrame.applymap has been deprecated. Use DataFrame.map instead.\n",
      "  df = df.applymap(truncate_cell)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Average Metric: 23 / 50  (46.0%)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<style type=\"text/css\">\n",
       "#T_aaaf4 th {\n",
       "  text-align: left;\n",
       "}\n",
       "#T_aaaf4 td {\n",
       "  text-align: left;\n",
       "}\n",
       "#T_aaaf4_row0_col0, #T_aaaf4_row0_col1, #T_aaaf4_row0_col2, #T_aaaf4_row0_col3, #T_aaaf4_row0_col4, #T_aaaf4_row1_col0, #T_aaaf4_row1_col1, #T_aaaf4_row1_col2, #T_aaaf4_row1_col3, #T_aaaf4_row1_col4, #T_aaaf4_row2_col0, #T_aaaf4_row2_col1, #T_aaaf4_row2_col2, #T_aaaf4_row2_col3, #T_aaaf4_row2_col4, #T_aaaf4_row3_col0, #T_aaaf4_row3_col1, #T_aaaf4_row3_col2, #T_aaaf4_row3_col3, #T_aaaf4_row3_col4, #T_aaaf4_row4_col0, #T_aaaf4_row4_col1, #T_aaaf4_row4_col2, #T_aaaf4_row4_col3, #T_aaaf4_row4_col4 {\n",
       "  text-align: left;\n",
       "  white-space: pre-wrap;\n",
       "  word-wrap: break-word;\n",
       "  max-width: 400px;\n",
       "}\n",
       "</style>\n",
       "<table id=\"T_aaaf4\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th class=\"blank level0\" >&nbsp;</th>\n",
       "      <th id=\"T_aaaf4_level0_col0\" class=\"col_heading level0 col0\" >question</th>\n",
       "      <th id=\"T_aaaf4_level0_col1\" class=\"col_heading level0 col1\" >example_answer</th>\n",
       "      <th id=\"T_aaaf4_level0_col2\" class=\"col_heading level0 col2\" >gold_titles</th>\n",
       "      <th id=\"T_aaaf4_level0_col3\" class=\"col_heading level0 col3\" >pred_answer</th>\n",
       "      <th id=\"T_aaaf4_level0_col4\" class=\"col_heading level0 col4\" >answer_exact_match</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th id=\"T_aaaf4_level0_row0\" class=\"row_heading level0 row0\" >0</th>\n",
       "      <td id=\"T_aaaf4_row0_col0\" class=\"data row0 col0\" >Are both Cangzhou and Qionghai in the Hebei province of China?</td>\n",
       "      <td id=\"T_aaaf4_row0_col1\" class=\"data row0 col1\" >no</td>\n",
       "      <td id=\"T_aaaf4_row0_col2\" class=\"data row0 col2\" >{'Cangzhou', 'Qionghai'}</td>\n",
       "      <td id=\"T_aaaf4_row0_col3\" class=\"data row0 col3\" >No</td>\n",
       "      <td id=\"T_aaaf4_row0_col4\" class=\"data row0 col4\" >✔️ [True]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_aaaf4_level0_row1\" class=\"row_heading level0 row1\" >1</th>\n",
       "      <td id=\"T_aaaf4_row1_col0\" class=\"data row1 col0\" >Who conducts the draft in which Marc-Andre Fleury was drafted to the Vegas Golden Knights for the 2017-18 season?</td>\n",
       "      <td id=\"T_aaaf4_row1_col1\" class=\"data row1 col1\" >National Hockey League</td>\n",
       "      <td id=\"T_aaaf4_row1_col2\" class=\"data row1 col2\" >{'2017–18 Pittsburgh Penguins season', '2017 NHL Expansion Draft'}</td>\n",
       "      <td id=\"T_aaaf4_row1_col3\" class=\"data row1 col3\" >National Hockey League</td>\n",
       "      <td id=\"T_aaaf4_row1_col4\" class=\"data row1 col4\" >✔️ [True]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_aaaf4_level0_row2\" class=\"row_heading level0 row2\" >2</th>\n",
       "      <td id=\"T_aaaf4_row2_col0\" class=\"data row2 col0\" >The Wings entered a new era, following the retirement of which Canadian retired professional ice hockey player and current general manager of the Tampa Bay...</td>\n",
       "      <td id=\"T_aaaf4_row2_col1\" class=\"data row2 col1\" >Steve Yzerman</td>\n",
       "      <td id=\"T_aaaf4_row2_col2\" class=\"data row2 col2\" >{'2006–07 Detroit Red Wings season', 'Steve Yzerman'}</td>\n",
       "      <td id=\"T_aaaf4_row2_col3\" class=\"data row2 col3\" >Steve Yzerman</td>\n",
       "      <td id=\"T_aaaf4_row2_col4\" class=\"data row2 col4\" >✔️ [True]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_aaaf4_level0_row3\" class=\"row_heading level0 row3\" >3</th>\n",
       "      <td id=\"T_aaaf4_row3_col0\" class=\"data row3 col0\" >What river is near the Crichton Collegiate Church?</td>\n",
       "      <td id=\"T_aaaf4_row3_col1\" class=\"data row3 col1\" >the River Tyne</td>\n",
       "      <td id=\"T_aaaf4_row3_col2\" class=\"data row3 col2\" >{'Crichton Collegiate Church', 'Crichton Castle'}</td>\n",
       "      <td id=\"T_aaaf4_row3_col3\" class=\"data row3 col3\" >Tyne River</td>\n",
       "      <td id=\"T_aaaf4_row3_col4\" class=\"data row3 col4\" >❌ [False]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_aaaf4_level0_row4\" class=\"row_heading level0 row4\" >4</th>\n",
       "      <td id=\"T_aaaf4_row4_col0\" class=\"data row4 col0\" >In the 10th Century A.D. Ealhswith had a son called Æthelweard by which English king?</td>\n",
       "      <td id=\"T_aaaf4_row4_col1\" class=\"data row4 col1\" >King Alfred the Great</td>\n",
       "      <td id=\"T_aaaf4_row4_col2\" class=\"data row4 col2\" >{'Æthelweard (son of Alfred)', 'Ealhswith'}</td>\n",
       "      <td id=\"T_aaaf4_row4_col3\" class=\"data row4 col3\" >Alfred the Great</td>\n",
       "      <td id=\"T_aaaf4_row4_col4\" class=\"data row4 col4\" >❌ [False]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n"
      ],
      "text/plain": [
       "<pandas.io.formats.style.Styler at 0x7f2d430888b0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "                <div style='\n",
       "                    text-align: center; \n",
       "                    font-size: 16px; \n",
       "                    font-weight: bold; \n",
       "                    color: #555; \n",
       "                    margin: 10px 0;'>\n",
       "                    ... 45 more rows not displayed ...\n",
       "                </div>\n",
       "                "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "(46.0,\n",
       "           question  example_answer     gold_titles     pred_answer  answer_exact_match\n",
       " 0   Are both Ca...              no  {Cangzhou, ...              No            True    \n",
       " 1   Who conduct...  National Ho...  {2017–18 Pi...  National Ho...            True    \n",
       " 2   The Wings e...   Steve Yzerman  {2006–07 De...   Steve Yzerman            True    \n",
       " 3   What river ...  the River Tyne  {Crichton C...      Tyne River           False    \n",
       " 4   In the 10th...  King Alfred...  {Æthelweard...  Alfred the ...           False    \n",
       " 5   The Newark ...  Port Author...  {Newark Air...  Port Author...            True    \n",
       " 6   Where did a...      Bundesliga  {Claudio Pi...            Peru           False    \n",
       " 7   Are both Ch...              no  {Chico Muni...              No            True    \n",
       " 8   In which Ma...  Waldo Count...  {Stockton S...    Waldo County           False    \n",
       " 9   Which 90s r...  The Afghan ...  {Gene (band...  The Afghan ...            True    \n",
       " 10  What year d...           79 AD  {Curse of t...           79 AD            True    \n",
       " 11  Is the 72nd...      the oldest  {First Unit...          Oldest            True    \n",
       " 12  Was Stanisl...             not  {Stanisław ...             Yes           False    \n",
       " 13  Which film ...        Del Lord  {Wang Xiaos...  Wang Xiaoshuai           False    \n",
       " 14  Lord North ...  Jonathan Wi...  {Jonathan A...  Jonathan Ai...           False    \n",
       " 15  What is the...          Marche  {Marche, Po...          Marche            True    \n",
       " 16  William Hug...  7,402 at th...  {Kosciusko,...         Unknown           False    \n",
       " 17  What do stu...  design thei...  {Gallatin S...       Study Art           False    \n",
       " 18  What is the...         English  {Restaurant...         British           False    \n",
       " 19  What Americ...  Robert F. Chew  {Robert F. ...     Wood Harris           False    \n",
       " 20  What city i...      Manchester  {Toby Sawye...          London           False    \n",
       " 21  Who was bor...     Deepa Mehta  {Tony Kaye ...       Tony Kaye           False    \n",
       " 22  What is the...  the good ma...  {Boise Town...  The Bon Marché           False    \n",
       " 23  Who did Liz...  Christine C...  {Lizzette R...  Christine C...            True    \n",
       " 24  What was th...  William Str...  {P. T. Barn...   Zerah Colburn           False    \n",
       " 25  Which battl...  Battle of t...  {Meuse-Argo...  Battle of t...            True    \n",
       " 26  What cricke...      Ian Botham  {Ian Botham...  Terry Alderman           False    \n",
       " 27  What is the...  defensive a...  {1982 NC St...        Deceased           False    \n",
       " 28  Which Scott...   Ewan McGregor  {Come What ...   Ewan McGregor            True    \n",
       " 29  Where have ...           space  {Frank De W...           Space            True    \n",
       " 30  The origina...  Maria Yermo...  {Wild Honey...  Maria Yermo...            True    \n",
       " 31  Are Roswell...              no  {Pago Pago ...              No            True    \n",
       " 32  Untold: The...  the voice o...  {Marv Alber...     Gus Johnson           False    \n",
       " 33  Are Walt Di...             yes  {Sacro GRA,...              No           False    \n",
       " 34  What is the...           Hamas  {Status of ...           Hamas            True    \n",
       " 35  What album ...            1989  {Wildest Dr...            1989            True    \n",
       " 36  Which is co...           Apera  {Apera, Gun...           Apera            True    \n",
       " 37  Do The Drum...              no  {Pussy Galo...              No            True    \n",
       " 38  What is the...            Exon  {Banded Bro...             UoE           False    \n",
       " 39  Are both Be...             yes  {Len Wisema...             Yes            True    \n",
       " 40  Steven Cuit...   Bill Melendez  {Steven C. ...   Bill Melendez            True    \n",
       " 41  Shark Creek...  Clarence River  {Clarence R...  Clarence River            True    \n",
       " 42  Who was the...           Pixar  {Finding Do...  Lindsey Col...           False    \n",
       " 43  Who purchas...         Renault  {Benetton F...       Jean Todt           False    \n",
       " 44  Fredrick La...  Cadwalader ...  {Frederick ...  Cadwalader ...            True    \n",
       " 45  Gordon Warn...  \"Forza Ital...  {Franco Zef...  Democratic ...           False    \n",
       " 46  André Zucca...   the Wehrmacht  {André Zucc...  The Propaga...           False    \n",
       " 47  Both Bill P...       cricketer  {Bill Ponsf...         Cricket           False    \n",
       " 48   Suzana S. ...   Danny Wallace  {Yes Man (f...    James Ellroy           False    \n",
       " 49  In what cit...        Portland  {Election L...        New York           False    )"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from dspy.evaluate.evaluate import Evaluate\n",
    "\n",
    "# Set up the `evaluate_on_hotpotqa` function. We'll use this many times below.\n",
    "evaluate_on_hotpotqa = Evaluate(devset=devset, num_threads=1, display_progress=True, display_table=5)\n",
    "\n",
    "# Evaluate the `compiled_knn` program with the `answer_exact_match` metric.\n",
    "metric = dspy.evaluate.answer_exact_match\n",
    "\n",
    "    \n",
    "evaluate_on_hotpotqa(compiled_knn, metric)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.9.18 64-bit ('pipenv')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.18"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "1d3dd57b136d599382d2667f19e9aed8a929301e79008dd54c75c965d79714ea"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}