openai/openai-python
Publicmirrored fromhttps://github.com/openai/openai-pythonAvailable
examples/embeddings/Code_search.ipynb
396lines · modecode
| 1 | { |
| 2 | "cells": [ |
| 3 | { |
| 4 | "cell_type": "markdown", |
| 5 | "metadata": {}, |
| 6 | "source": [ |
| 7 | "## Code search\n", |
| 8 | "\n", |
| 9 | "We index our own openai-python code repository, and show how it can be searched. We implement a simple version of file parsing and extracting of functions from python files. The dataset is created in the [Obtain_dataset Notebook](Obtain_dataset.ipynb)." |
| 10 | ] |
| 11 | }, |
| 12 | { |
| 13 | "cell_type": "code", |
| 14 | "execution_count": 1, |
| 15 | "metadata": {}, |
| 16 | "outputs": [ |
| 17 | { |
| 18 | "name": "stdout", |
| 19 | "output_type": "stream", |
| 20 | "text": [ |
| 21 | "Total number of py files: 40\n", |
| 22 | "Total number of functions extracted: 64\n" |
| 23 | ] |
| 24 | } |
| 25 | ], |
| 26 | "source": [ |
| 27 | "import os\n", |
| 28 | "from glob import glob\n", |
| 29 | "import pandas as pd\n", |
| 30 | "\n", |
| 31 | "def get_function_name(code):\n", |
| 32 | " \"\"\"\n", |
| 33 | " Extract function name from a line beginning with \"def \"\n", |
| 34 | " \"\"\"\n", |
| 35 | " assert code.startswith(\"def \")\n", |
| 36 | " return code[len(\"def \"): code.index(\"(\")]\n", |
| 37 | "\n", |
| 38 | "def get_until_no_space(all_lines, i) -> str:\n", |
| 39 | " \"\"\"\n", |
| 40 | " Get all lines until a line outside the function definition is found.\n", |
| 41 | " \"\"\"\n", |
| 42 | " ret = [all_lines[i]]\n", |
| 43 | " for j in range(i + 1, i + 10000):\n", |
| 44 | " if j < len(all_lines):\n", |
| 45 | " if len(all_lines[j]) == 0 or all_lines[j][0] in [\" \", \"\\t\", \")\"]:\n", |
| 46 | " ret.append(all_lines[j])\n", |
| 47 | " else:\n", |
| 48 | " break\n", |
| 49 | " return \"\\n\".join(ret)\n", |
| 50 | "\n", |
| 51 | "def get_functions(filepath):\n", |
| 52 | " \"\"\"\n", |
| 53 | " Get all functions in a Python file.\n", |
| 54 | " \"\"\"\n", |
| 55 | " whole_code = open(filepath).read().replace(\"\\r\", \"\\n\")\n", |
| 56 | " all_lines = whole_code.split(\"\\n\")\n", |
| 57 | " for i, l in enumerate(all_lines):\n", |
| 58 | " if l.startswith(\"def \"):\n", |
| 59 | " code = get_until_no_space(all_lines, i)\n", |
| 60 | " function_name = get_function_name(code)\n", |
| 61 | " yield {\"code\": code, \"function_name\": function_name, \"filepath\": filepath}\n", |
| 62 | "\n", |
| 63 | "\n", |
| 64 | "# get user root directory\n", |
| 65 | "root_dir = os.path.expanduser(\"~\")\n", |
| 66 | "\n", |
| 67 | "# path to code repository directory\n", |
| 68 | "code_root = root_dir + \"/openai-python\"\n", |
| 69 | "code_files = [y for x in os.walk(code_root) for y in glob(os.path.join(x[0], '*.py'))]\n", |
| 70 | "print(\"Total number of py files:\", len(code_files))\n", |
| 71 | "all_funcs = []\n", |
| 72 | "for code_file in code_files:\n", |
| 73 | " funcs = list(get_functions(code_file))\n", |
| 74 | " for func in funcs:\n", |
| 75 | " all_funcs.append(func)\n", |
| 76 | "\n", |
| 77 | "print(\"Total number of functions extracted:\", len(all_funcs))\n" |
| 78 | ] |
| 79 | }, |
| 80 | { |
| 81 | "cell_type": "markdown", |
| 82 | "metadata": {}, |
| 83 | "source": [ |
| 84 | "For code search models we use code-search-{model}-code to obtain embeddings for code snippets, and code-search-{model}-text to embed natural language queries." |
| 85 | ] |
| 86 | }, |
| 87 | { |
| 88 | "cell_type": "code", |
| 89 | "execution_count": 2, |
| 90 | "metadata": {}, |
| 91 | "outputs": [ |
| 92 | { |
| 93 | "data": { |
| 94 | "text/html": [ |
| 95 | "<div>\n", |
| 96 | "<style scoped>\n", |
| 97 | " .dataframe tbody tr th:only-of-type {\n", |
| 98 | " vertical-align: middle;\n", |
| 99 | " }\n", |
| 100 | "\n", |
| 101 | " .dataframe tbody tr th {\n", |
| 102 | " vertical-align: top;\n", |
| 103 | " }\n", |
| 104 | "\n", |
| 105 | " .dataframe thead th {\n", |
| 106 | " text-align: right;\n", |
| 107 | " }\n", |
| 108 | "</style>\n", |
| 109 | "<table border=\"1\" class=\"dataframe\">\n", |
| 110 | " <thead>\n", |
| 111 | " <tr style=\"text-align: right;\">\n", |
| 112 | " <th></th>\n", |
| 113 | " <th>code</th>\n", |
| 114 | " <th>function_name</th>\n", |
| 115 | " <th>filepath</th>\n", |
| 116 | " <th>code_embedding</th>\n", |
| 117 | " </tr>\n", |
| 118 | " </thead>\n", |
| 119 | " <tbody>\n", |
| 120 | " <tr>\n", |
| 121 | " <th>0</th>\n", |
| 122 | " <td>def semantic_search(engine, query, documents):...</td>\n", |
| 123 | " <td>semantic_search</td>\n", |
| 124 | " <td>/examples/semanticsearch/semanticsearch.py</td>\n", |
| 125 | " <td>[-0.038976121693849564, -0.0031428150832653046...</td>\n", |
| 126 | " </tr>\n", |
| 127 | " <tr>\n", |
| 128 | " <th>1</th>\n", |
| 129 | " <td>def main():\\n parser = argparse.ArgumentPar...</td>\n", |
| 130 | " <td>main</td>\n", |
| 131 | " <td>/examples/semanticsearch/semanticsearch.py</td>\n", |
| 132 | " <td>[-0.024289356544613838, -0.017748363316059113,...</td>\n", |
| 133 | " </tr>\n", |
| 134 | " <tr>\n", |
| 135 | " <th>2</th>\n", |
| 136 | " <td>def get_candidates(\\n prompt: str,\\n sto...</td>\n", |
| 137 | " <td>get_candidates</td>\n", |
| 138 | " <td>/examples/codex/backtranslation.py</td>\n", |
| 139 | " <td>[-0.04161201789975166, -0.0169310811907053, 0....</td>\n", |
| 140 | " </tr>\n", |
| 141 | " <tr>\n", |
| 142 | " <th>3</th>\n", |
| 143 | " <td>def rindex(lst: List, value: str) -> int:\\n ...</td>\n", |
| 144 | " <td>rindex</td>\n", |
| 145 | " <td>/examples/codex/backtranslation.py</td>\n", |
| 146 | " <td>[-0.027255680412054062, -0.007931121625006199,...</td>\n", |
| 147 | " </tr>\n", |
| 148 | " <tr>\n", |
| 149 | " <th>4</th>\n", |
| 150 | " <td>def eval_candidate(\\n candidate_answer: str...</td>\n", |
| 151 | " <td>eval_candidate</td>\n", |
| 152 | " <td>/examples/codex/backtranslation.py</td>\n", |
| 153 | " <td>[-0.00999179296195507, -0.01640152558684349, 0...</td>\n", |
| 154 | " </tr>\n", |
| 155 | " </tbody>\n", |
| 156 | "</table>\n", |
| 157 | "</div>" |
| 158 | ], |
| 159 | "text/plain": [ |
| 160 | " code function_name \\\n", |
| 161 | "0 def semantic_search(engine, query, documents):... semantic_search \n", |
| 162 | "1 def main():\\n parser = argparse.ArgumentPar... main \n", |
| 163 | "2 def get_candidates(\\n prompt: str,\\n sto... get_candidates \n", |
| 164 | "3 def rindex(lst: List, value: str) -> int:\\n ... rindex \n", |
| 165 | "4 def eval_candidate(\\n candidate_answer: str... eval_candidate \n", |
| 166 | "\n", |
| 167 | " filepath \\\n", |
| 168 | "0 /examples/semanticsearch/semanticsearch.py \n", |
| 169 | "1 /examples/semanticsearch/semanticsearch.py \n", |
| 170 | "2 /examples/codex/backtranslation.py \n", |
| 171 | "3 /examples/codex/backtranslation.py \n", |
| 172 | "4 /examples/codex/backtranslation.py \n", |
| 173 | "\n", |
| 174 | " code_embedding \n", |
| 175 | "0 [-0.038976121693849564, -0.0031428150832653046... \n", |
| 176 | "1 [-0.024289356544613838, -0.017748363316059113,... \n", |
| 177 | "2 [-0.04161201789975166, -0.0169310811907053, 0.... \n", |
| 178 | "3 [-0.027255680412054062, -0.007931121625006199,... \n", |
| 179 | "4 [-0.00999179296195507, -0.01640152558684349, 0... " |
| 180 | ] |
| 181 | }, |
| 182 | "execution_count": 2, |
| 183 | "metadata": {}, |
| 184 | "output_type": "execute_result" |
| 185 | } |
| 186 | ], |
| 187 | "source": [ |
| 188 | "from openai.embeddings_utils import get_embedding\n", |
| 189 | "\n", |
| 190 | "df = pd.DataFrame(all_funcs)\n", |
| 191 | "df['code_embedding'] = df['code'].apply(lambda x: get_embedding(x, engine='code-search-babbage-code-001'))\n", |
| 192 | "df['filepath'] = df['filepath'].apply(lambda x: x.replace(code_root, \"\"))\n", |
| 193 | "df.to_csv(\"output/code_search_openai-python.csv\", index=False)\n", |
| 194 | "df.head()" |
| 195 | ] |
| 196 | }, |
| 197 | { |
| 198 | "cell_type": "code", |
| 199 | "execution_count": 5, |
| 200 | "metadata": {}, |
| 201 | "outputs": [ |
| 202 | { |
| 203 | "name": "stdout", |
| 204 | "output_type": "stream", |
| 205 | "text": [ |
| 206 | "/openai/tests/test_endpoints.py:test_completions_multiple_prompts score=0.681\n", |
| 207 | "def test_completions_multiple_prompts():\n", |
| 208 | " result = openai.Completion.create(\n", |
| 209 | " prompt=[\"This was a test\", \"This was another test\"], n=5, engine=\"ada\"\n", |
| 210 | " )\n", |
| 211 | " assert len(result.choices) == 10\n", |
| 212 | "\n", |
| 213 | "----------------------------------------------------------------------\n", |
| 214 | "/openai/tests/test_endpoints.py:test_completions score=0.675\n", |
| 215 | "def test_completions():\n", |
| 216 | " result = openai.Completion.create(prompt=\"This was a test\", n=5, engine=\"ada\")\n", |
| 217 | " assert len(result.choices) == 5\n", |
| 218 | "\n", |
| 219 | "\n", |
| 220 | "----------------------------------------------------------------------\n", |
| 221 | "/openai/tests/test_api_requestor.py:test_requestor_sets_request_id score=0.635\n", |
| 222 | "def test_requestor_sets_request_id(mocker: MockerFixture) -> None:\n", |
| 223 | " # Fake out 'requests' and confirm that the X-Request-Id header is set.\n", |
| 224 | "\n", |
| 225 | " got_headers = {}\n", |
| 226 | "\n", |
| 227 | " def fake_request(self, *args, **kwargs):\n", |
| 228 | " nonlocal got_headers\n", |
| 229 | "----------------------------------------------------------------------\n" |
| 230 | ] |
| 231 | } |
| 232 | ], |
| 233 | "source": [ |
| 234 | "from openai.embeddings_utils import cosine_similarity\n", |
| 235 | "\n", |
| 236 | "def search_functions(df, code_query, n=3, pprint=True, n_lines=7):\n", |
| 237 | " embedding = get_embedding(code_query, engine='code-search-babbage-text-001')\n", |
| 238 | " df['similarities'] = df.code_embedding.apply(lambda x: cosine_similarity(x, embedding))\n", |
| 239 | "\n", |
| 240 | " res = df.sort_values('similarities', ascending=False).head(n)\n", |
| 241 | " if pprint:\n", |
| 242 | " for r in res.iterrows():\n", |
| 243 | " print(r[1].filepath+\":\"+r[1].function_name + \" score=\" + str(round(r[1].similarities, 3)))\n", |
| 244 | " print(\"\\n\".join(r[1].code.split(\"\\n\")[:n_lines]))\n", |
| 245 | " print('-'*70)\n", |
| 246 | " return res\n", |
| 247 | "res = search_functions(df, 'Completions API tests', n=3)\n" |
| 248 | ] |
| 249 | }, |
| 250 | { |
| 251 | "cell_type": "code", |
| 252 | "execution_count": 6, |
| 253 | "metadata": {}, |
| 254 | "outputs": [ |
| 255 | { |
| 256 | "name": "stdout", |
| 257 | "output_type": "stream", |
| 258 | "text": [ |
| 259 | "/openai/validators.py:format_inferrer_validator score=0.655\n", |
| 260 | "def format_inferrer_validator(df):\n", |
| 261 | " \"\"\"\n", |
| 262 | " This validator will infer the likely fine-tuning format of the data, and display it to the user if it is classification.\n", |
| 263 | " It will also suggest to use ada and explain train/validation split benefits.\n", |
| 264 | " \"\"\"\n", |
| 265 | " ft_type = infer_task_type(df)\n", |
| 266 | " immediate_msg = None\n", |
| 267 | "----------------------------------------------------------------------\n", |
| 268 | "/openai/validators.py:long_examples_validator score=0.649\n", |
| 269 | "def long_examples_validator(df):\n", |
| 270 | " \"\"\"\n", |
| 271 | " This validator will suggest to the user to remove examples that are too long.\n", |
| 272 | " \"\"\"\n", |
| 273 | " immediate_msg = None\n", |
| 274 | " optional_msg = None\n", |
| 275 | " optional_fn = None\n", |
| 276 | "----------------------------------------------------------------------\n", |
| 277 | "/openai/validators.py:non_empty_completion_validator score=0.646\n", |
| 278 | "def non_empty_completion_validator(df):\n", |
| 279 | " \"\"\"\n", |
| 280 | " This validator will ensure that no completion is empty.\n", |
| 281 | " \"\"\"\n", |
| 282 | " necessary_msg = None\n", |
| 283 | " necessary_fn = None\n", |
| 284 | " immediate_msg = None\n", |
| 285 | "----------------------------------------------------------------------\n" |
| 286 | ] |
| 287 | } |
| 288 | ], |
| 289 | "source": [ |
| 290 | "res = search_functions(df, 'fine-tuning input data validation logic', n=3)" |
| 291 | ] |
| 292 | }, |
| 293 | { |
| 294 | "cell_type": "code", |
| 295 | "execution_count": 7, |
| 296 | "metadata": {}, |
| 297 | "outputs": [ |
| 298 | { |
| 299 | "name": "stdout", |
| 300 | "output_type": "stream", |
| 301 | "text": [ |
| 302 | "/openai/validators.py:common_completion_suffix_validator score=0.665\n", |
| 303 | "def common_completion_suffix_validator(df):\n", |
| 304 | " \"\"\"\n", |
| 305 | " This validator will suggest to add a common suffix to the completion if one doesn't already exist in case of classification or conditional generation.\n", |
| 306 | " \"\"\"\n", |
| 307 | " error_msg = None\n", |
| 308 | " immediate_msg = None\n", |
| 309 | " optional_msg = None\n", |
| 310 | " optional_fn = None\n", |
| 311 | "\n", |
| 312 | " ft_type = infer_task_type(df)\n", |
| 313 | "----------------------------------------------------------------------\n", |
| 314 | "/openai/validators.py:get_outfnames score=0.66\n", |
| 315 | "def get_outfnames(fname, split):\n", |
| 316 | " suffixes = [\"_train\", \"_valid\"] if split else [\"\"]\n", |
| 317 | " i = 0\n", |
| 318 | " while True:\n", |
| 319 | " index_suffix = f\" ({i})\" if i > 0 else \"\"\n", |
| 320 | " candidate_fnames = [\n", |
| 321 | " fname.split(\".\")[0] + \"_prepared\" + suffix + index_suffix + \".jsonl\"\n", |
| 322 | " for suffix in suffixes\n", |
| 323 | " ]\n", |
| 324 | " if not any(os.path.isfile(f) for f in candidate_fnames):\n", |
| 325 | "----------------------------------------------------------------------\n" |
| 326 | ] |
| 327 | } |
| 328 | ], |
| 329 | "source": [ |
| 330 | "res = search_functions(df, 'find common suffix', n=2, n_lines=10)" |
| 331 | ] |
| 332 | }, |
| 333 | { |
| 334 | "cell_type": "code", |
| 335 | "execution_count": 8, |
| 336 | "metadata": {}, |
| 337 | "outputs": [ |
| 338 | { |
| 339 | "name": "stdout", |
| 340 | "output_type": "stream", |
| 341 | "text": [ |
| 342 | "/openai/cli.py:tools_register score=0.651\n", |
| 343 | "def tools_register(parser):\n", |
| 344 | " subparsers = parser.add_subparsers(\n", |
| 345 | " title=\"Tools\", help=\"Convenience client side tools\"\n", |
| 346 | " )\n", |
| 347 | "\n", |
| 348 | " def help(args):\n", |
| 349 | " parser.print_help()\n", |
| 350 | "\n", |
| 351 | " parser.set_defaults(func=help)\n", |
| 352 | "\n", |
| 353 | " sub = subparsers.add_parser(\"fine_tunes.prepare_data\")\n", |
| 354 | " sub.add_argument(\n", |
| 355 | " \"-f\",\n", |
| 356 | " \"--file\",\n", |
| 357 | " required=True,\n", |
| 358 | " help=\"JSONL, JSON, CSV, TSV, TXT or XLSX file containing prompt-completion examples to be analyzed.\"\n", |
| 359 | " \"This should be the local file path.\",\n", |
| 360 | " )\n", |
| 361 | " sub.add_argument(\n", |
| 362 | " \"-q\",\n", |
| 363 | "----------------------------------------------------------------------\n" |
| 364 | ] |
| 365 | } |
| 366 | ], |
| 367 | "source": [ |
| 368 | "res = search_functions(df, 'Command line interface for fine-tuning', n=1, n_lines=20)" |
| 369 | ] |
| 370 | } |
| 371 | ], |
| 372 | "metadata": { |
| 373 | "interpreter": { |
| 374 | "hash": "be4b5d5b73a21c599de40d6deb1129796d12dc1cc33a738f7bac13269cfcafe8" |
| 375 | }, |
| 376 | "kernelspec": { |
| 377 | "display_name": "Python 3.7.3 64-bit ('base': conda)", |
| 378 | "name": "python3" |
| 379 | }, |
| 380 | "language_info": { |
| 381 | "codemirror_mode": { |
| 382 | "name": "ipython", |
| 383 | "version": 3 |
| 384 | }, |
| 385 | "file_extension": ".py", |
| 386 | "mimetype": "text/x-python", |
| 387 | "name": "python", |
| 388 | "nbconvert_exporter": "python", |
| 389 | "pygments_lexer": "ipython3", |
| 390 | "version": "3.7.3" |
| 391 | }, |
| 392 | "orig_nbformat": 4 |
| 393 | }, |
| 394 | "nbformat": 4, |
| 395 | "nbformat_minor": 2 |
| 396 | } |
| 397 | |