GitVita
New repositoryImport repositoryNew issue
Your repositories Your dashboard Sign in

openai/openai-python

Public

mirrored from https://github.com/openai/openai-pythonAvailable

Watch0Fork0Star0
CodeCommitsIssuesPull requestsActionsInsightsSecurity
v0.20.0

Branches

Tags

  • No tags available.
0Branches0Tags
Go to file
Add file
Code

Clone

HTTPS

Download ZIP

openai-python/examples/embeddings

examples/embeddings/Code_search.ipynb

396lines · modeblame

RawDownload
Latest commit unavailable.
1f324723Boris Power4 years ago1{
2"cells": [
3{
4"cell_type": "markdown",
5"metadata": {},
6"source": [
7"## Code search\n",
8"\n",
9"We index our own openai-python code repository, and show how it can be searched. We implement a simple version of file parsing and extracting of functions from python files. The dataset is created in the [Obtain_dataset Notebook](Obtain_dataset.ipynb)."
10]
11},
12{
13"cell_type": "code",
14"execution_count": 1,
15"metadata": {},
16"outputs": [
17{
18"name": "stdout",
19"output_type": "stream",
20"text": [
21"Total number of py files: 40\n",
22"Total number of functions extracted: 64\n"
23]
24}
25],
26"source": [
27"import os\n",
28"from glob import glob\n",
29"import pandas as pd\n",
30"\n",
31"def get_function_name(code):\n",
32" \"\"\"\n",
33" Extract function name from a line beginning with \"def \"\n",
34" \"\"\"\n",
35" assert code.startswith(\"def \")\n",
36" return code[len(\"def \"): code.index(\"(\")]\n",
37"\n",
38"def get_until_no_space(all_lines, i) -> str:\n",
39" \"\"\"\n",
40" Get all lines until a line outside the function definition is found.\n",
41" \"\"\"\n",
42" ret = [all_lines[i]]\n",
43" for j in range(i + 1, i + 10000):\n",
44" if j < len(all_lines):\n",
45" if len(all_lines[j]) == 0 or all_lines[j][0] in [\" \", \"\\t\", \")\"]:\n",
46" ret.append(all_lines[j])\n",
47" else:\n",
48" break\n",
49" return \"\\n\".join(ret)\n",
50"\n",
51"def get_functions(filepath):\n",
52" \"\"\"\n",
53" Get all functions in a Python file.\n",
54" \"\"\"\n",
55" whole_code = open(filepath).read().replace(\"\\r\", \"\\n\")\n",
56" all_lines = whole_code.split(\"\\n\")\n",
57" for i, l in enumerate(all_lines):\n",
58" if l.startswith(\"def \"):\n",
59" code = get_until_no_space(all_lines, i)\n",
60" function_name = get_function_name(code)\n",
61" yield {\"code\": code, \"function_name\": function_name, \"filepath\": filepath}\n",
62"\n",
63"\n",
64"# get user root directory\n",
65"root_dir = os.path.expanduser(\"~\")\n",
66"\n",
67"# path to code repository directory\n",
68"code_root = root_dir + \"/openai-python\"\n",
69"code_files = [y for x in os.walk(code_root) for y in glob(os.path.join(x[0], '*.py'))]\n",
70"print(\"Total number of py files:\", len(code_files))\n",
71"all_funcs = []\n",
72"for code_file in code_files:\n",
73" funcs = list(get_functions(code_file))\n",
74" for func in funcs:\n",
75" all_funcs.append(func)\n",
76"\n",
77"print(\"Total number of functions extracted:\", len(all_funcs))\n"
78]
79},
80{
81"cell_type": "markdown",
82"metadata": {},
83"source": [
eabf01f0Ted Sanders4 years ago84"For code search models we use code-search-{model}-code to obtain embeddings for code snippets, and code-search-{model}-text to embed natural language queries."
1f324723Boris Power4 years ago85]
86},
87{
88"cell_type": "code",
89"execution_count": 2,
90"metadata": {},
91"outputs": [
92{
93"data": {
94"text/html": [
95"<div>\n",
96"<style scoped>\n",
97" .dataframe tbody tr th:only-of-type {\n",
98" vertical-align: middle;\n",
99" }\n",
100"\n",
101" .dataframe tbody tr th {\n",
102" vertical-align: top;\n",
103" }\n",
104"\n",
105" .dataframe thead th {\n",
106" text-align: right;\n",
107" }\n",
108"</style>\n",
109"<table border=\"1\" class=\"dataframe\">\n",
110" <thead>\n",
111" <tr style=\"text-align: right;\">\n",
112" <th></th>\n",
113" <th>code</th>\n",
114" <th>function_name</th>\n",
115" <th>filepath</th>\n",
116" <th>code_embedding</th>\n",
117" </tr>\n",
118" </thead>\n",
119" <tbody>\n",
120" <tr>\n",
121" <th>0</th>\n",
122" <td>def semantic_search(engine, query, documents):...</td>\n",
123" <td>semantic_search</td>\n",
124" <td>/examples/semanticsearch/semanticsearch.py</td>\n",
125" <td>[-0.038976121693849564, -0.0031428150832653046...</td>\n",
126" </tr>\n",
127" <tr>\n",
128" <th>1</th>\n",
129" <td>def main():\\n parser = argparse.ArgumentPar...</td>\n",
130" <td>main</td>\n",
131" <td>/examples/semanticsearch/semanticsearch.py</td>\n",
132" <td>[-0.024289356544613838, -0.017748363316059113,...</td>\n",
133" </tr>\n",
134" <tr>\n",
135" <th>2</th>\n",
136" <td>def get_candidates(\\n prompt: str,\\n sto...</td>\n",
137" <td>get_candidates</td>\n",
138" <td>/examples/codex/backtranslation.py</td>\n",
139" <td>[-0.04161201789975166, -0.0169310811907053, 0....</td>\n",
140" </tr>\n",
141" <tr>\n",
142" <th>3</th>\n",
143" <td>def rindex(lst: List, value: str) -&gt; int:\\n ...</td>\n",
144" <td>rindex</td>\n",
145" <td>/examples/codex/backtranslation.py</td>\n",
146" <td>[-0.027255680412054062, -0.007931121625006199,...</td>\n",
147" </tr>\n",
148" <tr>\n",
149" <th>4</th>\n",
150" <td>def eval_candidate(\\n candidate_answer: str...</td>\n",
151" <td>eval_candidate</td>\n",
152" <td>/examples/codex/backtranslation.py</td>\n",
153" <td>[-0.00999179296195507, -0.01640152558684349, 0...</td>\n",
154" </tr>\n",
155" </tbody>\n",
156"</table>\n",
157"</div>"
158],
159"text/plain": [
160" code function_name \\\n",
161"0 def semantic_search(engine, query, documents):... semantic_search \n",
162"1 def main():\\n parser = argparse.ArgumentPar... main \n",
163"2 def get_candidates(\\n prompt: str,\\n sto... get_candidates \n",
164"3 def rindex(lst: List, value: str) -> int:\\n ... rindex \n",
165"4 def eval_candidate(\\n candidate_answer: str... eval_candidate \n",
166"\n",
167" filepath \\\n",
168"0 /examples/semanticsearch/semanticsearch.py \n",
169"1 /examples/semanticsearch/semanticsearch.py \n",
170"2 /examples/codex/backtranslation.py \n",
171"3 /examples/codex/backtranslation.py \n",
172"4 /examples/codex/backtranslation.py \n",
173"\n",
174" code_embedding \n",
175"0 [-0.038976121693849564, -0.0031428150832653046... \n",
176"1 [-0.024289356544613838, -0.017748363316059113,... \n",
177"2 [-0.04161201789975166, -0.0169310811907053, 0.... \n",
178"3 [-0.027255680412054062, -0.007931121625006199,... \n",
179"4 [-0.00999179296195507, -0.01640152558684349, 0... "
180]
181},
182"execution_count": 2,
183"metadata": {},
184"output_type": "execute_result"
185}
186],
187"source": [
b39bddd9hallacy4 years ago188"from openai.embeddings_utils import get_embedding\n",
1f324723Boris Power4 years ago189"\n",
190"df = pd.DataFrame(all_funcs)\n",
eabf01f0Ted Sanders4 years ago191"df['code_embedding'] = df['code'].apply(lambda x: get_embedding(x, engine='code-search-babbage-code-001'))\n",
1f324723Boris Power4 years ago192"df['filepath'] = df['filepath'].apply(lambda x: x.replace(code_root, \"\"))\n",
193"df.to_csv(\"output/code_search_openai-python.csv\", index=False)\n",
194"df.head()"
195]
196},
197{
198"cell_type": "code",
199"execution_count": 5,
200"metadata": {},
201"outputs": [
202{
203"name": "stdout",
204"output_type": "stream",
205"text": [
206"/openai/tests/test_endpoints.py:test_completions_multiple_prompts score=0.681\n",
207"def test_completions_multiple_prompts():\n",
208" result = openai.Completion.create(\n",
209" prompt=[\"This was a test\", \"This was another test\"], n=5, engine=\"ada\"\n",
210" )\n",
211" assert len(result.choices) == 10\n",
212"\n",
213"----------------------------------------------------------------------\n",
214"/openai/tests/test_endpoints.py:test_completions score=0.675\n",
215"def test_completions():\n",
216" result = openai.Completion.create(prompt=\"This was a test\", n=5, engine=\"ada\")\n",
217" assert len(result.choices) == 5\n",
218"\n",
219"\n",
220"----------------------------------------------------------------------\n",
221"/openai/tests/test_api_requestor.py:test_requestor_sets_request_id score=0.635\n",
222"def test_requestor_sets_request_id(mocker: MockerFixture) -> None:\n",
223" # Fake out 'requests' and confirm that the X-Request-Id header is set.\n",
224"\n",
225" got_headers = {}\n",
226"\n",
227" def fake_request(self, *args, **kwargs):\n",
228" nonlocal got_headers\n",
229"----------------------------------------------------------------------\n"
230]
231}
232],
233"source": [
b39bddd9hallacy4 years ago234"from openai.embeddings_utils import cosine_similarity\n",
1f324723Boris Power4 years ago235"\n",
236"def search_functions(df, code_query, n=3, pprint=True, n_lines=7):\n",
eabf01f0Ted Sanders4 years ago237" embedding = get_embedding(code_query, engine='code-search-babbage-text-001')\n",
1f324723Boris Power4 years ago238" df['similarities'] = df.code_embedding.apply(lambda x: cosine_similarity(x, embedding))\n",
239"\n",
240" res = df.sort_values('similarities', ascending=False).head(n)\n",
241" if pprint:\n",
242" for r in res.iterrows():\n",
243" print(r[1].filepath+\":\"+r[1].function_name + \" score=\" + str(round(r[1].similarities, 3)))\n",
244" print(\"\\n\".join(r[1].code.split(\"\\n\")[:n_lines]))\n",
245" print('-'*70)\n",
246" return res\n",
247"res = search_functions(df, 'Completions API tests', n=3)\n"
248]
249},
250{
251"cell_type": "code",
252"execution_count": 6,
253"metadata": {},
254"outputs": [
255{
256"name": "stdout",
257"output_type": "stream",
258"text": [
259"/openai/validators.py:format_inferrer_validator score=0.655\n",
260"def format_inferrer_validator(df):\n",
261" \"\"\"\n",
262" This validator will infer the likely fine-tuning format of the data, and display it to the user if it is classification.\n",
26fbacb7Michael Wu4 years ago263" It will also suggest to use ada and explain train/validation split benefits.\n",
1f324723Boris Power4 years ago264" \"\"\"\n",
265" ft_type = infer_task_type(df)\n",
266" immediate_msg = None\n",
267"----------------------------------------------------------------------\n",
268"/openai/validators.py:long_examples_validator score=0.649\n",
269"def long_examples_validator(df):\n",
270" \"\"\"\n",
271" This validator will suggest to the user to remove examples that are too long.\n",
272" \"\"\"\n",
273" immediate_msg = None\n",
274" optional_msg = None\n",
275" optional_fn = None\n",
276"----------------------------------------------------------------------\n",
277"/openai/validators.py:non_empty_completion_validator score=0.646\n",
278"def non_empty_completion_validator(df):\n",
279" \"\"\"\n",
280" This validator will ensure that no completion is empty.\n",
281" \"\"\"\n",
282" necessary_msg = None\n",
283" necessary_fn = None\n",
284" immediate_msg = None\n",
285"----------------------------------------------------------------------\n"
286]
287}
288],
289"source": [
290"res = search_functions(df, 'fine-tuning input data validation logic', n=3)"
291]
292},
293{
294"cell_type": "code",
295"execution_count": 7,
296"metadata": {},
297"outputs": [
298{
299"name": "stdout",
300"output_type": "stream",
301"text": [
302"/openai/validators.py:common_completion_suffix_validator score=0.665\n",
303"def common_completion_suffix_validator(df):\n",
304" \"\"\"\n",
305" This validator will suggest to add a common suffix to the completion if one doesn't already exist in case of classification or conditional generation.\n",
306" \"\"\"\n",
307" error_msg = None\n",
308" immediate_msg = None\n",
309" optional_msg = None\n",
310" optional_fn = None\n",
311"\n",
312" ft_type = infer_task_type(df)\n",
313"----------------------------------------------------------------------\n",
314"/openai/validators.py:get_outfnames score=0.66\n",
315"def get_outfnames(fname, split):\n",
316" suffixes = [\"_train\", \"_valid\"] if split else [\"\"]\n",
317" i = 0\n",
318" while True:\n",
319" index_suffix = f\" ({i})\" if i > 0 else \"\"\n",
320" candidate_fnames = [\n",
321" fname.split(\".\")[0] + \"_prepared\" + suffix + index_suffix + \".jsonl\"\n",
322" for suffix in suffixes\n",
323" ]\n",
324" if not any(os.path.isfile(f) for f in candidate_fnames):\n",
325"----------------------------------------------------------------------\n"
326]
327}
328],
329"source": [
330"res = search_functions(df, 'find common suffix', n=2, n_lines=10)"
331]
332},
333{
334"cell_type": "code",
335"execution_count": 8,
336"metadata": {},
337"outputs": [
338{
339"name": "stdout",
340"output_type": "stream",
341"text": [
342"/openai/cli.py:tools_register score=0.651\n",
343"def tools_register(parser):\n",
344" subparsers = parser.add_subparsers(\n",
345" title=\"Tools\", help=\"Convenience client side tools\"\n",
346" )\n",
347"\n",
348" def help(args):\n",
349" parser.print_help()\n",
350"\n",
351" parser.set_defaults(func=help)\n",
352"\n",
353" sub = subparsers.add_parser(\"fine_tunes.prepare_data\")\n",
354" sub.add_argument(\n",
355" \"-f\",\n",
356" \"--file\",\n",
357" required=True,\n",
358" help=\"JSONL, JSON, CSV, TSV, TXT or XLSX file containing prompt-completion examples to be analyzed.\"\n",
359" \"This should be the local file path.\",\n",
360" )\n",
361" sub.add_argument(\n",
362" \"-q\",\n",
363"----------------------------------------------------------------------\n"
364]
365}
366],
367"source": [
368"res = search_functions(df, 'Command line interface for fine-tuning', n=1, n_lines=20)"
369]
370}
371],
372"metadata": {
373"interpreter": {
374"hash": "be4b5d5b73a21c599de40d6deb1129796d12dc1cc33a738f7bac13269cfcafe8"
375},
376"kernelspec": {
377"display_name": "Python 3.7.3 64-bit ('base': conda)",
378"name": "python3"
379},
380"language_info": {
381"codemirror_mode": {
382"name": "ipython",
383"version": 3
384},
385"file_extension": ".py",
386"mimetype": "text/x-python",
387"name": "python",
388"nbconvert_exporter": "python",
389"pygments_lexer": "ipython3",
390"version": "3.7.3"
391},
392"orig_nbformat": 4
393},
394"nbformat": 4,
395"nbformat_minor": 2
396}