Merge pull request #9 from AnswerDotAI/read

read_docs walks up parent paths if llms.txt not found.

03_download.ipynb

@@ -45,7 +45,8 @@
  "metadata": {},
  "outputs": [],
  "source": [
- "from IPython.display import Markdown,HTML"
+ "from IPython.display import Markdown,HTML\n",
+ "from fastcore.test import *"
  ]
  },
  {
@@ -236,7 +237,7 @@
  " url = (base+path+fname).strip('/')\n",
  " if fname=='/llms.txt': return url\n",
  " if Path(fname).suffix in('.md', '.txt', '.rst'): return _tryget(url)\n",
- " if '.' in fname: return _tryget(url+'.md')\n",
+ " if '.' in fname: return _tryget(url+'.md') or find_docs(url[:url.rfind('/')])\n",
  " res = _tryget(url+'/llms.txt')\n",
  " if res: return res\n",
  " res = _tryget(url+'/index.md')\n",
@@ -245,7 +246,9 @@
  " if res: return res\n",
  " res = _tryget(url+'/index-commonmark.md')\n",
  " if res: return res\n",
- " return None"
+ " parsed_url = urlparse(url)\n",
+ " if parsed_url.path == '/' or not parsed_url.path: return None\n",
+ " return find_docs(urljoin(url, '..'))"
  ]
  },
  {
@@ -289,7 +292,7 @@
  "name": "stdout",
  "output_type": "stream",
  "text": [
- "None\n",
+ "https://claudette.answer.ai/index.html.md\n",
  "https://claudette.answer.ai/index.html.md\n",
  "https://claudette.answer.ai/index.html.md\n",
  "https://llmstxt.org/llms.txt\n",
@@ -301,6 +304,30 @@
  "for o in urls: print(find_docs(o))"
  ]
  },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "439546d4",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "suffixes = [\"/\", \"/tmp\", \"/tmp/\", \"/tmp/tmp\", \"/tmp/tmp/\"]\n",
+ "for suff in suffixes:\n",
+ " for o in urls: test_eq(find_docs(o), find_docs(o+suff))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "07d1b763",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "test_eq(find_docs(\"https://github.com\"), \"https://github.com/llms.txt\")\n",
+ "test_eq(find_docs(\"https://github.com/AnswerDotAI\"), \"https://github.com/llms.txt\")\n",
+ "test_eq(find_docs(\"https://github.com/AnswerDotAI/\"), \"https://github.com/llms.txt\")"
+ ]
+ },
  {
  "cell_type": "code",
  "execution_count": null,
@@ -312,7 +339,6 @@
  "def read_docs(url, optional=False, n_workers=None, rm_comments=True, rm_details=True):\n",
  " \"If available, return LLM-friendly llms.txt context or markdown file response for `url`\"\n",
  " url = find_docs(url)\n",
- " if not url: return\n",
  " if url.endswith('/llms.txt'): res = get_llmstxt(url, optional=optional, n_workers=n_workers)\n",
  " else: res = get(url).text\n",
  " return clean_md(res, rm_comments=rm_comments, rm_details=rm_details)"

toolslm/download.py

@@ -76,7 +76,7 @@ def find_docs(url):
  url = (base+path+fname).strip('/')
  if fname=='/llms.txt': return url
  if Path(fname).suffix in('.md', '.txt', '.rst'): return _tryget(url)
- if '.' in fname: return _tryget(url+'.md')
+ if '.' in fname: return _tryget(url+'.md') or find_docs(url[:url.rfind('/')])
  res = _tryget(url+'/llms.txt')
  if res: return res
  res = _tryget(url+'/index.md')
@@ -85,13 +85,14 @@ def find_docs(url):
  if res: return res
  res = _tryget(url+'/index-commonmark.md')
  if res: return res
- return None
+ parsed_url = urlparse(url)
+ if parsed_url.path == '/' or not parsed_url.path: return None
+ return find_docs(urljoin(url, '..'))
 
-# %% ../03_download.ipynb 19
+# %% ../03_download.ipynb 21
 def read_docs(url, optional=False, n_workers=None, rm_comments=True, rm_details=True):
  "If available, return LLM-friendly llms.txt context or markdown file response for `url`"
  url = find_docs(url)
- if not url: return
  if url.endswith('/llms.txt'): res = get_llmstxt(url, optional=optional, n_workers=n_workers)
  else: res = get(url).text
  return clean_md(res, rm_comments=rm_comments, rm_details=rm_details)