🗂 Add site option to include folder structure in url paths

.changeset/shiny-dragons-glow.md

@@ -0,0 +1,6 @@
+---
+'myst-cli': patch
+'mystmd': patch
+---
+
+Add site option to include folder structure in url paths

packages/myst-cli/src/process/mdast.ts

@@ -262,6 +262,7 @@ export async function transformMdast(
     url = `/${useSlug ? pageSlug : ''}`;
     dataUrl = `/${pageSlug}.json`;
   }
+  url = url?.replace('.', '/');
   updateFileInfoFromFrontmatter(session, file, frontmatter, url, dataUrl);
   const data: RendererData = {
     kind: isJupytext ? SourceFileKind.Notebook : kind,

packages/myst-cli/src/process/site.ts

@@ -172,14 +172,14 @@ export async function writeMystSearchJson(session: ISession, pages: LocalProject
     .map((page) => selectFile(session, page.file))
     .map((file) => {
       const { mdast, slug, frontmatter } = file ?? {};
-      if (!mdast || !frontmatter) {
+      if (!mdast || !frontmatter || !slug) {
         return [];
       }
       const title = frontmatter.title ?? '';
 
       // Group by section (simple running accumulator)
       const sections = toSectionedParts(mdast);
-      const pageURL = slug && DEFAULT_INDEX_FILENAMES.includes(slug) ? '/' : `/${slug}`;
+      const pageURL = DEFAULT_INDEX_FILENAMES.includes(slug) ? '/' : `/${slug.replace('.', '/')}`;
       // Build sections into search records
       return sections
         .map((section, index) => {

packages/myst-cli/src/project/fromPath.ts

@@ -4,7 +4,7 @@ import { isDirectory } from 'myst-cli-utils';
 import { RuleId } from 'myst-common';
 import type { ISession } from '../session/types.js';
 import { addWarningForFile } from '../utils/addWarningForFile.js';
-import { fileInfo } from '../utils/fileInfo.js';
+import { fileInfo, fileTitle } from '../utils/fileInfo.js';
 import { nextLevel } from '../utils/nextLevel.js';
 import { VALID_FILE_EXTENSIONS, isValidFile } from '../utils/resolveExtension.js';
 import { shouldIgnoreFile } from '../utils/shouldIgnoreFile.js';
@@ -20,9 +20,10 @@ import type {
   LocalProjectPage,
   LocalProject,
   PageSlugs,
+  SlugOptions,
 } from './types.js';
 
-type Options = {
+type Options = SlugOptions & {
   ignore?: string[];
   suppressWarnings?: boolean;
 };
@@ -54,7 +55,7 @@ function projectPagesFromPath(
     try {
       // TODO: We don't yet have a way to do nested tocs with new-style toc
       session.log.debug(`Respecting legacy TOC in subdirectory: ${join(path, '_toc.yml')}`);
-      return pagesFromSphinxTOC(session, path, prevLevel);
+      return pagesFromSphinxTOC(session, path, prevLevel, opts);
     } catch {
       if (!suppressWarnings) {
         addWarningForFile(
@@ -73,15 +74,18 @@ function projectPagesFromPath(
       return {
         file,
         level,
-        slug: fileInfo(file, pageSlugs).slug,
+        slug: fileInfo(file, pageSlugs, opts).slug,
         implicit: true,
       } as LocalProjectPage;
     });
   const folders = contents
     .filter((file) => isDirectory(file))
     .sort(comparePaths)
     .map((dir) => {
-      const projectFolder: LocalProjectFolder = { title: fileInfo(dir, pageSlugs).title, level };
+      const projectFolder: LocalProjectFolder = {
+        title: fileTitle(dir),
+        level,
+      };
       const pages = projectPagesFromPath(session, dir, nextLevel(level), pageSlugs, opts);
       if (!pages.length) {
         return [];
@@ -145,13 +149,15 @@ export function projectFromPath(
   session: ISession,
   path: string,
   indexFile?: string,
+  opts?: SlugOptions,
 ): Omit<LocalProject, 'bibliography'> {
   const ext_string = VALID_FILE_EXTENSIONS.join(' or ');
   if (indexFile) {
     if (!isValidFile(indexFile))
       throw Error(`Index file ${indexFile} has invalid extension; must be ${ext_string}}`);
     if (!fs.existsSync(indexFile)) throw Error(`Index file ${indexFile} not found`);
   }
+  if (opts?.urlFolders && !opts.projectPath) opts.projectPath = path;
   const ignoreFiles = getIgnoreFiles(session, path);
   let implicitIndex = false;
   if (!indexFile) {
@@ -160,7 +166,7 @@ export function projectFromPath(
       path,
       1,
       {},
-      { ignore: ignoreFiles, suppressWarnings: true },
+      { ...opts, ignore: ignoreFiles, suppressWarnings: true },
     );
     if (!searchPages.length) {
       throw Error(`No valid files with extensions ${ext_string} found in path "${path}"`);
@@ -170,8 +176,9 @@ export function projectFromPath(
     implicitIndex = true;
   }
   const pageSlugs: PageSlugs = {};
-  const { slug } = fileInfo(indexFile, pageSlugs);
+  const { slug } = fileInfo(indexFile, pageSlugs, { ...opts, session });
   const pages = projectPagesFromPath(session, path, 1, pageSlugs, {
+    ...opts,
     ignore: [indexFile, ...ignoreFiles],
   });
   return { file: indexFile, index: slug, path, pages, implicitIndex };

packages/myst-cli/src/project/fromTOC.ts

@@ -16,6 +16,7 @@ import type {
   LocalProjectPage,
   LocalProject,
   PageSlugs,
+  SlugOptions,
 } from './types.js';
 import type {
   TOC,
@@ -160,6 +161,7 @@ function pagesFromEntries(
   pages: (LocalProjectFolder | LocalProjectPage)[] = [],
   level: PageLevels = 1,
   pageSlugs: PageSlugs,
+  opts?: SlugOptions,
 ): (LocalProjectFolder | LocalProjectPage)[] {
   const configFile = selectors.selectLocalConfigFile(session.store.getState(), path);
   for (const entry of entries) {
@@ -174,7 +176,7 @@ function pagesFromEntries(
         });
       });
       if (file && fs.existsSync(file) && !isDirectory(file)) {
-        const { slug } = fileInfo(file, pageSlugs);
+        const { slug } = fileInfo(file, pageSlugs, { ...opts, session });
         pages.push({ file, level: entryLevel, slug, implicit: entry.implicit });
       }
     } else if (isURL(entry)) {
@@ -203,6 +205,7 @@ function pagesFromEntries(
         pages,
         nextLevel(entryLevel),
         pageSlugs,
+        opts,
       );
     }
   }
@@ -240,6 +243,7 @@ export function projectFromTOC(
   toc: TOC,
   level: PageLevels = 1,
   file?: string,
+  opts?: SlugOptions,
 ): Omit<LocalProject, 'bibliography'> {
   const pageSlugs: PageSlugs = {};
   const ignoreFiles = [...getIgnoreFiles(session, path), ...listExplicitFiles(toc, path)];
@@ -268,9 +272,10 @@ export function projectFromTOC(
   if (!indexFile) {
     throw Error(`Could not resolve project index file: ${root.file}`);
   }
-  const { slug } = fileInfo(indexFile, pageSlugs);
+  if (opts?.urlFolders && !opts.projectPath) opts.projectPath = path;
+  const { slug } = fileInfo(indexFile, pageSlugs, { ...opts, session });
   const pages: (LocalProjectFolder | LocalProjectPage)[] = [];
-  pagesFromEntries(session, path, entries, pages, level, pageSlugs);
+  pagesFromEntries(session, path, entries, pages, level, pageSlugs, opts);
   return { path: path || '.', file: indexFile, index: slug, pages };
 }
 
@@ -281,14 +286,15 @@ function pagesFromSphinxChapters(
   pages: (LocalProjectFolder | LocalProjectPage)[] = [],
   level: PageLevels = 1,
   pageSlugs: PageSlugs,
+  opts?: SlugOptions,
 ): (LocalProjectFolder | LocalProjectPage)[] {
   const filename = tocFile(path);
   const { dir } = parse(filename);
   chapters.forEach((chapter) => {
     // TODO: support globs and urls
     const file = chapter.file ? resolveExtension(join(dir, chapter.file)) : undefined;
     if (file) {
-      const { slug } = fileInfo(file, pageSlugs);
+      const { slug } = fileInfo(file, pageSlugs, { ...opts, session });
       pages.push({ file, level, slug });
     }
     if (!file && chapter.file) {
@@ -304,7 +310,15 @@ function pagesFromSphinxChapters(
       pages.push({ level, title: chapter.title });
     }
     if (chapter.sections) {
-      pagesFromSphinxChapters(session, path, chapter.sections, pages, nextLevel(level), pageSlugs);
+      pagesFromSphinxChapters(
+        session,
+        path,
+        chapter.sections,
+        pages,
+        nextLevel(level),
+        pageSlugs,
+        opts,
+      );
     }
   });
   return pages;
@@ -323,6 +337,7 @@ export function projectFromSphinxTOC(
   session: ISession,
   path: string,
   level: PageLevels = 1,
+  opts?: SlugOptions,
 ): Omit<LocalProject, 'bibliography'> {
   const filename = tocFile(path);
   if (!fs.existsSync(filename)) {
@@ -342,16 +357,17 @@ export function projectFromSphinxTOC(
       ).join('\n- ')}\n`,
     );
   }
-  const { slug } = fileInfo(indexFile, pageSlugs);
+  if (opts?.urlFolders && !opts.projectPath) opts.projectPath = path;
+  const { slug } = fileInfo(indexFile, pageSlugs, { ...opts, session });
   const pages: (LocalProjectFolder | LocalProjectPage)[] = [];
   if (toc.sections) {
     // Do not allow sections to have level < 1
     if (level < 1) level = 1;
-    pagesFromSphinxChapters(session, path, toc.sections, pages, level, pageSlugs);
+    pagesFromSphinxChapters(session, path, toc.sections, pages, level, pageSlugs, opts);
   } else if (toc.chapters) {
     // Do not allow chapters to have level < 0
     if (level < 0) level = 0;
-    pagesFromSphinxChapters(session, path, toc.chapters, pages, level, pageSlugs);
+    pagesFromSphinxChapters(session, path, toc.chapters, pages, level, pageSlugs, opts);
   } else if (toc.parts) {
     // Do not allow parts to have level < -1
     if (level < -1) level = -1;
@@ -360,7 +376,15 @@ export function projectFromSphinxTOC(
         pages.push({ title: part.caption || `Part ${index + 1}`, level });
       }
       if (part.chapters) {
-        pagesFromSphinxChapters(session, path, part.chapters, pages, nextLevel(level), pageSlugs);
+        pagesFromSphinxChapters(
+          session,
+          path,
+          part.chapters,
+          pages,
+          nextLevel(level),
+          pageSlugs,
+          opts,
+        );
       }
     });
   }
@@ -377,8 +401,16 @@ export function pagesFromTOC(
   path: string,
   toc: TOC,
   level: PageLevels,
+  opts?: SlugOptions,
 ): (LocalProjectFolder | LocalProjectPage)[] {
-  const { file, index, pages } = projectFromTOC(session, path, toc, nextLevel(level));
+  const { file, index, pages } = projectFromTOC(
+    session,
+    path,
+    toc,
+    nextLevel(level),
+    undefined,
+    opts,
+  );
   pages.unshift({ file, slug: index, level });
   return pages;
 }
@@ -392,8 +424,9 @@ export function pagesFromSphinxTOC(
   session: ISession,
   path: string,
   level: PageLevels,
+  opts?: SlugOptions,
 ): (LocalProjectFolder | LocalProjectPage)[] {
-  const { file, index, pages } = projectFromSphinxTOC(session, path, nextLevel(level));
+  const { file, index, pages } = projectFromSphinxTOC(session, path, nextLevel(level), opts);
   pages.unshift({ file, slug: index, level });
   return pages;
 }

packages/myst-cli/src/project/load.ts

@@ -57,9 +57,13 @@ export async function loadProjectFromDisk(
   let newProject: Omit<LocalProject, 'bibliography'> | undefined;
   let { index, writeTOC } = opts || {};
   let legacyToc = false;
+  const siteConfig = selectors.selectLocalSiteConfig(state, path);
+  const urlFolders = !!siteConfig?.options?.url_folders;
   const sphinxTOCFile = validateSphinxTOC(session, path) ? tocFile(path) : undefined;
   if (projectConfig?.toc !== undefined) {
-    newProject = projectFromTOC(session, path, projectConfig.toc, 1, projectConfigFile);
+    newProject = projectFromTOC(session, path, projectConfig.toc, 1, projectConfigFile, {
+      urlFolders,
+    });
     if (sphinxTOCFile) {
       addWarningForFile(
         session,
@@ -94,14 +98,14 @@ export async function loadProjectFromDisk(
       //   },
       // );
     }
-    newProject = projectFromSphinxTOC(session, path);
+    newProject = projectFromSphinxTOC(session, path, undefined, { urlFolders });
   } else {
     const project = selectors.selectLocalProject(state, path);
     if (!index && !project?.implicitIndex && project?.file) {
       // If there is no new index, keep the original unless it was implicit previously
       index = project.file;
     }
-    newProject = projectFromPath(session, path, index);
+    newProject = projectFromPath(session, path, index, { urlFolders });
   }
   if (!newProject) {
     throw new Error(`Could not load project from ${path}`);