🗂 Add site option to include folder structure in url paths (#1601)

Co-authored-by: Rowan Cockett <[email protected]>

.changeset/fresh-cooks-move.md

@@ -0,0 +1,5 @@
+---
+"myst-common": patch
+---
+
+Add `slugToUrl` function that changes from a myst slug to an equivalent url

.changeset/shiny-dragons-glow.md

@@ -0,0 +1,6 @@
+---
+'myst-cli': patch
+'mystmd': patch
+---
+
+Add site option to include folder structure in url paths

docs/website-templates.md

@@ -78,6 +78,10 @@ Below is a table of options for each theme bundled with MyST.
 :heading-depth: 3
 ```
 
+### Site URL Options
+
+By default, MyST URLs only contain the file name for each page; folder structure is respected in the table of contents but is not reflected in URLs. If you would like to maintain nested folder structure in the URLs, you may provide the site option `folders: true`. This causes each folder in your MyST directory to become a path segment. For this feature to work correctly, your chosen theme must also support `folders` as an option. Both `book-theme` and `article-theme` bundled with MyST support this.
+
 ### Page Options
 
 Depending on the option, these can also be controlled in the frontmatter on each page under the `site` key.

packages/myst-cli/src/build/html/index.ts

@@ -8,6 +8,7 @@ import { getSiteManifest } from '../site/manifest.js';
 import type { StartOptions } from '../site/start.js';
 import { startServer } from '../site/start.js';
 import { getSiteTemplate } from '../site/template.js';
+import { slugToUrl } from 'myst-common';
 
 export async function currentSiteRoutes(
   session: ISession,
@@ -27,9 +28,10 @@ export async function currentSiteRoutes(
       return [
         { url: `${host}${projSlug}${siteIndex}`, path: path.join(proj.slug ?? '', 'index.html') },
         ...pages.map((page) => {
+          const pageSlug = slugToUrl(page.slug);
           return {
-            url: `${host}${projSlug}/${page.slug}`,
-            path: path.join(proj.slug ?? '', `${page.slug}.html`),
+            url: `${host}${projSlug}/${pageSlug}`,
+            path: path.join(proj.slug ?? '', `${pageSlug}.html`),
           };
         }),
         // Download all of the configured JSON

packages/myst-cli/src/build/site/manifest.ts

@@ -18,6 +18,7 @@ import type { RootState } from '../../store/index.js';
 import { selectors } from '../../store/index.js';
 import { transformBanner, transformThumbnail } from '../../transforms/images.js';
 import { addWarningForFile } from '../../utils/addWarningForFile.js';
+import { fileTitle } from '../../utils/fileInfo.js';
 import { resolveFrontmatterParts } from '../../utils/resolveFrontmatterParts.js';
 import version from '../../version.js';
 import { getSiteTemplate } from './template.js';
@@ -139,7 +140,7 @@ export async function localToManifestProject(
     proj.pages.map(async (page) => {
       if ('file' in page) {
         const fileInfo = selectors.selectFileInfo(state, page.file);
-        const title = fileInfo.title || page.slug;
+        const title = fileInfo.title || fileTitle(page.file);
         const short_title = fileInfo.short_title ?? undefined;
         const description = fileInfo.description ?? '';
         const thumbnail = fileInfo.thumbnail ?? '';

packages/myst-cli/src/process/mdast.ts

@@ -1,7 +1,7 @@
 import path from 'node:path';
 import { tic } from 'myst-cli-utils';
 import type { GenericParent, IExpressionResult, PluginUtils, References } from 'myst-common';
-import { fileError, fileWarn, RuleId } from 'myst-common';
+import { fileError, fileWarn, RuleId, slugToUrl } from 'myst-common';
 import type { PageFrontmatter } from 'myst-frontmatter';
 import { SourceFileKind } from 'myst-spec-ext';
 import type { LinkTransformer } from 'myst-transforms';
@@ -262,6 +262,7 @@ export async function transformMdast(
     url = `/${useSlug ? pageSlug : ''}`;
     dataUrl = `/${pageSlug}.json`;
   }
+  url = slugToUrl(url);
   updateFileInfoFromFrontmatter(session, file, frontmatter, url, dataUrl);
   const data: RendererData = {
     kind: isJupytext ? SourceFileKind.Notebook : kind,

packages/myst-cli/src/process/site.ts

@@ -3,7 +3,7 @@ import { basename, extname, join } from 'node:path';
 import chalk from 'chalk';
 import { Inventory, Domains } from 'intersphinx';
 import { writeFileToFolder, tic, hashAndCopyStaticFile } from 'myst-cli-utils';
-import { RuleId, toText, plural } from 'myst-common';
+import { RuleId, toText, plural, slugToUrl } from 'myst-common';
 import type { SiteConfig, SiteProject } from 'myst-config';
 import type { Node } from 'myst-spec';
 import type { SearchRecord, MystSearchIndex } from 'myst-spec-ext';
@@ -172,14 +172,14 @@ export async function writeMystSearchJson(session: ISession, pages: LocalProject
     .map((page) => selectFile(session, page.file))
     .map((file) => {
       const { mdast, slug, frontmatter } = file ?? {};
-      if (!mdast || !frontmatter) {
+      if (!mdast || !frontmatter || !slug) {
         return [];
       }
       const title = frontmatter.title ?? '';
 
       // Group by section (simple running accumulator)
       const sections = toSectionedParts(mdast);
-      const pageURL = slug && DEFAULT_INDEX_FILENAMES.includes(slug) ? '/' : `/${slug}`;
+      const pageURL = DEFAULT_INDEX_FILENAMES.includes(slug) ? '/' : `/${slugToUrl(slug)}`;
       // Build sections into search records
       return sections
         .map((section, index) => {

packages/myst-cli/src/project/fromPath.ts

@@ -4,7 +4,7 @@ import { isDirectory } from 'myst-cli-utils';
 import { RuleId } from 'myst-common';
 import type { ISession } from '../session/types.js';
 import { addWarningForFile } from '../utils/addWarningForFile.js';
-import { fileInfo } from '../utils/fileInfo.js';
+import { fileInfo, fileTitle } from '../utils/fileInfo.js';
 import { nextLevel } from '../utils/nextLevel.js';
 import { VALID_FILE_EXTENSIONS, isValidFile } from '../utils/resolveExtension.js';
 import { shouldIgnoreFile } from '../utils/shouldIgnoreFile.js';
@@ -20,9 +20,10 @@ import type {
   LocalProjectPage,
   LocalProject,
   PageSlugs,
+  SlugOptions,
 } from './types.js';
 
-type Options = {
+type Options = SlugOptions & {
   ignore?: string[];
   suppressWarnings?: boolean;
 };
@@ -54,7 +55,7 @@ function projectPagesFromPath(
     try {
       // TODO: We don't yet have a way to do nested tocs with new-style toc
       session.log.debug(`Respecting legacy TOC in subdirectory: ${join(path, '_toc.yml')}`);
-      return pagesFromSphinxTOC(session, path, prevLevel);
+      return pagesFromSphinxTOC(session, path, prevLevel, opts);
     } catch {
       if (!suppressWarnings) {
         addWarningForFile(
@@ -73,15 +74,18 @@ function projectPagesFromPath(
       return {
         file,
         level,
-        slug: fileInfo(file, pageSlugs).slug,
+        slug: fileInfo(file, pageSlugs, opts).slug,
         implicit: true,
       } as LocalProjectPage;
     });
   const folders = contents
     .filter((file) => isDirectory(file))
     .sort(comparePaths)
     .map((dir) => {
-      const projectFolder: LocalProjectFolder = { title: fileInfo(dir, pageSlugs).title, level };
+      const projectFolder: LocalProjectFolder = {
+        title: fileTitle(dir),
+        level,
+      };
       const pages = projectPagesFromPath(session, dir, nextLevel(level), pageSlugs, opts);
       if (!pages.length) {
         return [];
@@ -145,13 +149,15 @@ export function projectFromPath(
   session: ISession,
   path: string,
   indexFile?: string,
+  opts?: SlugOptions,
 ): Omit<LocalProject, 'bibliography'> {
   const ext_string = VALID_FILE_EXTENSIONS.join(' or ');
   if (indexFile) {
     if (!isValidFile(indexFile))
       throw Error(`Index file ${indexFile} has invalid extension; must be ${ext_string}}`);
     if (!fs.existsSync(indexFile)) throw Error(`Index file ${indexFile} not found`);
   }
+  if (opts?.urlFolders && !opts.projectPath) opts.projectPath = path;
   const ignoreFiles = getIgnoreFiles(session, path);
   let implicitIndex = false;
   if (!indexFile) {
@@ -160,7 +166,7 @@ export function projectFromPath(
       path,
       1,
       {},
-      { ignore: ignoreFiles, suppressWarnings: true },
+      { ...opts, ignore: ignoreFiles, suppressWarnings: true },
     );
     if (!searchPages.length) {
       throw Error(`No valid files with extensions ${ext_string} found in path "${path}"`);
@@ -170,8 +176,9 @@ export function projectFromPath(
     implicitIndex = true;
   }
   const pageSlugs: PageSlugs = {};
-  const { slug } = fileInfo(indexFile, pageSlugs);
+  const { slug } = fileInfo(indexFile, pageSlugs, { ...opts, session });
   const pages = projectPagesFromPath(session, path, 1, pageSlugs, {
+    ...opts,
     ignore: [indexFile, ...ignoreFiles],
   });
   return { file: indexFile, index: slug, path, pages, implicitIndex };

packages/myst-cli/src/project/fromTOC.ts

@@ -16,6 +16,7 @@ import type {
   LocalProjectPage,
   LocalProject,
   PageSlugs,
+  SlugOptions,
 } from './types.js';
 import type {
   TOC,
@@ -160,6 +161,7 @@ function pagesFromEntries(
   pages: (LocalProjectFolder | LocalProjectPage)[] = [],
   level: PageLevels = 1,
   pageSlugs: PageSlugs,
+  opts?: SlugOptions,
 ): (LocalProjectFolder | LocalProjectPage)[] {
   const configFile = selectors.selectLocalConfigFile(session.store.getState(), path);
   for (const entry of entries) {
@@ -174,7 +176,7 @@ function pagesFromEntries(
         });
       });
       if (file && fs.existsSync(file) && !isDirectory(file)) {
-        const { slug } = fileInfo(file, pageSlugs);
+        const { slug } = fileInfo(file, pageSlugs, { ...opts, session });
         pages.push({ file, level: entryLevel, slug, implicit: entry.implicit });
       }
     } else if (isURL(entry)) {
@@ -203,6 +205,7 @@ function pagesFromEntries(
         pages,
         nextLevel(entryLevel),
         pageSlugs,
+        opts,
       );
     }
   }
@@ -240,6 +243,7 @@ export function projectFromTOC(
   toc: TOC,
   level: PageLevels = 1,
   file?: string,
+  opts?: SlugOptions,
 ): Omit<LocalProject, 'bibliography'> {
   const pageSlugs: PageSlugs = {};
   const ignoreFiles = [...getIgnoreFiles(session, path), ...listExplicitFiles(toc, path)];
@@ -268,9 +272,10 @@ export function projectFromTOC(
   if (!indexFile) {
     throw Error(`Could not resolve project index file: ${root.file}`);
   }
-  const { slug } = fileInfo(indexFile, pageSlugs);
+  if (opts?.urlFolders && !opts.projectPath) opts.projectPath = path;
+  const { slug } = fileInfo(indexFile, pageSlugs, { ...opts, session });
   const pages: (LocalProjectFolder | LocalProjectPage)[] = [];
-  pagesFromEntries(session, path, entries, pages, level, pageSlugs);
+  pagesFromEntries(session, path, entries, pages, level, pageSlugs, opts);
   return { path: path || '.', file: indexFile, index: slug, pages };
 }
 
@@ -281,14 +286,15 @@ function pagesFromSphinxChapters(
   pages: (LocalProjectFolder | LocalProjectPage)[] = [],
   level: PageLevels = 1,
   pageSlugs: PageSlugs,
+  opts?: SlugOptions,
 ): (LocalProjectFolder | LocalProjectPage)[] {
   const filename = tocFile(path);
   const { dir } = parse(filename);
   chapters.forEach((chapter) => {
     // TODO: support globs and urls
     const file = chapter.file ? resolveExtension(join(dir, chapter.file)) : undefined;
     if (file) {
-      const { slug } = fileInfo(file, pageSlugs);
+      const { slug } = fileInfo(file, pageSlugs, { ...opts, session });
       pages.push({ file, level, slug });
     }
     if (!file && chapter.file) {
@@ -304,7 +310,15 @@ function pagesFromSphinxChapters(
       pages.push({ level, title: chapter.title });
     }
     if (chapter.sections) {
-      pagesFromSphinxChapters(session, path, chapter.sections, pages, nextLevel(level), pageSlugs);
+      pagesFromSphinxChapters(
+        session,
+        path,
+        chapter.sections,
+        pages,
+        nextLevel(level),
+        pageSlugs,
+        opts,
+      );
     }
   });
   return pages;
@@ -323,6 +337,7 @@ export function projectFromSphinxTOC(
   session: ISession,
   path: string,
   level: PageLevels = 1,
+  opts?: SlugOptions,
 ): Omit<LocalProject, 'bibliography'> {
   const filename = tocFile(path);
   if (!fs.existsSync(filename)) {
@@ -342,16 +357,17 @@ export function projectFromSphinxTOC(
       ).join('\n- ')}\n`,
     );
   }
-  const { slug } = fileInfo(indexFile, pageSlugs);
+  if (opts?.urlFolders && !opts.projectPath) opts.projectPath = path;
+  const { slug } = fileInfo(indexFile, pageSlugs, { ...opts, session });
   const pages: (LocalProjectFolder | LocalProjectPage)[] = [];
   if (toc.sections) {
     // Do not allow sections to have level < 1
     if (level < 1) level = 1;
-    pagesFromSphinxChapters(session, path, toc.sections, pages, level, pageSlugs);
+    pagesFromSphinxChapters(session, path, toc.sections, pages, level, pageSlugs, opts);
   } else if (toc.chapters) {
     // Do not allow chapters to have level < 0
     if (level < 0) level = 0;
-    pagesFromSphinxChapters(session, path, toc.chapters, pages, level, pageSlugs);
+    pagesFromSphinxChapters(session, path, toc.chapters, pages, level, pageSlugs, opts);
   } else if (toc.parts) {
     // Do not allow parts to have level < -1
     if (level < -1) level = -1;
@@ -360,7 +376,15 @@ export function projectFromSphinxTOC(
         pages.push({ title: part.caption || `Part ${index + 1}`, level });
       }
       if (part.chapters) {
-        pagesFromSphinxChapters(session, path, part.chapters, pages, nextLevel(level), pageSlugs);
+        pagesFromSphinxChapters(
+          session,
+          path,
+          part.chapters,
+          pages,
+          nextLevel(level),
+          pageSlugs,
+          opts,
+        );
       }
     });
   }
@@ -377,8 +401,16 @@ export function pagesFromTOC(
   path: string,
   toc: TOC,
   level: PageLevels,
+  opts?: SlugOptions,
 ): (LocalProjectFolder | LocalProjectPage)[] {
-  const { file, index, pages } = projectFromTOC(session, path, toc, nextLevel(level));
+  const { file, index, pages } = projectFromTOC(
+    session,
+    path,
+    toc,
+    nextLevel(level),
+    undefined,
+    opts,
+  );
   pages.unshift({ file, slug: index, level });
   return pages;
 }
@@ -392,8 +424,9 @@ export function pagesFromSphinxTOC(
   session: ISession,
   path: string,
   level: PageLevels,
+  opts?: SlugOptions,
 ): (LocalProjectFolder | LocalProjectPage)[] {
-  const { file, index, pages } = projectFromSphinxTOC(session, path, nextLevel(level));
+  const { file, index, pages } = projectFromSphinxTOC(session, path, nextLevel(level), opts);
   pages.unshift({ file, slug: index, level });
   return pages;
 }

packages/myst-cli/src/project/load.ts

@@ -57,9 +57,13 @@ export async function loadProjectFromDisk(
   let newProject: Omit<LocalProject, 'bibliography'> | undefined;
   let { index, writeTOC } = opts || {};
   let legacyToc = false;
+  const siteConfig = selectors.selectLocalSiteConfig(state, path);
+  const folders = !!siteConfig?.options?.folders;
   const sphinxTOCFile = validateSphinxTOC(session, path) ? tocFile(path) : undefined;
   if (projectConfig?.toc !== undefined) {
-    newProject = projectFromTOC(session, path, projectConfig.toc, 1, projectConfigFile);
+    newProject = projectFromTOC(session, path, projectConfig.toc, 1, projectConfigFile, {
+      urlFolders: folders,
+    });
     if (sphinxTOCFile) {
       addWarningForFile(
         session,
@@ -94,14 +98,14 @@ export async function loadProjectFromDisk(
       //   },
       // );
     }
-    newProject = projectFromSphinxTOC(session, path);
+    newProject = projectFromSphinxTOC(session, path, undefined, { urlFolders: folders });
   } else {
     const project = selectors.selectLocalProject(state, path);
     if (!index && !project?.implicitIndex && project?.file) {
       // If there is no new index, keep the original unless it was implicit previously
       index = project.file;
     }
-    newProject = projectFromPath(session, path, index);
+    newProject = projectFromPath(session, path, index, { urlFolders: folders });
   }
   if (!newProject) {
     throw new Error(`Could not load project from ${path}`);