feat: rewrite preprocessgdschool, greatly increase speed, rewrite dow…

…nload file paths
GDQuest · Jan 29, 2023 · 014e023 · 014e023
1 parent 73f3f03
commit 014e023
Show file tree

Hide file tree

Showing 2 changed files with 123 additions and 111 deletions.
diff --git a/src/md/preprocessgdschool.nim b/src/md/preprocessgdschool.nim
@@ -2,131 +2,141 @@ import std/
   [ algorithm
   , logging
   , os
-  , re
-  , sequtils
+  , nre
   , strformat
   , strutils
   , tables
-  , unicode
+  , options
   ]
 import assets
-import parser
-import parserparagraph
-import shortcodes
 import utils
 
 
-const
-  META_MDFILE* = "_index.md"
-  ROOT_SECTION_FOLDERS = @["courses", "bundles", "pages", "posts"]
+var cacheSlug: Table[string, seq[string]]
 
 let
-  regexGodotBuiltIns = ["(`(", CACHE_GODOT_BUILTIN_CLASSES.join("|"), ")`)"].join.re
+  regexShortcodeInclude = re"{{ *include.+}}"
+  regexMarkdownImage = re"!\[(?P<alt>.*)\]\((?P<path>.+)\)"
+  regexMarkdownCodeBlock = re"(?m)(?s)```(?P<language>\w+?)?\n(?P<body>.+?)```"
+  regexShortcodeArgsInclude = re"{{ *include (?P<file>.+?\.[a-zA-Z0-9]+) *(?P<anchor>\w+)? *}}"
+  regexGodotBuiltIns = ["(`(?P<class>", CACHE_GODOT_BUILTIN_CLASSES.join("|"), ")`)"].join.re
   regexCapital = re"([23]D|[A-Z])"
-  regexSlug = re"slug: *"
-
-var cacheSlug: Table[string, seq[string]]
-
-
-proc addGodotIcon(line: string): string =
-  var
-    line = line
-    bounds = line.findBounds(regexGodotBuiltIns)
-
-  while bounds != (-1, 0):
-    let class = ["icon", line[bounds.first ..< bounds.last].strip(chars = {'`'}).replacef(regexCapital, "_$#")].join.toLower
-
-    if class in CACHE_GODOT_ICONS:
-      result.add [line[0 ..< bounds.first], "<Icon name=\"", class, "\"/>", line[bounds.first .. bounds.last]].join
+  regexAnchorLine = re"(?m)(?s)\h*(#|\/\/)\h*(ANCHOR|END):.*?(\v|$)"
+
+  regexDownloads = re"(?m)(?s)<Downloads .+?>"
+  regexDownloadsSingleFile = re("file=\"(?P<url>.+?)\">")
+  regexDownloadsMultipleFiles = re("url: *\"(?P<url>.+?)\">")
+
+
+proc preprocessCodeListings(content: string): string =
+  ## Finds code blocks in the markdown document and searches for include
+  ## shortcodes in each code block. If any include shortcode is found, appends
+  ## the included file name to the code block and replaces include shortcodes
+  ## with the corresponding code. Returns a new string.
+
+  proc replaceIncludeShortcode(match: RegexMatch): string =
+    ## Processes one include shortcode to replace. Finds and loads the
+    ## appropriate GDScript or other code file and extracts and returns the
+    ## contents corresponding to the requested anchor.
+    let newMatch = match(match.match, regexShortcodeArgsInclude, 0, int.high)
+    if newMatch.isSome():
+      let
+        args = newMatch.get.captures.toTable()
+        includeFileName = cache.findFile(args["file"])
+      result = readFile(includeFileName)
+      if "anchor" in args:
+        let
+          anchor = args["anchor"]
+          regexAnchor = re(fmt(r"(?s)\h*(?:#|\/\/)\h*ANCHOR:\h*\b{anchor}\b\h*\v(.*?)\s*(?:#|\/\/)\h*END:\h*\b{anchor}\b"))
+
+        var anchorMatch = result.find(regexAnchor)
+        if anchorMatch.isSome():
+          result = anchorMatch.get.match
+        else:
+          raise newException(ValueError, "Can't find matching contents for anchor. {SYNOPSIS}")
+
+      result = result.replace(regexAnchorLine, "")
     else:
-      info fmt"Couldn't find icon for `{class}` in line: `{line}`. Skipping..."
-      result.add line[0 .. bounds.last]
-
-    line = line[bounds.last + 1 .. ^1]
-    bounds = line.findBounds(regexGodotBuiltIns)
-
-  result.add line
-
-
-proc preprocessParagraphLine(pl: seq[ParagraphLineSection], mdBlocks: seq[Block]; fileName: string): string =
-  pl.mapIt(
-    case it.kind
-    of plskShortcode:
-      SHORTCODESv2.getOrDefault(it.shortcode.name, noOpShortcode)(it.shortcode, mdBlocks, fileName)
-    of plskRegular: it.render
-  ).join.addGodotIcon
-
-
-func computeCodeBlockAnnotation(mdBlock: Block): string =
-  for cl in mdBlock.code:
-    if cl.kind == clkShortcode and cl.shortcode.name == "include" and cl.shortcode.args.len > 0:
-      result = ":" & cl.shortcode.args[0]
-      break
-
-
-proc preprocessCodeLine(cl: CodeLine, mdBlocks: seq[Block]; fileName: string): string =
-  case cl.kind
-  of clkShortcode:
-    SHORTCODESv2.getOrDefault(cl.shortcode.name, noOpShortcode)(cl.shortcode, mdBlocks, fileName)
-  of clkRegular: cl.line
-
-
-proc computeImageSlugs(fileName: string, imagePathPrefix: string): seq[string] =
-  if fileName in cacheSlug:
-    return cacheSlug[fileName]
-
-  for dir in fileName.parentDirs(inclusive = false):
-    if dir.endsWith(imagePathPrefix):
-      break
-
-    let meta_mdpath = dir / META_MDFILE
-    if meta_mdpath.fileExists:
-      for mdYAMLFrontMatter in readFile(meta_mdpath).parse.filterIt(it.kind == bkYAMLFrontMatter):
-        for line in mdYAMLFrontMatter.body:
-          if line.startsWith(regexSlug):
-            result.add line.replace(regexSlug, "")
+      error [ "Synopsis: `{{ include fileName(.gd|.shader) [anchorName] }}`"
+            , fmt"{result}: Incorrect include arguments. Expected 1 or 2 arguments. Skipping..."
+            ].join(NL)
+      return match.match
+
+  proc replaceMarkdownCodeBlock(match: RegexMatch): string =
+    let parts = match.captures.toTable()
+    let language = parts.getOrDefault("language", "gdscript")
+    result = "```" & language & "\n" & parts["body"].replace(regexShortcodeInclude, replaceIncludeShortcode) & "```"
+
+  result = content.replace(regexMarkdownCodeBlock, replaceMarkdownCodeBlock)
+
+
+proc makePathsAbsolute(content: string, fileName: string, pathPrefix = ""): string =
+  ## Find image paths and download file paths and turns relative paths into absolute paths.
+  ## pathPrefix is an optional prefix to preprend to the file path.
+
+  proc makeUrlAbsolute(relativePath: string): string =
+    ## Calculates and returns an absolute url for a relative file path.
+    const META_MDFILE = "_index.md"
+
+    var slug: seq[string]
+    if fileName in cacheSlug:
+      slug = cacheSlug[fileName]
     else:
-      result.add(dir.lastPathPart())
-
-  result = result.reversed
-  cacheSlug[fileName] = result
-
-
-proc preprocessImage(img: Block, mdBlocks: seq[Block], fileName: string, imagePathPrefix: string): string =
-  var img = img
-  img.path.removePrefix("./")
-  img.path = (@["", imagePathPrefix] & computeImageSlugs(fileName, imagePathPrefix) & img.path.split(AltSep)).join($AltSep)
-  img.render
-
-
-proc preprocessBlock(mdBlock: Block, mdBlocks: seq[Block]; fileName: string, imagePathPrefix: string): string =
-  case mdBlock.kind
-  of bkShortcode:
-    SHORTCODESv2.getOrDefault(mdBlock.name, noOpShortcode)(mdBlock, mdBlocks, fileName)
-
-  of bkParagraph:
-    mdBlock.body.mapIt(it.toParagraphLine.preprocessParagraphLine(mdBlocks, fileName)).join(NL)
-
-  of bkList:
-    mdBlock.items.mapIt(it.render.addGodotIcon).join(NL)
-
-  of bkCode:
-    [ fmt"```{mdBlock.language}" & computeCodeBlockAnnotation(mdBlock)
-    , mdBlock.code.mapIt(it.preprocessCodeLine(mdBlocks, fileName)).join(NL)
-    , "```"
-    ].join(NL)
-
-  of bkImage:
-    mdBlock.preprocessImage(mdBlocks, fileName, imagePathPrefix)
+      for directory in fileName.parentDirs(inclusive = false):
+        if directory.endsWith("content"):
+          break
+
+        let meta_mdpath = directory / META_MDFILE
+        if meta_mdpath.fileExists:
+          for line in readFile(meta_mdpath).split("\n"):
+            if line.startsWith("slug: "):
+              slug.add(line.replace("slug: ", ""))
+              break
+        else:
+          slug.add(directory.lastPathPart())
+      slug = slug.reversed
+      cacheSlug[fileName] = slug
+
+    result = (@[pathPrefix] & slug & relativePath.split(AltSep)).join($AltSep)
+
+  proc replaceImagePaths(match: RegexMatch): string =
+    let
+      parts = match.captures.toTable()
+      alt = parts.getOrDefault("alt", "")
+      pathAbsolute = makeUrlAbsolute(parts["path"])
+
+    result = fmt"![{alt}]({pathAbsolute})"
+
+  proc replaceDownloadPaths(match: RegexMatch): string =
+
+    proc replaceDownloadPath(fileMatch: RegexMatch): string =
+      let url = fileMatch.captures.toTable()["url"]
+      result = fileMatch.match.replace(url, makeUrlAbsolute(url))
+
+    result = match.match.replace(regexDownloadsSingleFile, replaceDownloadPath)
+    result = result.replace(regexDownloadsMultipleFiles, replaceDownloadPath)
+
+  result = content.replace(regexMarkdownImage, replaceImagePaths)
+  result = result.replace(regexDownloads, replaceDownloadPaths)
+
+
+proc addGodotIcons(content: string): string =
+  proc replaceGodotIcon(match: RegexMatch): string =
+    let className = match.captures.toTable()["class"]
+    let cssClass = ["icon", className.strip(chars = {'`'}).replace(regexCapital, "_$#")].join().toLower()
+    if cssClass in CACHE_GODOT_ICONS:
+      result = "<Icon name=\"" & cssClass & "\"/>"
+    else:
+      info fmt"Couldn't find icon for `{cssClass}`. Skipping..."
+      result = match.match
 
-  else:
-    mdBlock.render
+  result = content.replace(regexGodotBuiltIns, replaceGodotIcon)
 
 
-proc preprocess*(fileName, contents: string, imagePathPrefix: string): string =
-  let mdBlocks = contents.parse
+proc processContent*(fileContent: string, fileName: string, pathPrefix = ""): string =
+  const ROOT_SECTION_FOLDERS = @["courses", "bundles", "pages", "posts"]
 
-  var prefix = imagePathPrefix
+  var prefix = pathPrefix
   if prefix == "":
     for folderName in ROOT_SECTION_FOLDERS:
       if folderName & AltSep in fileName:
@@ -135,4 +145,4 @@ proc preprocess*(fileName, contents: string, imagePathPrefix: string): string =
     if prefix.isEmptyOrWhitespace():
       error fmt"The file {fileName} should be in one of the following folders: {ROOT_SECTION_FOLDERS}"
 
-  mdBlocks.mapIt(preprocessBlock(it, mdBlocks, fileName, prefix)).join(NL)
+  result = fileContent.preprocessCodeListings().makePathsAbsolute(fileName, pathPrefix).addGodotIcons()
diff --git a/src/preprocessgdschool.nim b/src/preprocessgdschool.nim
@@ -221,8 +221,10 @@ proc process(appSettings: AppSettingsBuildGDSchool) =
     createDir(fileOut.parentDir)
     if not appSettings.isQuiet:
       info fmt"Creating output `{fileOut.parentDir}` directory..."
-
-    writeFile(fileOut, preprocess(fileIn, fileInContents, appSettings.imagePathPrefix))
+    if fileIn.endsWith("_index.md"):
+      writeFile(fileOut, fileInContents)
+    else:
+      writeFile(fileOut, processContent(fileInContents, fileIn, appSettings.imagePathPrefix))
 
 
 when isMainModule: