feat: allow templating in markdown files (#10)

stuart-core/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "stuart_core"
-version = "0.2.6"
+version = "0.2.7"
 edition = "2021"
 license = "MIT"
 homepage = "https://github.com/w-henderson/Stuart"

stuart-core/src/fs/mod.rs

@@ -118,6 +118,19 @@ impl Node {
         }
     }
 
+    /// Returns the node's parsed contents mutably.
+    /// (This goes against everything Stuart is supposed to be but don't worry about it, it's for markdown preprocessing)
+    pub fn parsed_contents_mut(&mut self) -> &mut ParsedContents {
+        match self {
+            Node::File {
+                parsed_contents, ..
+            } => parsed_contents,
+            Node::Directory { .. } => {
+                panic!("`Node::parsed_contents_mut` should only be used on files")
+            }
+        }
+    }
+
     /// Returns the filesystem source of the node.
     pub fn source(&self) -> &Path {
         match self {
@@ -198,7 +211,8 @@ impl Node {
                     parse_html(contents_string?, file, plugins).map_err(Error::Parse)?,
                 ),
                 Some("md") => ParsedContents::Markdown(
-                    parse_markdown(contents_string?.to_string(), file).map_err(Error::Parse)?,
+                    parse_markdown(contents_string?.to_string(), file, plugins)
+                        .map_err(Error::Parse)?,
                 ),
                 Some("json") => ParsedContents::Json(
                     humphrey_json::from_str(contents_string?).map_err(|_| {

stuart-core/src/lib.rs

@@ -89,16 +89,22 @@ impl Stuart {
         }
     }
 
-    /// Creates a new builder from a virtual filesystem tree.
-    pub fn new_from_node(node: Node) -> Self {
-        Self {
+    /// Creates a new builder from a virtual filesystem tree. (for tests)
+    pub fn new_from_node(mut node: Node) -> Self {
+        let mut stuart = Self {
             dir: node.source().to_path_buf(),
-            input: Some(node),
+            input: Some(node.clone()),
             output: None,
             config: Config::default(),
-            base: None,
+            base: Some(StackFrame::new("base")),
             plugins: None,
-        }
+        };
+
+        stuart.preprocess_markdown_node(&mut node).unwrap();
+
+        stuart.input = Some(node);
+
+        stuart
     }
 
     /// Sets the configuration to use.
@@ -118,24 +124,23 @@ impl Stuart {
 
     /// Attempts to build the project.
     pub fn build(&mut self, stuart_env: String) -> Result<(), Error> {
-        self.input = Some(match self.plugins {
+        let mut input = match self.plugins {
             Some(ref plugins) => Node::new_with_plugins(&self.dir, true, plugins.as_ref())?,
             None => Node::new(&self.dir, true)?,
-        });
+        };
+
+        // This needs some explaining...
+        // We have to clone the input node here so that we can have an immutable copy in case
+        // something tries to change it during the markdown preprocessing stage.
+        // I hate this as much as you, TODO: come up with a better solution.
+        self.input = Some(input.clone());
 
         let vars = {
             let mut env = std::env::vars().collect::<Vec<_>>();
             env.push(("STUART_ENV".into(), stuart_env));
             env
         };
 
-        let env = Environment {
-            vars: &vars,
-            md: None,
-            root: None,
-        }
-        .update_from_children(self.input.as_ref().unwrap().children().unwrap());
-
         let base = StackFrame::new("base").with_variable(
             "env",
             Value::Object(
@@ -146,6 +151,17 @@ impl Stuart {
         );
 
         self.base = Some(base);
+
+        self.preprocess_markdown_node(&mut input)?;
+        self.input = Some(input);
+
+        let env = Environment {
+            vars: &vars,
+            md: None,
+            root: None,
+        }
+        .update_from_children(self.input.as_ref().unwrap().children().unwrap());
+
         self.output = Some(self.build_node(self.input.as_ref().unwrap(), env)?);
 
         Ok(())
@@ -211,6 +227,25 @@ impl Stuart {
             Node::File { .. } => node.process(self, env),
         }
     }
+
+    /// Preprocess the given markdown node and its descendants, executing functions
+    /// and adding the result to the node's metadata in place.
+    fn preprocess_markdown_node(&mut self, node: &mut Node) -> Result<(), Error> {
+        match node {
+            Node::Directory { children, .. } => {
+                for child in children.iter_mut() {
+                    self.preprocess_markdown_node(child)?;
+                }
+
+                Ok(())
+            }
+            Node::File {
+                parsed_contents: ParsedContents::Markdown(_),
+                ..
+            } => node.preprocess_markdown(self).map_err(Error::Process),
+            _ => Ok(()),
+        }
+    }
 }
 
 impl<'a> Environment<'a> {

stuart-core/src/parse/markdown.rs

@@ -1,25 +1,31 @@
 //! Provides functionality for parsing markdown files.
 
-use super::{ParseError, TracebackError};
+use crate::plugins::Manager;
+
+use super::{parse_html, LocatableToken, ParseError, TracebackError};
 
 use humphrey_json::Value;
-use pulldown_cmark::{html, Options, Parser};
 
 use std::path::Path;
 
 /// Represents the parsed contents of a markdown file.
 #[derive(Clone, Debug)]
 pub struct ParsedMarkdown {
     /// The frontmatter of the file.
-    frontmatter: Vec<(String, String)>,
-    /// The body of the file, parsed into HTML.
-    body: String,
+    pub(crate) frontmatter: Vec<(String, String)>,
+    /// The raw markdown body of the file.
+    pub(crate) markdown: Vec<LocatableToken>,
+    /// The raw markdown body of the file as a string.
+    pub(crate) markdown_string: String,
+    /// The final processed HTML body of the file.
+    pub(crate) html: Option<String>,
 }
 
 /// Attempts to parse a markdown file into a [`ParsedMarkdown`] struct.
 pub fn parse_markdown(
     input: String,
     path: &Path,
+    plugins: Option<&dyn Manager>,
 ) -> Result<ParsedMarkdown, TracebackError<ParseError>> {
     let (lines_to_skip, frontmatter) = if input.starts_with("---\n") || input.starts_with("---\r\n")
     {
@@ -77,17 +83,20 @@ pub fn parse_markdown(
         (0, Vec::new())
     };
 
-    let markdown = input
+    let raw_markdown = input
         .lines()
         .skip(lines_to_skip)
         .collect::<Vec<_>>()
         .join("\n");
 
-    let parser = Parser::new_ext(&markdown, Options::all());
-    let mut body = String::new();
-    html::push_html(&mut body, parser);
+    let markdown = parse_html(&raw_markdown, path, plugins)?;
 
-    Ok(ParsedMarkdown { frontmatter, body })
+    Ok(ParsedMarkdown {
+        frontmatter,
+        markdown,
+        markdown_string: raw_markdown,
+        html: None,
+    })
 }
 
 impl ParsedMarkdown {
@@ -97,7 +106,8 @@ impl ParsedMarkdown {
     ///   is not required, consider using [`ParsedMarkdown::to_json`], which does the same thing without returning the contents.
     pub fn to_value(&self) -> Value {
         let mut v = self.frontmatter_to_value();
-        v["content"] = Value::String(self.body.clone());
+        v["content"] = Value::String(self.html.as_ref().unwrap().clone());
+        v["markdown"] = Value::String(self.markdown_string.clone());
         v
     }
 

stuart-core/src/parse/mod.rs

@@ -102,7 +102,7 @@ pub fn parse_html(
 
     let (mut line, mut column) = parser.location();
 
-    while let Some(raw) = parser.extract_until("{{") {
+    while let Some(raw) = parser.extract_until("{{", true) {
         if !raw.is_empty() {
             tokens.push(LocatableToken {
                 inner: Token::Raw(raw),
@@ -135,7 +135,7 @@ pub fn parse_html(
         (line, column) = parser.location();
     }
 
-    let remaining = parser.extract_remaining();
+    let remaining = parser.extract_remaining(true);
     if !remaining.is_empty() {
         tokens.push(LocatableToken {
             inner: Token::Raw(remaining),

stuart-core/src/parse/parser.rs

@@ -83,7 +83,7 @@ impl<'a> Parser<'a> {
     /// Extracts a string from the parser until the given string is found.
     ///
     /// The string is not included in the output.
-    pub fn extract_until(&mut self, s: &str) -> Option<String> {
+    pub fn extract_until(&mut self, s: &str, allow_escape: bool) -> Option<String> {
         let mut result = String::with_capacity(128);
         let old_chars = self.chars.clone();
 
@@ -92,6 +92,14 @@ impl<'a> Parser<'a> {
                 result.push(c);
 
                 if result.ends_with(s) {
+                    if allow_escape
+                        && result.len() > s.len()
+                        && result.as_bytes()[result.len() - s.len() - 1] == b'\\'
+                    {
+                        result.remove(result.len() - s.len() - 1);
+                        continue;
+                    }
+
                     result.truncate(result.len() - s.len());
 
                     return Some(result);
@@ -123,13 +131,17 @@ impl<'a> Parser<'a> {
     }
 
     /// Extracts all remaining characters in the parser.
-    pub fn extract_remaining(&mut self) -> String {
+    pub fn extract_remaining(&mut self, allow_escape: bool) -> String {
         let mut result = String::with_capacity(128);
 
         while let Ok(c) = self.next() {
             result.push(c);
         }
 
+        if allow_escape {
+            result = result.replace("\\{{", "{{");
+        }
+
         result
     }
 

stuart-core/src/process/mod.rs

@@ -14,6 +14,7 @@ use crate::parse::{LocatableToken, ParsedMarkdown, Token};
 use crate::{Environment, Error, Stuart};
 
 use humphrey_json::Value;
+use pulldown_cmark::{html, Options, Parser};
 
 /// Represents the scope of a function execution.
 pub struct Scope<'a> {
@@ -212,6 +213,61 @@ impl Node {
             new_name: Some(new_name),
         })
     }
+
+    /// Preprocess the markdown node, executing functions within the raw markdown and
+    /// converting it to HTML. The implementation of this is currently quite dodgy but
+    /// it works for the time being.
+    pub(crate) fn preprocess_markdown(
+        &mut self,
+        processor: &Stuart,
+    ) -> Result<(), TracebackError<ProcessError>> {
+        let source = self.source().to_path_buf();
+
+        let md = match self.parsed_contents_mut() {
+            ParsedContents::Markdown(md) => md,
+            _ => return Ok(()),
+        };
+
+        let mut token_iter = TokenIter::new(&md.markdown);
+        let mut stack: Vec<StackFrame> = vec![processor.base.as_ref().unwrap().clone()];
+        let mut sections: Vec<(String, Vec<u8>)> = Vec::new();
+        let mut scope = Scope {
+            tokens: &mut token_iter,
+            stack: &mut stack,
+            processor,
+            sections: &mut sections,
+        };
+
+        while let Some(token) = scope.tokens.next() {
+            token.process(&mut scope)?;
+        }
+
+        if let Some(frame) = scope.stack.pop() {
+            if frame.name == "base" {
+                let processed_markdown =
+                    String::from_utf8(frame.output).map_err(|_| TracebackError {
+                        path: source.clone(),
+                        line: 0,
+                        column: 0,
+                        kind: ProcessError::StackError,
+                    })?;
+
+                let parser = Parser::new_ext(&processed_markdown, Options::all());
+                let mut processed_html = String::new();
+                html::push_html(&mut processed_html, parser);
+
+                md.html = Some(processed_html);
+                return Ok(());
+            }
+        }
+
+        Err(TracebackError {
+            path: self.source().to_path_buf(),
+            line: 0,
+            column: 0,
+            kind: ProcessError::StackError,
+        })
+    }
 }
 
 impl LocatableToken {

stuart-core/src/tests/mod.rs

@@ -15,7 +15,9 @@ define_testcases![
     dateformat,
     excerpt,
     ifdefined,
-    conditionals
+    conditionals,
+    markdown_functions,
+    escape
 ];
 
 pub struct Testcase {
@@ -38,7 +40,7 @@ impl Testcase {
         context.merge(specific_context).unwrap();
 
         let input = Node::create_from_file(path.join("in.html"), true, None).unwrap();
-        let output = Node::create_from_file(path.join("out.html"), true, None).unwrap();
+        let output = Node::create_from_file(path.join("out"), false, None).unwrap();
 
         // Add the input to the base context.
         match context {

stuart-core/src/tests/testcases/escape/in.html

@@ -0,0 +1,7 @@
+{{ begin("main") }}
+<ul>
+{{ for($post, "posts/", sortby=$post.title, order="asc") }}
+<li>{{ $post.content }} \{{ $self.author }} • \{{ dateformat($self.date, "%d %B %Y") }}</li>
+{{ end(for) }}
+</ul>
+{{ end("main") }}