From 52b9a7392a23bdfb96e314aa19a117479eda83af Mon Sep 17 00:00:00 2001 From: William Henderson Date: Sun, 31 Dec 2023 17:36:11 +0000 Subject: [PATCH] feat: allow templating in markdown files (#10) --- Cargo.lock | 4 +- stuart-core/Cargo.toml | 2 +- stuart-core/src/fs/mod.rs | 16 ++++- stuart-core/src/lib.rs | 65 ++++++++++++++----- stuart-core/src/parse/markdown.rs | 32 +++++---- stuart-core/src/parse/mod.rs | 4 +- stuart-core/src/parse/parser.rs | 16 ++++- stuart-core/src/process/mod.rs | 56 ++++++++++++++++ stuart-core/src/tests/mod.rs | 6 +- .../testcases/conditionals/{out.html => out} | 0 .../testcases/dateformat/{out.html => out} | 0 .../src/tests/testcases/escape/in.html | 7 ++ stuart-core/src/tests/testcases/escape/out | 9 +++ .../tests/testcases/escape/posts/post_3.md | 6 ++ .../tests/testcases/excerpt/{out.html => out} | 0 .../for_loop_json_file/{out.html => out} | 0 .../for_loop_json_object/{out.html => out} | 0 .../for_loop_markdown/{out.html => out} | 0 .../for_loop_nested/{out.html => out} | 0 .../for_loop_skip_limit/{out.html => out} | 0 .../testcases/ifdefined/{out.html => out} | 0 .../testcases/markdown_functions/in.html | 7 ++ .../tests/testcases/markdown_functions/out | 9 +++ .../markdown_functions/posts/post_3.md | 8 +++ .../markdown_functions/testcase.json | 3 + stuart/Cargo.toml | 2 +- 26 files changed, 215 insertions(+), 37 deletions(-) rename stuart-core/src/tests/testcases/conditionals/{out.html => out} (100%) rename stuart-core/src/tests/testcases/dateformat/{out.html => out} (100%) create mode 100644 stuart-core/src/tests/testcases/escape/in.html create mode 100644 stuart-core/src/tests/testcases/escape/out create mode 100644 stuart-core/src/tests/testcases/escape/posts/post_3.md rename stuart-core/src/tests/testcases/excerpt/{out.html => out} (100%) rename stuart-core/src/tests/testcases/for_loop_json_file/{out.html => out} (100%) rename stuart-core/src/tests/testcases/for_loop_json_object/{out.html => out} (100%) rename stuart-core/src/tests/testcases/for_loop_markdown/{out.html => out} (100%) rename stuart-core/src/tests/testcases/for_loop_nested/{out.html => out} (100%) rename stuart-core/src/tests/testcases/for_loop_skip_limit/{out.html => out} (100%) rename stuart-core/src/tests/testcases/ifdefined/{out.html => out} (100%) create mode 100644 stuart-core/src/tests/testcases/markdown_functions/in.html create mode 100644 stuart-core/src/tests/testcases/markdown_functions/out create mode 100644 stuart-core/src/tests/testcases/markdown_functions/posts/post_3.md create mode 100644 stuart-core/src/tests/testcases/markdown_functions/testcase.json diff --git a/Cargo.lock b/Cargo.lock index 5cda3dd..7360979 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -697,7 +697,7 @@ checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" [[package]] name = "stuart" -version = "0.2.6" +version = "0.2.7" dependencies = [ "clap", "humphrey", @@ -715,7 +715,7 @@ dependencies = [ [[package]] name = "stuart_core" -version = "0.2.6" +version = "0.2.7" dependencies = [ "chrono", "dateparser", diff --git a/stuart-core/Cargo.toml b/stuart-core/Cargo.toml index 55f8162..0cf85f4 100644 --- a/stuart-core/Cargo.toml +++ b/stuart-core/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "stuart_core" -version = "0.2.6" +version = "0.2.7" edition = "2021" license = "MIT" homepage = "https://github.com/w-henderson/Stuart" diff --git a/stuart-core/src/fs/mod.rs b/stuart-core/src/fs/mod.rs index e795cee..0f3129a 100644 --- a/stuart-core/src/fs/mod.rs +++ b/stuart-core/src/fs/mod.rs @@ -118,6 +118,19 @@ impl Node { } } + /// Returns the node's parsed contents mutably. + /// (This goes against everything Stuart is supposed to be but don't worry about it, it's for markdown preprocessing) + pub fn parsed_contents_mut(&mut self) -> &mut ParsedContents { + match self { + Node::File { + parsed_contents, .. + } => parsed_contents, + Node::Directory { .. } => { + panic!("`Node::parsed_contents_mut` should only be used on files") + } + } + } + /// Returns the filesystem source of the node. pub fn source(&self) -> &Path { match self { @@ -198,7 +211,8 @@ impl Node { parse_html(contents_string?, file, plugins).map_err(Error::Parse)?, ), Some("md") => ParsedContents::Markdown( - parse_markdown(contents_string?.to_string(), file).map_err(Error::Parse)?, + parse_markdown(contents_string?.to_string(), file, plugins) + .map_err(Error::Parse)?, ), Some("json") => ParsedContents::Json( humphrey_json::from_str(contents_string?).map_err(|_| { diff --git a/stuart-core/src/lib.rs b/stuart-core/src/lib.rs index 34c5975..d603c01 100644 --- a/stuart-core/src/lib.rs +++ b/stuart-core/src/lib.rs @@ -89,16 +89,22 @@ impl Stuart { } } - /// Creates a new builder from a virtual filesystem tree. - pub fn new_from_node(node: Node) -> Self { - Self { + /// Creates a new builder from a virtual filesystem tree. (for tests) + pub fn new_from_node(mut node: Node) -> Self { + let mut stuart = Self { dir: node.source().to_path_buf(), - input: Some(node), + input: Some(node.clone()), output: None, config: Config::default(), - base: None, + base: Some(StackFrame::new("base")), plugins: None, - } + }; + + stuart.preprocess_markdown_node(&mut node).unwrap(); + + stuart.input = Some(node); + + stuart } /// Sets the configuration to use. @@ -118,10 +124,16 @@ impl Stuart { /// Attempts to build the project. pub fn build(&mut self, stuart_env: String) -> Result<(), Error> { - self.input = Some(match self.plugins { + let mut input = match self.plugins { Some(ref plugins) => Node::new_with_plugins(&self.dir, true, plugins.as_ref())?, None => Node::new(&self.dir, true)?, - }); + }; + + // This needs some explaining... + // We have to clone the input node here so that we can have an immutable copy in case + // something tries to change it during the markdown preprocessing stage. + // I hate this as much as you, TODO: come up with a better solution. + self.input = Some(input.clone()); let vars = { let mut env = std::env::vars().collect::>(); @@ -129,13 +141,6 @@ impl Stuart { env }; - let env = Environment { - vars: &vars, - md: None, - root: None, - } - .update_from_children(self.input.as_ref().unwrap().children().unwrap()); - let base = StackFrame::new("base").with_variable( "env", Value::Object( @@ -146,6 +151,17 @@ impl Stuart { ); self.base = Some(base); + + self.preprocess_markdown_node(&mut input)?; + self.input = Some(input); + + let env = Environment { + vars: &vars, + md: None, + root: None, + } + .update_from_children(self.input.as_ref().unwrap().children().unwrap()); + self.output = Some(self.build_node(self.input.as_ref().unwrap(), env)?); Ok(()) @@ -211,6 +227,25 @@ impl Stuart { Node::File { .. } => node.process(self, env), } } + + /// Preprocess the given markdown node and its descendants, executing functions + /// and adding the result to the node's metadata in place. + fn preprocess_markdown_node(&mut self, node: &mut Node) -> Result<(), Error> { + match node { + Node::Directory { children, .. } => { + for child in children.iter_mut() { + self.preprocess_markdown_node(child)?; + } + + Ok(()) + } + Node::File { + parsed_contents: ParsedContents::Markdown(_), + .. + } => node.preprocess_markdown(self).map_err(Error::Process), + _ => Ok(()), + } + } } impl<'a> Environment<'a> { diff --git a/stuart-core/src/parse/markdown.rs b/stuart-core/src/parse/markdown.rs index a0ce287..3b24ee1 100644 --- a/stuart-core/src/parse/markdown.rs +++ b/stuart-core/src/parse/markdown.rs @@ -1,9 +1,10 @@ //! Provides functionality for parsing markdown files. -use super::{ParseError, TracebackError}; +use crate::plugins::Manager; + +use super::{parse_html, LocatableToken, ParseError, TracebackError}; use humphrey_json::Value; -use pulldown_cmark::{html, Options, Parser}; use std::path::Path; @@ -11,15 +12,20 @@ use std::path::Path; #[derive(Clone, Debug)] pub struct ParsedMarkdown { /// The frontmatter of the file. - frontmatter: Vec<(String, String)>, - /// The body of the file, parsed into HTML. - body: String, + pub(crate) frontmatter: Vec<(String, String)>, + /// The raw markdown body of the file. + pub(crate) markdown: Vec, + /// The raw markdown body of the file as a string. + pub(crate) markdown_string: String, + /// The final processed HTML body of the file. + pub(crate) html: Option, } /// Attempts to parse a markdown file into a [`ParsedMarkdown`] struct. pub fn parse_markdown( input: String, path: &Path, + plugins: Option<&dyn Manager>, ) -> Result> { let (lines_to_skip, frontmatter) = if input.starts_with("---\n") || input.starts_with("---\r\n") { @@ -77,17 +83,20 @@ pub fn parse_markdown( (0, Vec::new()) }; - let markdown = input + let raw_markdown = input .lines() .skip(lines_to_skip) .collect::>() .join("\n"); - let parser = Parser::new_ext(&markdown, Options::all()); - let mut body = String::new(); - html::push_html(&mut body, parser); + let markdown = parse_html(&raw_markdown, path, plugins)?; - Ok(ParsedMarkdown { frontmatter, body }) + Ok(ParsedMarkdown { + frontmatter, + markdown, + markdown_string: raw_markdown, + html: None, + }) } impl ParsedMarkdown { @@ -97,7 +106,8 @@ impl ParsedMarkdown { /// is not required, consider using [`ParsedMarkdown::to_json`], which does the same thing without returning the contents. pub fn to_value(&self) -> Value { let mut v = self.frontmatter_to_value(); - v["content"] = Value::String(self.body.clone()); + v["content"] = Value::String(self.html.as_ref().unwrap().clone()); + v["markdown"] = Value::String(self.markdown_string.clone()); v } diff --git a/stuart-core/src/parse/mod.rs b/stuart-core/src/parse/mod.rs index cab020f..22cffb4 100644 --- a/stuart-core/src/parse/mod.rs +++ b/stuart-core/src/parse/mod.rs @@ -102,7 +102,7 @@ pub fn parse_html( let (mut line, mut column) = parser.location(); - while let Some(raw) = parser.extract_until("{{") { + while let Some(raw) = parser.extract_until("{{", true) { if !raw.is_empty() { tokens.push(LocatableToken { inner: Token::Raw(raw), @@ -135,7 +135,7 @@ pub fn parse_html( (line, column) = parser.location(); } - let remaining = parser.extract_remaining(); + let remaining = parser.extract_remaining(true); if !remaining.is_empty() { tokens.push(LocatableToken { inner: Token::Raw(remaining), diff --git a/stuart-core/src/parse/parser.rs b/stuart-core/src/parse/parser.rs index 6476fec..bf76b0c 100644 --- a/stuart-core/src/parse/parser.rs +++ b/stuart-core/src/parse/parser.rs @@ -83,7 +83,7 @@ impl<'a> Parser<'a> { /// Extracts a string from the parser until the given string is found. /// /// The string is not included in the output. - pub fn extract_until(&mut self, s: &str) -> Option { + pub fn extract_until(&mut self, s: &str, allow_escape: bool) -> Option { let mut result = String::with_capacity(128); let old_chars = self.chars.clone(); @@ -92,6 +92,14 @@ impl<'a> Parser<'a> { result.push(c); if result.ends_with(s) { + if allow_escape + && result.len() > s.len() + && result.as_bytes()[result.len() - s.len() - 1] == b'\\' + { + result.remove(result.len() - s.len() - 1); + continue; + } + result.truncate(result.len() - s.len()); return Some(result); @@ -123,13 +131,17 @@ impl<'a> Parser<'a> { } /// Extracts all remaining characters in the parser. - pub fn extract_remaining(&mut self) -> String { + pub fn extract_remaining(&mut self, allow_escape: bool) -> String { let mut result = String::with_capacity(128); while let Ok(c) = self.next() { result.push(c); } + if allow_escape { + result = result.replace("\\{{", "{{"); + } + result } diff --git a/stuart-core/src/process/mod.rs b/stuart-core/src/process/mod.rs index 0d55f58..fc32c0b 100644 --- a/stuart-core/src/process/mod.rs +++ b/stuart-core/src/process/mod.rs @@ -14,6 +14,7 @@ use crate::parse::{LocatableToken, ParsedMarkdown, Token}; use crate::{Environment, Error, Stuart}; use humphrey_json::Value; +use pulldown_cmark::{html, Options, Parser}; /// Represents the scope of a function execution. pub struct Scope<'a> { @@ -212,6 +213,61 @@ impl Node { new_name: Some(new_name), }) } + + /// Preprocess the markdown node, executing functions within the raw markdown and + /// converting it to HTML. The implementation of this is currently quite dodgy but + /// it works for the time being. + pub(crate) fn preprocess_markdown( + &mut self, + processor: &Stuart, + ) -> Result<(), TracebackError> { + let source = self.source().to_path_buf(); + + let md = match self.parsed_contents_mut() { + ParsedContents::Markdown(md) => md, + _ => return Ok(()), + }; + + let mut token_iter = TokenIter::new(&md.markdown); + let mut stack: Vec = vec![processor.base.as_ref().unwrap().clone()]; + let mut sections: Vec<(String, Vec)> = Vec::new(); + let mut scope = Scope { + tokens: &mut token_iter, + stack: &mut stack, + processor, + sections: &mut sections, + }; + + while let Some(token) = scope.tokens.next() { + token.process(&mut scope)?; + } + + if let Some(frame) = scope.stack.pop() { + if frame.name == "base" { + let processed_markdown = + String::from_utf8(frame.output).map_err(|_| TracebackError { + path: source.clone(), + line: 0, + column: 0, + kind: ProcessError::StackError, + })?; + + let parser = Parser::new_ext(&processed_markdown, Options::all()); + let mut processed_html = String::new(); + html::push_html(&mut processed_html, parser); + + md.html = Some(processed_html); + return Ok(()); + } + } + + Err(TracebackError { + path: self.source().to_path_buf(), + line: 0, + column: 0, + kind: ProcessError::StackError, + }) + } } impl LocatableToken { diff --git a/stuart-core/src/tests/mod.rs b/stuart-core/src/tests/mod.rs index 2db23be..c5272c4 100644 --- a/stuart-core/src/tests/mod.rs +++ b/stuart-core/src/tests/mod.rs @@ -15,7 +15,9 @@ define_testcases![ dateformat, excerpt, ifdefined, - conditionals + conditionals, + markdown_functions, + escape ]; pub struct Testcase { @@ -38,7 +40,7 @@ impl Testcase { context.merge(specific_context).unwrap(); let input = Node::create_from_file(path.join("in.html"), true, None).unwrap(); - let output = Node::create_from_file(path.join("out.html"), true, None).unwrap(); + let output = Node::create_from_file(path.join("out"), false, None).unwrap(); // Add the input to the base context. match context { diff --git a/stuart-core/src/tests/testcases/conditionals/out.html b/stuart-core/src/tests/testcases/conditionals/out similarity index 100% rename from stuart-core/src/tests/testcases/conditionals/out.html rename to stuart-core/src/tests/testcases/conditionals/out diff --git a/stuart-core/src/tests/testcases/dateformat/out.html b/stuart-core/src/tests/testcases/dateformat/out similarity index 100% rename from stuart-core/src/tests/testcases/dateformat/out.html rename to stuart-core/src/tests/testcases/dateformat/out diff --git a/stuart-core/src/tests/testcases/escape/in.html b/stuart-core/src/tests/testcases/escape/in.html new file mode 100644 index 0000000..c084914 --- /dev/null +++ b/stuart-core/src/tests/testcases/escape/in.html @@ -0,0 +1,7 @@ +{{ begin("main") }} +
    +{{ for($post, "posts/", sortby=$post.title, order="asc") }} +
  • {{ $post.content }} \{{ $self.author }} • \{{ dateformat($self.date, "%d %B %Y") }}
    • +{{ end(for) }} +
+{{ end("main") }} \ No newline at end of file diff --git a/stuart-core/src/tests/testcases/escape/out b/stuart-core/src/tests/testcases/escape/out new file mode 100644 index 0000000..e298a4f --- /dev/null +++ b/stuart-core/src/tests/testcases/escape/out @@ -0,0 +1,9 @@ + + +
    +

  • This is post 1

    {{ $self.author }} • {{ dateformat($self.date, "%d %B %Y") }}
    • +

  • This is post 2

    {{ $self.author }} • {{ dateformat($self.date, "%d %B %Y") }}
    • +

  • {{ hello }}

    {{ $self.author }} • {{ dateformat($self.date, "%d %B %Y") }}
    • +
+ + \ No newline at end of file diff --git a/stuart-core/src/tests/testcases/escape/posts/post_3.md b/stuart-core/src/tests/testcases/escape/posts/post_3.md new file mode 100644 index 0000000..2379e88 --- /dev/null +++ b/stuart-core/src/tests/testcases/escape/posts/post_3.md @@ -0,0 +1,6 @@ +--- +title: "Post 3" +date: "2022-09-03" +--- + +\{{ hello }} \ No newline at end of file diff --git a/stuart-core/src/tests/testcases/excerpt/out.html b/stuart-core/src/tests/testcases/excerpt/out similarity index 100% rename from stuart-core/src/tests/testcases/excerpt/out.html rename to stuart-core/src/tests/testcases/excerpt/out diff --git a/stuart-core/src/tests/testcases/for_loop_json_file/out.html b/stuart-core/src/tests/testcases/for_loop_json_file/out similarity index 100% rename from stuart-core/src/tests/testcases/for_loop_json_file/out.html rename to stuart-core/src/tests/testcases/for_loop_json_file/out diff --git a/stuart-core/src/tests/testcases/for_loop_json_object/out.html b/stuart-core/src/tests/testcases/for_loop_json_object/out similarity index 100% rename from stuart-core/src/tests/testcases/for_loop_json_object/out.html rename to stuart-core/src/tests/testcases/for_loop_json_object/out diff --git a/stuart-core/src/tests/testcases/for_loop_markdown/out.html b/stuart-core/src/tests/testcases/for_loop_markdown/out similarity index 100% rename from stuart-core/src/tests/testcases/for_loop_markdown/out.html rename to stuart-core/src/tests/testcases/for_loop_markdown/out diff --git a/stuart-core/src/tests/testcases/for_loop_nested/out.html b/stuart-core/src/tests/testcases/for_loop_nested/out similarity index 100% rename from stuart-core/src/tests/testcases/for_loop_nested/out.html rename to stuart-core/src/tests/testcases/for_loop_nested/out diff --git a/stuart-core/src/tests/testcases/for_loop_skip_limit/out.html b/stuart-core/src/tests/testcases/for_loop_skip_limit/out similarity index 100% rename from stuart-core/src/tests/testcases/for_loop_skip_limit/out.html rename to stuart-core/src/tests/testcases/for_loop_skip_limit/out diff --git a/stuart-core/src/tests/testcases/ifdefined/out.html b/stuart-core/src/tests/testcases/ifdefined/out similarity index 100% rename from stuart-core/src/tests/testcases/ifdefined/out.html rename to stuart-core/src/tests/testcases/ifdefined/out diff --git a/stuart-core/src/tests/testcases/markdown_functions/in.html b/stuart-core/src/tests/testcases/markdown_functions/in.html new file mode 100644 index 0000000..cd318d2 --- /dev/null +++ b/stuart-core/src/tests/testcases/markdown_functions/in.html @@ -0,0 +1,7 @@ +{{ begin("main") }} +
    +{{ for($post, "posts/", sortby=$post.title, order="asc") }} +
  • {{ $post.content }}
    • +{{ end(for) }} +
+{{ end("main") }} \ No newline at end of file diff --git a/stuart-core/src/tests/testcases/markdown_functions/out b/stuart-core/src/tests/testcases/markdown_functions/out new file mode 100644 index 0000000..3da5c97 --- /dev/null +++ b/stuart-core/src/tests/testcases/markdown_functions/out @@ -0,0 +1,9 @@ + + +
    +

  • This is post 1

    • +

  • This is post 2

    • +

  • This contains functions in markdown.

    • +
+ + \ No newline at end of file diff --git a/stuart-core/src/tests/testcases/markdown_functions/posts/post_3.md b/stuart-core/src/tests/testcases/markdown_functions/posts/post_3.md new file mode 100644 index 0000000..316a720 --- /dev/null +++ b/stuart-core/src/tests/testcases/markdown_functions/posts/post_3.md @@ -0,0 +1,8 @@ +--- +title: "Post 3" +date: "2022-09-03" +--- + +{{ import($data, "testcase.json") }} + +This contains {{ $data.string }}. \ No newline at end of file diff --git a/stuart-core/src/tests/testcases/markdown_functions/testcase.json b/stuart-core/src/tests/testcases/markdown_functions/testcase.json new file mode 100644 index 0000000..55151ec --- /dev/null +++ b/stuart-core/src/tests/testcases/markdown_functions/testcase.json @@ -0,0 +1,3 @@ +{ + "string": "functions in markdown" +} \ No newline at end of file diff --git a/stuart/Cargo.toml b/stuart/Cargo.toml index a1c897c..6876641 100644 --- a/stuart/Cargo.toml +++ b/stuart/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "stuart" -version = "0.2.6" +version = "0.2.7" edition = "2021" license = "MIT" homepage = "https://github.com/w-henderson/Stuart"