Merge pull request #234 from ThinkR-open/change-parser

Change parser, remove {parsermd}
ThinkR-open · Dec 8, 2023 · 8045a50 · 8045a50
2 parents 0fd8d64 + 4a15cc4
commit 8045a50
Show file tree

Hide file tree

Showing 18 changed files with 308 additions and 254 deletions.
diff --git a/.github/workflows/R-CMD-check-devel.yaml b/.github/workflows/R-CMD-check-devel.yaml
@@ -74,7 +74,7 @@ jobs:
         run: |
           remotes::install_deps(dependencies = TRUE, upgrade = "always")
           # remotes::install_cran("rcmdcheck")
-          remotes::install_github("rundel/parsermd", upgrade = "always")
+          remotes::install_github("ThinkR-open/lightparser", upgrade = "always")
         shell: Rscript {0}
 
       # - name: Setup tmate session

diff --git a/DESCRIPTION b/DESCRIPTION
@@ -1,6 +1,6 @@
 Package: fusen
 Title: Build a Package from Rmarkdown Files
-Version: 0.5.2.9000
+Version: 0.6.0
 Authors@R: c(
     person("Sebastien", "Rochette", , "[email protected]", role = c("aut", "cre"),
            comment = c(ORCID = "0000-0002-1565-9313")),
@@ -24,9 +24,9 @@ Imports:
     devtools,
     glue,
     here (>= 1.0.0),
+    lightparser,
     magrittr,
     methods,
-    parsermd (>= 0.1.0),
     roxygen2,
     stats,
     stringi,
@@ -48,6 +48,8 @@ Suggests:
     withr
 VignetteBuilder: 
     knitr
+Remotes:
+    ThinkR-open/lightparser
 Config/fusen/version: 0.5.2.9000
 Config/Needs/website: ThinkR-open/thinkrtemplate
 Config/testthat/edition: 3

diff --git a/NAMESPACE b/NAMESPACE
@@ -27,13 +27,10 @@ importFrom(devtools,check)
 importFrom(glue,glue)
 importFrom(magrittr,"%>%")
 importFrom(methods,formalArgs)
-importFrom(parsermd,as_tibble)
-importFrom(parsermd,parse_rmd)
-importFrom(parsermd,rmd_get_chunk)
-importFrom(parsermd,rmd_node_code)
 importFrom(stats,na.omit)
 importFrom(stats,setNames)
 importFrom(stringi,stri_trans_general)
+importFrom(tibble,as_tibble)
 importFrom(tibble,tibble)
 importFrom(tools,file_path_sans_ext)
 importFrom(utils,getFromNamespace)

diff --git a/NEWS.md b/NEWS.md
@@ -1,4 +1,10 @@
-# fusen (development version)
+# fusen 0.6.0
+
+## Breaking changes
+
+- {fusen} now relies on {lightparser} instead of {parsermd} to parse flat file. This allows to avoid installation problems with {parsermd}, which is not updated anymore. As {lightparser} is lighter, this may have unattended effects on specific flat file cases. Please report any issue you may encounter. (#233)
+
+## New features
 
 - Allow `organisation` in `init_share_on_github()` to send to a GitHub organisation
 - Fix `load_flat_functions()` to work with VSCode

diff --git a/R/inflate-utils.R b/R/inflate-utils.R
@@ -19,11 +19,10 @@ regex_extract_fun_name <- paste(
 
 #' Parse function code as tibble and get positions
 #' @param x One row out of function parsed tibble
-#' @importFrom parsermd rmd_node_code
 #' @noRd
 parse_fun <- function(x) { # x <- rmd_fun[3,]
 
-  code <- unlist(rmd_node_code(x[["ast"]]))
+  code <- unlist(x[["code"]])
 
   # Clean extra space between #' and @
   code <- gsub(pattern = "#'\\s*@", "#' @", code)
@@ -276,7 +275,7 @@ parse_test <- function(x, pkg, relative_flat_file) { # x <- rmd_test[1,]
 #' Add examples in function code
 #' @param parsed_tbl tibble of a parsed Rmd
 #' @param fun_code R code of functions in Rmd as character
-#' @importFrom parsermd rmd_node_code
+#' @importFrom tibble as_tibble
 #' @noRd
 add_fun_code_examples <- function(parsed_tbl, fun_code) {
   # Example in separate chunk
@@ -323,9 +322,8 @@ add_fun_code_examples <- function(parsed_tbl, fun_code) {
 
   if (nrow(rmd_ex) != 0) {
     # Group rmd_ex for the same function
-    rmd_ex$rmd_ex_code <- lapply(1:nrow(rmd_ex), function(x) {
-      rmd_node_code(rmd_ex[x, ][["ast"]])
-    })
+    rmd_ex$rmd_ex_code <- rmd_ex[["code"]]
+
     rmd_ex_group <- group_code(df = rmd_ex, group_col = "fun_name", code_col = "rmd_ex_code")
 
     # Get example code
@@ -353,7 +351,6 @@ add_fun_code_examples <- function(parsed_tbl, fun_code) {
 
   # Remove if example is empty
   fun_code[["example"]] <- lapply(fun_code[["example"]], function(example) {
-    # example <- fun_code[["example"]][[1]]
     example <- gsub("^#' $", "#'", example) # clean empty lines
 
     if (length(example) == 0) {
@@ -370,32 +367,37 @@ add_fun_code_examples <- function(parsed_tbl, fun_code) {
   })
 
   # Add to function code
-  fun_code[["code_example"]] <- lapply(seq_len(nrow(fun_code)), function(x) {
-    fun_code_x <- fun_code[x, ]
-    if (is.na(fun_code_x[["fun_name"]])) {
-      return(
-        unlist(fun_code_x[["code"]])
-      )
-    }
+  fun_code[["code_example"]] <- lapply(
+    seq_len(nrow(fun_code)),
+    function(x) {
+      fun_code_x <- fun_code[x, ]
+      if (is.na(fun_code_x[["fun_name"]])) {
+        return(
+          unlist(fun_code_x[["code"]])
+        )
+      }
 
-    end_skeleton <- ifelse(is.na(fun_code_x[["example_pos_start"]]),
-      fun_code_x[["example_pos_end"]],
-      fun_code_x[["example_pos_start"]] - 1
-    )
+      end_skeleton <- ifelse(is.na(fun_code_x[["example_pos_start"]]),
+        fun_code_x[["example_pos_end"]],
+        fun_code_x[["example_pos_start"]] - 1
+      )
 
-    all_fun_code <- stats::na.omit(c(
-      # begin
-      if (!is.na(end_skeleton)) {
-        unlist(fun_code_x[["code"]])[1:end_skeleton]
-      },
-      # examples
-      unlist(fun_code_x[["example"]]),
-      # end
-      unlist(fun_code_x[["code"]])[
-        (fun_code_x[["example_pos_end"]] + 1):length(unlist(fun_code_x[["code"]]))
-      ]
-    ))
-  })
+      all_fun_code <- stats::na.omit(c(
+        # begin
+        if (!is.na(end_skeleton)) {
+          unlist(fun_code_x[["code"]])[1:end_skeleton]
+        },
+        # examples
+        unlist(fun_code_x[["example"]]),
+        # end
+        unlist(fun_code_x[["code"]])[
+          (fun_code_x[["example_pos_end"]] + 1):
+          length(unlist(fun_code_x[["code"]]))
+        ]
+      ))
+      return(all_fun_code)
+    }
+  )
 
   # Clean double #' due to dontrun
   fun_code[["code_example"]] <- lapply(fun_code[["code_example"]], function(example) {

diff --git a/R/inflate.R b/R/inflate.R
@@ -28,7 +28,6 @@ regex_example <- paste(regex_example_vec, collapse = "|")
 #'     For example, you can do `inflate(check = TRUE, quiet = TRUE)`, where `quiet` is
 #'     passed to `devtools::check()`.
 #'
-#' @importFrom parsermd parse_rmd as_tibble
 #' @importFrom utils getFromNamespace
 #' @importFrom glue glue
 #' @importFrom methods formalArgs
@@ -206,35 +205,28 @@ inflate <- function(pkg = ".", flat_file,
     roxygen2::roxygenise(pkg)
   }
 
-  parsed_flat_file <- parse_rmd(flat_file)
-  parsed_tbl <- as_tibble(parsed_flat_file)
+  parsed_tbl <- lightparser::split_to_tbl(flat_file)
+
+  parsed_tbl$order <- seq_len(nrow(parsed_tbl))
 
-  parsed_tbl$order <- 1:nrow(parsed_tbl)
 
   # Set start for group variables ----
-  parsed_tbl$options <- parsermd::rmd_get_options(parsed_tbl)
+  parsed_tbl$options <- parsed_tbl$params
+
   # Get filename option in chunk
   parsed_tbl$chunk_filename <- unlist(
     lapply(
       parsed_tbl[["options"]],
       function(x) {
-        ifelse(is.null(x[["filename"]]),
+        ifelse(!is.list(x) || is.null(x[["filename"]]),
           NA_character_, gsub('"', "", x[["filename"]])
         )
       }
     )
   )
   # Define sec_title to group functions in same R file
-  sec_title <- paste(parsed_tbl[["sec_h1"]],
-    parsed_tbl[["sec_h2"]],
-    sep = "-"
-  )
-
-  if (length(sec_title) != 0) {
-    parsed_tbl$sec_title <- sec_title
-  } else {
-    parsed_tbl$sec_title <- "fake-section-title"
-  }
+  parsed_tbl$sec_title <- parsed_tbl$section
+  parsed_tbl$sec_title[is.na(parsed_tbl$sec_title)] <- "fake-section-title"
 
   # Get flat file path relative to package root
   # To be inserted in "DO NOT EDIT" comments
@@ -410,7 +402,6 @@ create_functions_all <- function(parsed_tbl, fun_code, pkg, relative_flat_file)
 
 #' Get function names ----
 #' @param parsed_tbl tibble of a parsed Rmd
-#' @importFrom parsermd rmd_get_chunk
 #' @noRd
 get_functions_tests <- function(parsed_tbl) {
   which_parsed_fun <- which(!is.na(parsed_tbl$label) &
@@ -424,7 +415,6 @@ get_functions_tests <- function(parsed_tbl) {
     # At least one function
     fun_code <- lapply(seq_len(nrow(rmd_fun)), function(x) parse_fun(rmd_fun[x, ]))
     fun_code <- do.call("rbind", fun_code)
-    fun_code$sec_h1 <- rmd_fun[["sec_h1"]]
     fun_code$sec_title <- rmd_fun[["sec_title"]]
   } else if (length(which_parsed_tests) != 0) {
     # Some tests but no function at all
@@ -486,7 +476,6 @@ create_r_files <- function(fun_code, pkg, relative_flat_file) {
 #' @param pkg Path to package
 #' @param relative_flat_file Path to the flat file to show in R scripts
 #'
-#' @importFrom parsermd rmd_node_code
 #'
 #' @noRd
 create_tests_files <- function(parsed_tbl, pkg, relative_flat_file) {
@@ -510,7 +499,6 @@ create_tests_files <- function(parsed_tbl, pkg, relative_flat_file) {
     }
 
     # Group code by file_name
-    rmd_test[["code"]] <- rmd_node_code(rmd_test[["ast"]])
     rmd_test <- group_code(rmd_test, group_col = "file_name", code_col = "code")
 
     # Filter if code is still empty after code grouped
@@ -584,45 +572,23 @@ create_vignette <- function(parsed_tbl, pkg, relative_flat_file, vignette_name,
       regex_functions
     ), collapse = "|")
   vignette_tbl <- parsed_tbl[
-    !(grepl(not_in_vignette, parsed_tbl[["label"]]) |
-      grepl("rmd_yaml_list", parsed_tbl[["type"]])),
+    !(
+      grepl(not_in_vignette, parsed_tbl[["label"]]) |
+        grepl("yaml", parsed_tbl[["type"]])
+    ),
   ]
 
-  flat_yaml <- parsed_tbl[grepl("rmd_yaml_list", parsed_tbl[["type"]]), ]
-  # Make chunk names unique
-  # vignette_tbl[["label"]][grepl("unnamed", vignette_tbl[["label"]])] <-
-  #   gsub("unnamed-", "parsermd-", vignette_tbl[["label"]][grepl("unnamed", vignette_tbl[["label"]])])
-  #   is.na(vignette_tbl[["label"]]) & vignette_tbl[["type"]] == "rmd_chunk",
-  #                                   gsub("[.]+", "-", make.names(vignette_name)),
-  #                                   vignette_tbl[["label"]])
-  #
-  # vignette_tbl[["label"]] <- make.unique(vignette_tbl[["label"]], sep = "-")
-  # # /!\ Not re-used in as_document(), this must be in ast
-
-  # ast <- vignette_tbl[["ast"]][[21]]
-
-  # To correct for {parsermd} unnamed attribution
-  fix_unnamed_chunks <- function(ast) {
-    if (inherits(ast, "rmd_chunk") && grepl("unnamed-chunk-", ast[["name"]])) {
-      ast[["name"]] <- gsub("unnamed-", "parsermd-", ast[["name"]])
-    }
-    ast
-  }
-
-  ast_class <- class(vignette_tbl[["ast"]])
-  vignette_tbl[["ast"]] <- lapply(vignette_tbl[["ast"]], fix_unnamed_chunks)
-  class(vignette_tbl[["ast"]]) <- ast_class
+  flat_yaml <- parsed_tbl[grepl("yaml", parsed_tbl[["type"]]), ]
 
   # File to save
   cleaned_vignette_name <- asciify_name(vignette_name)
   vignette_file <- file.path("vignettes", paste0(cleaned_vignette_name, ".Rmd"))
 
   # Yaml info
-  yaml_options <- flat_yaml[["ast"]][[1]]
+  yaml_options <- flat_yaml$params[[1]]
   # Vignette
   # Copied from usethis::use_vignette() to allow to not open vignette created
   usethis::use_package("knitr", "Suggests")
-  # desc <- desc::desc(file = usethis::proj_get())
   desc <- desc::desc(file = pkg)
   desc$set("VignetteBuilder", "knitr")
   desc$write()
@@ -647,7 +613,7 @@ create_vignette <- function(parsed_tbl, pkg, relative_flat_file, vignette_name,
   if (nrow(vignette_tbl) != 0) {
     lines <- c(
       lines,
-      parsermd::as_document(vignette_tbl)
+      lightparser::combine_tbl_to_file(vignette_tbl)
     )
   }
 

diff --git a/R/load_flat_functions.R b/R/load_flat_functions.R
@@ -39,15 +39,15 @@ load_flat_functions <- function(flat_file, envir = globalenv()) {
     )
   }
 
-  parsed_flat_file <- parse_rmd(flat_file)
-  parsed_tbl <- as_tibble(parsed_flat_file)
+  parsed_tbl <- lightparser::split_to_tbl(flat_file)
+
   which_parsed_fun <- which(
     !is.na(parsed_tbl$label) &
       grepl(regex_functions, parsed_tbl$label)
   )
 
-  if (nrow(parsed_tbl) > 0) {
-    content <- unlist(rmd_node_code(parsed_tbl[which_parsed_fun, ][["ast"]]))
+  if (nrow(parsed_tbl) > 0 && length(which_parsed_fun) > 0) {
+    content <- unlist(parsed_tbl[which_parsed_fun, ][["code"]])
 
     eval(parse(text = content), envir)
 

diff --git a/README.Rmd b/README.Rmd
@@ -195,6 +195,7 @@ Similarly, the {fusen} package uses a flat Rmd template, that you fill in a spec
 
 - Thanks to Deemah who asked me to go further 'Rmd first' after my presentation at useR 2019 in Toulouse: ['The "Rmd first" method: when projects start with documentation'](https://github.com/statnmap/prez/blob/master/2019-07_useR_Toulouse.pdf) (Video on Youtube: https://youtu.be/cB1BCxFbhtk). 
 - Thanks to @rundel and its package {parsermd} who helped me get back in this project with ease : https://github.com/rundel/parsermd
+    + Now replaced with {lightparser} : https://github.com/ThinkR-open/lightparser
 - Thanks to the [ThinkR team](https://rtask.thinkr.fr) who adopted this package for its daily production.
 
 ## Code of Conduct