JAL-3225 Removed dependency on command line pandoc to convert .md files to .html...
authorBen Soares <bsoares@dundee.ac.uk>
Thu, 4 Jun 2020 19:38:13 +0000 (20:38 +0100)
committersoares <bsoares@dundee.ac.uk>
Thu, 4 Jun 2020 19:41:06 +0000 (20:41 +0100)
build.gradle
gradle.properties
utils/doc/github.css [moved from doc/github.css with 100% similarity]

index 39c8261..0be087f 100644 (file)
@@ -12,13 +12,25 @@ import java.security.MessageDigest
 import groovy.transform.ExternalizeMethods
 import groovy.util.XmlParser
 import groovy.xml.XmlUtil
-
+import com.vladsch.flexmark.util.ast.Node
+import com.vladsch.flexmark.html.HtmlRenderer
+import com.vladsch.flexmark.parser.Parser
+import com.vladsch.flexmark.util.data.MutableDataSet
+import com.vladsch.flexmark.ext.gfm.tasklist.TaskListExtension
+import com.vladsch.flexmark.ext.tables.TablesExtension
+import com.vladsch.flexmark.ext.gfm.strikethrough.StrikethroughExtension
+import com.vladsch.flexmark.ext.autolink.AutolinkExtension
+import com.vladsch.flexmark.ext.anchorlink.AnchorLinkExtension
+import com.vladsch.flexmark.ext.toc.TocExtension
 
 buildscript {
   repositories {
     mavenCentral()
     mavenLocal()
   }
+  dependencies {
+    classpath "com.vladsch.flexmark:flexmark-all:0.62.0"
+  }
 }
 
 
@@ -1049,47 +1061,110 @@ task cleanBuildingHTML(type: Delete) {
 }
 
 
-task convertBuildingMD(type: Exec) {
+def convertMdToHtml (FileTree mdFiles, File cssFile) {
+  MutableDataSet options = new MutableDataSet()
+
+  def extensions = new ArrayList<>()
+  extensions.add(AnchorLinkExtension.create()) 
+  extensions.add(AutolinkExtension.create())
+  extensions.add(StrikethroughExtension.create())
+  extensions.add(TaskListExtension.create())
+  extensions.add(TablesExtension.create())
+  extensions.add(TocExtension.create())
+  
+  options.set(Parser.EXTENSIONS, extensions)
+
+  // set GFM table parsing options
+  options.set(TablesExtension.WITH_CAPTION, false)
+  options.set(TablesExtension.COLUMN_SPANS, false)
+  options.set(TablesExtension.MIN_HEADER_ROWS, 1)
+  options.set(TablesExtension.MAX_HEADER_ROWS, 1)
+  options.set(TablesExtension.APPEND_MISSING_COLUMNS, true)
+  options.set(TablesExtension.DISCARD_EXTRA_COLUMNS, true)
+  options.set(TablesExtension.HEADER_SEPARATOR_COLUMN_MATCH, true)
+  // GFM anchor links
+  options.set(AnchorLinkExtension.ANCHORLINKS_SET_ID, false)
+  options.set(AnchorLinkExtension.ANCHORLINKS_ANCHOR_CLASS, "anchor")
+  options.set(AnchorLinkExtension.ANCHORLINKS_SET_NAME, true)
+  options.set(AnchorLinkExtension.ANCHORLINKS_TEXT_PREFIX, "<span class=\"octicon octicon-link\"></span>")
+
+  Parser parser = Parser.builder(options).build()
+  HtmlRenderer renderer = HtmlRenderer.builder(options).build()
+
+  mdFiles.each { mdFile ->
+    // add table of contents
+    def mdText = "[TOC]\n"+mdFile.text
+
+    // grab the first top-level title
+    def title = null
+    def titleRegex = /(?m)^#(\s+|([^#]))(.*)/
+    def matcher = mdText =~ titleRegex
+    if (matcher.size() > 0) {
+      // matcher[0][2] is the first character of the title if there wasn't any whitespace after the #
+      title = (matcher[0][2] != null ? matcher[0][2] : "")+matcher[0][3]
+    }
+    // or use the filename if none found
+    if (title == null) {
+      title = mdFile.getName()
+    }
+
+    Node document = parser.parse(mdText)
+    String htmlBody = renderer.render(document)
+    def htmlText = '''<html>
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml">
+  <head>
+    <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
+    <meta http-equiv="Content-Style-Type" content="text/css" />
+    <meta name="generator" content="flexmark" />
+'''
+    htmlText += ((title != null) ? "  <title>${title}</title>" : '' )
+    htmlText += '''
+    <style type="text/css">code{white-space: pre;}</style>
+'''
+    htmlText += ((cssFile != null) ? cssFile.text : '')
+    htmlText += '''</head>
+  <body>
+'''
+    htmlText += htmlBody
+    htmlText += '''
+  </body>
+</html>
+'''
+
+    def htmlFilePath = mdFile.getPath().replaceAll(/\..*?$/, ".html")
+    def htmlFile = file(htmlFilePath)
+    htmlFile.text = htmlText
+  }
+}
+
+
+task convertMdFiles {
   dependsOn cleanBuildingHTML
-  def buildingMD = "${jalviewDir}/${doc_dir}/building.md"
-  def css = "${jalviewDir}/${doc_dir}/github.css"
+  def mdFiles = fileTree(dir: "${jalviewDir}/${doc_dir}", include: "*.md")
+  def cssFile = file("${jalviewDir}/${flexmark_css}")
 
-  def pandoc = null
-  pandoc_exec.split(",").each {
-    if (file(it.trim()).exists()) {
-      pandoc = it.trim()
-      return true
-    }
+  doLast {
+    convertMdToHtml(mdFiles, cssFile)
   }
 
-  def buildtoolsPandoc = System.getProperty("user.home")+"/buildtools/pandoc/bin/pandoc"
-  if ((pandoc == null || ! file(pandoc).exists()) && file(buildtoolsPandoc).exists()) {
-    pandoc = System.getProperty("user.home")+"/buildtools/pandoc/bin/pandoc"
-  }
+  inputs.files(mdFiles)
+  inputs.file(cssFile)
 
-  doFirst {
-    if (pandoc != null && file(pandoc).exists()) {
-        commandLine pandoc, '-s', '-o', buildingHTML, '--metadata', 'pagetitle="Building Jalview from Source"', '--toc', '-H', css, buildingMD
-    } else {
-        println("Cannot find pandoc. Skipping convert building.md to HTML")
-        throw new StopExecutionException("Cannot find pandoc. Skipping convert building.md to HTML")
-    }
+  def htmlFiles = []
+  mdFiles.each { mdFile ->
+    def htmlFilePath = mdFile.getPath().replaceAll(/\..*?$/, ".html")
+    htmlFiles.add(file(htmlFilePath))
   }
-
-  ignoreExitValue true
-
-  inputs.file(buildingMD)
-  inputs.file(css)
-  outputs.file(buildingHTML)
+  outputs.files(htmlFiles)
 }
 
 
 task syncDocs(type: Sync) {
-  //dependsOn convertBuildingMD
+  dependsOn convertMdFiles
   def syncDir = "${classesDir}/${doc_dir}"
   from fileTree("${jalviewDir}/${doc_dir}")
   into syncDir
-
 }
 
 
@@ -1790,7 +1865,7 @@ task sourceDist(type: Tar) {
   group "distribution"
   description "Create a source .tar.gz file for distribution"
   
-  dependsOn convertBuildingMD
+  dependsOn convertMdFiles
 
   def VERSION_UNDERSCORES = JALVIEW_VERSION.replaceAll("\\.", "_")
   def outputFileName = "${project.name}_${VERSION_UNDERSCORES}.tar.gz"
index 745e8c7..b99cdca 100644 (file)
@@ -110,6 +110,8 @@ j11libDir = j11lib
 j11modDir = j11mod
 j11modules = com.sun.istack.runtime,com.sun.xml.bind,com.sun.xml.fastinfoset,com.sun.xml.streambuffer,com.sun.xml.txw2,com.sun.xml.ws.policy,java.activation,java.annotation,java.base,java.compiler,java.datatransfer,java.desktop,java.logging,java.management,java.management.rmi,java.naming,java.prefs,java.rmi,java.scripting,java.security.sasl,java.sql,java.xml,java.xml.bind,java.xml.soap,java.xml.ws,javax.jws,jdk.httpserver,jdk.jsobject,jdk.unsupported,jdk.xml.dom,org.jvnet.mimepull,org.jvnet.staxex,javax.servlet.api,java.ws.rs
 
+flexmark_css = utils/doc/github.css
+
 install4j_home_dir = ~/buildtools/install4j8
 install4j_copyright_message = ...
 install4j_bundle_id = org.jalview.jalview-desktop
@@ -128,9 +130,6 @@ OSX_KEYSTORE =
 OSX_KEYPASS =
 JSIGN_SH = echo
 
-pandoc_exec = /usr/local/bin/pandoc,/usr/bin/pandoc
-dev = false
-
 CHANNEL=LOCAL
 getdown_channel_base = https://www.jalview.org/getdown
 getdown_app_dir_release = release
similarity index 100%
rename from doc/github.css
rename to utils/doc/github.css