From 729e39d052a71602d82f7808fdb3862b9d893ce8 Mon Sep 17 00:00:00 2001 From: Ben Soares Date: Thu, 5 May 2022 17:02:39 +0100 Subject: [PATCH] JAL-4004 Now generating hugo files from help/markdown/... --- build.gradle | 289 +++++++------------- gradle.properties | 2 +- help/templates/whatsNew.html | 97 +++++++ ..._to_extract_markdown_from_old_releases_html.txt | 221 +++++++++++++++ .../Version-__VERSION_UNDERSCORES__/_index.md | 2 +- 5 files changed, 425 insertions(+), 186 deletions(-) create mode 100755 help/templates/whatsNew.html create mode 100644 utils/hugo/old_task_to_extract_markdown_from_old_releases_html.txt diff --git a/build.gradle b/build.gradle index d89f3ed..e6cc590 100644 --- a/build.gradle +++ b/build.gradle @@ -2388,87 +2388,27 @@ task dataInstallersJson { } } -def hugoTemplateSubstitutions(String input) { +def hugoTemplateSubstitutions(String input, Map extras=null) { + def replacements = [ + DATE: getDate("yyyy-MM-dd"), + CHANNEL: propertiesChannelName, + APPLICATION_NAME: applicationName, + GIT_HASH: gitHash, + GIT_BRANCH: gitBranch, + VERSION: JALVIEW_VERSION, + JAVA_VERSION: JAVA_VERSION, + VERSION_UNDERSCORES: JALVIEW_VERSION_UNDERSCORES + ] def output = input - output = output.replaceAll("__DATE__", getDate("yyyy-MM-dd")) - output = output.replaceAll("__CHANNEL__", propertiesChannelName) - output = output.replaceAll("__APPLICATION_NAME__", applicationName) - output = output.replaceAll("__GIT_HASH__", gitHash) - output = output.replaceAll("__GIT_BRANCH__", gitBranch) - output = output.replaceAll("__VERSION__", JALVIEW_VERSION) - output = output.replaceAll("__JAVA_VERSION__", JAVA_VERSION) - output = output.replaceAll("__VERSION_UNDERSCORES__", JALVIEW_VERSION_UNDERSCORES) - return output -} - -def ulToList(Element ul, int indent) { - def list = "" - ul.children().each { e -> - if (e.tagName().toLowerCase() == "ul") { - list = list.stripTrailing() + "\n" - list += ulToList(e, indent+1) - } else if (e.tagName().toLowerCase() == "li") { - def issues = [] - def issuesText = "" - def mdItem = " " * indent + "- " - e.childNodes().any {n -> - if (n.nodeName().equals("#comment")) { - mdItem += "{{}}${n.getData()}{{}} " - issuesText = n.getData().trim() - issues = n.getData().trim().split(/[,\s]+/).findAll { it.startsWith("JAL-") } - return true - } - } - def safeText = issuesText.replaceAll('"', '\"') - def joinedIssues = issues.join(",") - def issuesLink = (issues.size() > 0) ? - " {{< jal issue=\"${joinedIssues}\" alt=\"${safeText}\" >}}" - : "" - def addedIssues = false - e.childNodes().each { node -> - if (node.nodeName().toLowerCase() == "ul") { - // add issues link before sub list if it's last - if (node.nextSibling() == null) { - mdItem += "${issuesLink}" - addedIssues = true - } - mdItem = mdItem.stripTrailing() + "\n" - mdItem += ulToList(node, indent+1) - } else if (node.nodeName() == "#text") { - mdItem += node.text() - } else if (node.nodeName().toLowerCase() == "strong") { - mdItem += "**${node.text()}**" - } else if (node.nodeName().toLowerCase() == "em") { - // if (true || (node.text().length() > 50 && node.text().count(" ") > 10) || (!node.nextSibling().text().startsWith("\\s*\\n"))) { - mdItem += "*${node.text()}*" - // } else { - // mdItem += "### "+node.text() - // } - } else if (node.nodeName().toLowerCase() == "br") { - mdItem += "
\n" - } else if (node.nodeName().toLowerCase() == "a") { - mdItem += "[${node.text()}](${node.attr('href')})" - } - } - if (!addedIssues) { - mdItem += issuesLink - } - list += mdItem.stripTrailing()+"\n" - } else if (e.tagName().toLowerCase() == "div") { - e.children().each { eee -> - list += ulToList(eee, indent) - } - } else if (e.tagName().toLowerCase() == "strong") { - list += "**${e.text()}**" - } else if (e.tagName().toLowerCase() == "em") { - if (e.text().length() > 50 && e.text().count(" ") > 20) { - list += "*${e.text()}*" - } else { - list += "\n\n### ${e.text()}\n\n" - } + if (extras != null) { + extras.each{ k, v -> + output = output.replaceAll("__${k}__", ((v == null)?"":v)) } } - return list + replacements.each{ k, v -> + output = output.replaceAll("__${k}__", ((v == null)?"":v)) + } + return output } task hugoTemplates { @@ -2478,104 +2418,94 @@ task hugoTemplates { def hugoTemplatesDir = file("${jalviewDir}/${hugo_templates_dir}") def hugoBuildDir = "${jalviewDir}/${hugo_build_dir}" def templateFiles = fileTree(dir: hugoTemplatesDir) + def releaseMdFile = file("${jalviewDir}/${releases_dir}/release-${JALVIEW_VERSION_UNDERSCORES}.md") + def whatsnewMdFile = file("${jalviewDir}/${whatsnew_dir}/whatsnew-${JALVIEW_VERSION_UNDERSCORES}.md") + def releaseTemplateFile = file("${jalviewDir}/${releases_template}") + def releasesTemplateFile = file("${jalviewDir}/${releases_template}") + def whatsnewTemplateFile = file("${jalviewDir}/${whatsnew_template}") doFirst { // specific release template for version archive - def summary = "${applicationName} ${JALVIEW_VERSION}" def changes = "" - def oldDate = null + def whatsnew = null + def givenDate = null + def givenChannel = null + def givenVersion = null if (CHANNEL == "RELEASE") { def releasesHtmlFile = file("${helpSourceDir}/${releases_html}") - //HTMLPARSE - def html = releasesHtmlFile.text - def doc = Jsoup.parse(html) - def table = doc.select("table").first() - def headings = table.select("tr").first().select("th").collect { it.text() } - def releaseRow = null - def trs = table.select("tr") - trs.any { tr -> - def tds = tr.select("td") - if (tds.size() == 0) - return false - def releaseTd = tds.first() - if (releaseTd.text().startsWith("${JALVIEW_VERSION} ")) { - releaseRow = tr - return true - } - } - - if (releaseRow != null && headings != null && headings.size() == 3) { - def releaseTd = releaseRow.select("td").first() - def spaceIndex = releaseTd.text().indexOf(" ") - if (spaceIndex >= 0) { - oldDate = new Date().parse("dd/MM/yyyy", releaseTd.text().substring(spaceIndex+1)) - } - def releaseCells = releaseRow.select("td") - if (releaseCells.size() == 3) { - def title1 = headings[1] - def title2 = headings[2] - - def lastDotIndex = JALVIEW_VERSION.lastIndexOf(".") - if (lastDotIndex > 0) { - try { - def patchRelease = JALVIEW_VERSION.substring(lastDotIndex+1) as Integer - def patchReleaseString = null - if (patchRelease == 0) { - patchReleaseString = "first minor" - } else if (patchRelease == 1) { - patchReleaseString = "first patch" - } else if (patchRelease == 2) { - patchReleaseString = "second patch" - } else if (patchRelease == 3) { - patchReleaseString = "third patch" - } else if (patchRelease == 4) { - patchReleaseString = "fourth patch" - } else if (patchRelease == 5) { - patchReleaseString = "fifth patch" - } else if (patchRelease == 6) { - patchReleaseString = "sixth patch" - } else if (patchRelease == 7) { - patchReleaseString = "seventh patch" - } else if (patchRelease > 13 && (patchRelease % 10 == 1)) { - patchReleaseString += "st" - } else if (patchRelease > 13 && (patchRelease % 10 == 2)) { - patchReleaseString += "nd" - } else if (patchRelease > 13 && (patchRelease % 10 == 3)) { - patchReleaseString += "rd" - } else if (patchRelease != null) { - patchReleaseString += "th" - } - summary += (patchReleaseString != null) ? " is the ${patchReleaseString} release in the ${JALVIEW_VERSION.substring(0,lastDotIndex)} series." : "" - } catch(Exception e) { - summary = "" - } + //MD_PARSE + if (releaseMdFile.exists()) { + def inFrontMatter = false + def itemComment = null + def firstLine = true + releaseMdFile.eachLine { line -> + if (line.matches("---")) { + def prev = inFrontMatter + inFrontMatter = firstLine + if (inFrontMatter != prev) + return } - - [1,2].each { col -> - def colChanges = "" - if (headings[col] != null && headings[col].size() > 0) { - releaseCells[col].children().each { e -> - if (e.tagName().toLowerCase() == "ul" || e.tagName().toLowerCase() == "div") { - if (colChanges != "") { - colChanges = colChanges.stripTrailing() + "\n" - } - colChanges += ulToList(e, 0) - } else if (e.tagName().toLowerCase() == "em") { - colChanges += "\n\n### ${e.text()}\n\n" - } - } - } - if (colChanges != "") { - def className = headings[col].replaceAll(" ","_").toLowerCase().replaceAll("[^_a-z]","") - changes += "
\n\n## ${headings[col]}\n\n${colChanges}\n
\n\n" + if (inFrontMatter) { + def m = null + if (m = line =~ /^date:\s*([0-9\-]+)/) { + givenDate = new Date().parse("yyyy-MM-dd", m[0][1]) + } else if (m = line =~ /^channel:\s*(\S+)/) { + givenChannel = m[0][1] + } else if (m = line =~ /^version:\s*(\S+)/) { + givenVersion = m[0][1] } + } else { + changes += line+"\n" } + firstLine = false + } + if (givenVersion != null && givenVersion != JALVIEW_VERSION) { + throw new GradleException("'version' header (${givenVersion}) found in ${releaseMdFile} does not match JALVIEW_VERSION (${JALVIEW_VERSION})") + } + if (whatsnewMdFile.exists()) { + whatsnew = whatsnewMdFile.text + } + def content = releaseMdFile.text + } + } - if (changes != "") { - changes = "
\n\n${changes}\n\n
" + def changesHugo = null + if (changes != null) { + changesHugo = '
\n\n' + def inSection = false + changes.eachLine { line -> + def m = null + if (m = line =~ /^##([^#].*)$/) { + if (inSection) { + changesHugo += "
\n\n" + } + def section = m[0][1].trim() + section = section.toLowerCase() + section = section.replaceAll(/ +/, "_") + section = section.replaceAll(/[^a-z0-9_\-]/, "") + changesHugo += "
\n\n" + inSection = true + } else if (m = line =~ /^(\s*-\s*)(.*)/) { + def comment = m[0][2].trim() + if (comment != "") { + comment = comment.replaceAll('"', """) + def issuekeys = [] + comment.eachMatch(/JAL-\d+/) { jal -> issuekeys += jal } + def newline = m[0][1] + if (comment.trim() != "") + newline += "{{}}${comment}{{}} " + newline += m[0][3].trim() + if (issuekeys.size() > 0) + newline += " {{< jal issue=\"${issuekeys.join(",")}\" alt=\"${comment}\" >}}" + line = newline } } + changesHugo += line+"\n" + } + if (inSection) { + changesHugo += "\n
\n\n" } + changesHugo += '' } templateFiles.each{ templateFile -> @@ -2584,7 +2514,7 @@ task hugoTemplates { def newRelPathName = hugoTemplateSubstitutions( relPath.toString() ) def outPathName = string("${hugoBuildDir}/$newRelPathName") - + copy { from templateFile rename(templateFile.getName(), newFileName) @@ -2593,30 +2523,21 @@ task hugoTemplates { def newFile = file("${outPathName}/${newFileName}".toString()) def content = newFile.text - // summary in version archive release pages only appears from 2.10.2b2 onwards - if (oldDate.format("yyyy") as Integer >= 2019 || JALVIEW_VERSION.startsWith("2.10")) { - def summaryFile = file("${jalviewDir}/${whatsnew_dir}/whatsnew-${JALVIEW_VERSION_UNDERSCORES}.md") - if (summaryFile.exists()) { - summary = summaryFile.text - } else { - summary = "" - } - content = content.replaceAll("__SUMMARY__", summary) - } else { - content = content.replaceAll("__SUMMARY__", "") - } - content = content.replaceAll("__CHANGES__", changes) - if (oldDate != null) { - if (oldDate[java.util.Calendar.YEAR] < 90) { - oldDate[java.util.Calendar.YEAR] += 2000 - } - content = content.replaceAll("__DATE__", oldDate.format("yyyy-MM-dd")) - } - newFile.text = hugoTemplateSubstitutions(content) + newFile.text = hugoTemplateSubstitutions(content, + [ + WHATSNEW: whatsnew, + CHANGES: changesHugo, + DATE: givenDate.format("yyyy-MM-dd") + ] + ) } + } inputs.dir(hugoTemplatesDir) + inputs.file(releaseTemplateFile) + inputs.file(releasesTemplateFile) + inputs.file(whatsnewTemplateFile) inputs.property("JALVIEW_VERSION", { JALVIEW_VERSION }) inputs.property("CHANNEL", { CHANNEL }) } diff --git a/gradle.properties b/gradle.properties index 239fcf6..58a5110 100644 --- a/gradle.properties +++ b/gradle.properties @@ -63,7 +63,7 @@ hugo_data_installers_dir = data/installers hugo_version_archive_dir = content/development/archive hugo_templates_dir = utils/hugo/templates releases_template = help/templates/releases.html -whatsnew_template = help/templates/whatsnew.html +whatsnew_template = help/templates/whatsNew.html releases_html = html/releases.html whatsnew_dir = help/markdown/whatsnew releases_dir = help/markdown/releases diff --git a/help/templates/whatsNew.html b/help/templates/whatsNew.html new file mode 100755 index 0000000..6828145 --- /dev/null +++ b/help/templates/whatsNew.html @@ -0,0 +1,97 @@ + + + +What's new ? + + +

+ Welcome to Jalview Version $$Version-Rel$$ !!
+

+

+__WHATSNEW__ +

+

+ This release series provides support for two popular 3D + structure visualisation tools, new features for discovery of 3D + structures, improved platform integration and a new command line + tool allowing Jalview to be more easily called from scripts.

+ +

+ View predicted protein structures via 3D-Beacons
+ Jalview 2.11.2's Structure + Chooser includes a client for the 3D-Beacons Network. Launched in + 2021, the 3D-Beacons network (www.ebi.ac.uk/pdbe/pdbe-kb/3dbeacons/) + provides a central point for the retrieval of predicted and observed + 3D structures for sequences in Uniprot, including homology models + from Swiss-model and deep learning based predictions from the EBI's + Alphafold database (Orengo et al. 2020, doi:10.12688/f1000research.20559.1).
+

+ +

+ Support for viewing structures with ChimeraX and + Pymol
Jalview's 3D structure viewer system has been + re-architected to allow easier integration of external structure + viewers, and takes advantage of the strucViz2 Chimera communications + library developed by Scooter Morris (doi:10.1093/bioinformatics/btm329).

+ The Structures + Preferences tab provides new options allowing ChimeraX and + Pymol to be used for visualising external 3D structures. Views + from all structure viewers are saved in Jalview Projects, allowing + them to be shared with others using Jalview 2.11.2 or later, + providing they have the same viewer installed and configured to be + used with Jalview.

Jalview + 2.11.2 has been tested with Pymol 2.5.0 (community) and 2.5.2 + (incentive). For ChimeraX, we recommend using v1.3 or later. +

+

Other highlights include:

+ + + +

+ For the full details, see the Jalview 2.11.2.1 + release notes. +

+

+ Known Issues
The following known issues will + be addressed in a minor patch release. + +

+

+ + diff --git a/utils/hugo/old_task_to_extract_markdown_from_old_releases_html.txt b/utils/hugo/old_task_to_extract_markdown_from_old_releases_html.txt new file mode 100644 index 0000000..8f2143e --- /dev/null +++ b/utils/hugo/old_task_to_extract_markdown_from_old_releases_html.txt @@ -0,0 +1,221 @@ + +def ulToList(Element ul, int indent) { + def list = "" + ul.children().each { e -> + if (e.tagName().toLowerCase() == "ul") { + list = list.stripTrailing() + "\n" + list += ulToList(e, indent+1) + } else if (e.tagName().toLowerCase() == "li") { + def issues = [] + def issuesText = "" + def mdItem = " " * indent + "- " + e.childNodes().any {n -> + if (n.nodeName().equals("#comment")) { + mdItem += "{{}}${n.getData()}{{}} " + issuesText = n.getData().trim() + issues = n.getData().trim().split(/[,\s]+/).findAll { it.startsWith("JAL-") } + return true + } + } + def safeText = issuesText.replaceAll('"', '\"') + def joinedIssues = issues.join(",") + def issuesLink = (issues.size() > 0) ? + " {{< jal issue=\"${joinedIssues}\" alt=\"${safeText}\" >}}" + : "" + def addedIssues = false + e.childNodes().each { node -> + if (node.nodeName().toLowerCase() == "ul") { + // add issues link before sub list if it's last + if (node.nextSibling() == null) { + mdItem += "${issuesLink}" + addedIssues = true + } + mdItem = mdItem.stripTrailing() + "\n" + mdItem += ulToList(node, indent+1) + } else if (node.nodeName() == "#text") { + mdItem += node.text() + } else if (node.nodeName().toLowerCase() == "strong") { + mdItem += "**${node.text()}**" + } else if (node.nodeName().toLowerCase() == "em") { + // if (true || (node.text().length() > 50 && node.text().count(" ") > 10) || (!node.nextSibling().text().startsWith("\\s*\\n"))) { + mdItem += "*${node.text()}*" + // } else { + // mdItem += "### "+node.text() + // } + } else if (node.nodeName().toLowerCase() == "br") { + mdItem += "
\n" + } else if (node.nodeName().toLowerCase() == "a") { + mdItem += "[${node.text()}](${node.attr('href')})" + } + } + if (!addedIssues) { + mdItem += issuesLink + } + list += mdItem.stripTrailing()+"\n" + } else if (e.tagName().toLowerCase() == "div") { + e.children().each { eee -> + list += ulToList(eee, indent) + } + } else if (e.tagName().toLowerCase() == "strong") { + list += "**${e.text()}**" + } else if (e.tagName().toLowerCase() == "em") { + if (e.text().length() > 50 && e.text().count(" ") > 20) { + list += "*${e.text()}*" + } else { + list += "\n\n### ${e.text()}\n\n" + } + } + } + return list +} + +task hugoTemplates { + group "website" + description "Create partially populated md pages for hugo website build" + + def hugoTemplatesDir = file("${jalviewDir}/${hugo_templates_dir}") + def hugoBuildDir = "${jalviewDir}/${hugo_build_dir}" + def templateFiles = fileTree(dir: hugoTemplatesDir) + + doFirst { + // specific release template for version archive + def summary = "${applicationName} ${JALVIEW_VERSION}" + def changes = "" + def oldDate = null + if (CHANNEL == "RELEASE") { + def releasesHtmlFile = file("${helpSourceDir}/${releases_html}") + //HTMLPARSE + def html = releasesHtmlFile.text + def doc = Jsoup.parse(html) + def table = doc.select("table").first() + def headings = table.select("tr").first().select("th").collect { it.text() } + def releaseRow = null + def trs = table.select("tr") + trs.any { tr -> + def tds = tr.select("td") + if (tds.size() == 0) + return false + def releaseTd = tds.first() + if (releaseTd.text().startsWith("${JALVIEW_VERSION} ")) { + releaseRow = tr + return true + } + } + + if (releaseRow != null && headings != null && headings.size() == 3) { + def releaseTd = releaseRow.select("td").first() + def spaceIndex = releaseTd.text().indexOf(" ") + if (spaceIndex >= 0) { + oldDate = new Date().parse("dd/MM/yyyy", releaseTd.text().substring(spaceIndex+1)) + } + def releaseCells = releaseRow.select("td") + if (releaseCells.size() == 3) { + def title1 = headings[1] + def title2 = headings[2] + + def lastDotIndex = JALVIEW_VERSION.lastIndexOf(".") + if (lastDotIndex > 0) { + try { + def patchRelease = JALVIEW_VERSION.substring(lastDotIndex+1) as Integer + def patchReleaseString = null + if (patchRelease == 0) { + patchReleaseString = "first minor" + } else if (patchRelease == 1) { + patchReleaseString = "first patch" + } else if (patchRelease == 2) { + patchReleaseString = "second patch" + } else if (patchRelease == 3) { + patchReleaseString = "third patch" + } else if (patchRelease == 4) { + patchReleaseString = "fourth patch" + } else if (patchRelease == 5) { + patchReleaseString = "fifth patch" + } else if (patchRelease == 6) { + patchReleaseString = "sixth patch" + } else if (patchRelease == 7) { + patchReleaseString = "seventh patch" + } else if (patchRelease > 13 && (patchRelease % 10 == 1)) { + patchReleaseString += "st" + } else if (patchRelease > 13 && (patchRelease % 10 == 2)) { + patchReleaseString += "nd" + } else if (patchRelease > 13 && (patchRelease % 10 == 3)) { + patchReleaseString += "rd" + } else if (patchRelease != null) { + patchReleaseString += "th" + } + summary += (patchReleaseString != null) ? " is the ${patchReleaseString} release in the ${JALVIEW_VERSION.substring(0,lastDotIndex)} series." : "" + } catch(Exception e) { + summary = "" + } + } + + [1,2].each { col -> + def colChanges = "" + if (headings[col] != null && headings[col].size() > 0) { + releaseCells[col].children().each { e -> + if (e.tagName().toLowerCase() == "ul" || e.tagName().toLowerCase() == "div") { + if (colChanges != "") { + colChanges = colChanges.stripTrailing() + "\n" + } + colChanges += ulToList(e, 0) + } else if (e.tagName().toLowerCase() == "em") { + colChanges += "\n\n### ${e.text()}\n\n" + } + } + } + if (colChanges != "") { + def className = headings[col].replaceAll(" ","_").toLowerCase().replaceAll("[^_a-z]","") + changes += "
\n\n## ${headings[col]}\n\n${colChanges}\n
\n\n" + } + } + + if (changes != "") { + changes = "
\n\n${changes}\n\n
" + } + } + } + } + + templateFiles.each{ templateFile -> + def newFileName = string(hugoTemplateSubstitutions(templateFile.getName())) + def relPath = hugoTemplatesDir.toPath().relativize(templateFile.toPath()).getParent() + def newRelPathName = hugoTemplateSubstitutions( relPath.toString() ) + + def outPathName = string("${hugoBuildDir}/$newRelPathName") + + copy { + from templateFile + rename(templateFile.getName(), newFileName) + into outPathName + } + + def newFile = file("${outPathName}/${newFileName}".toString()) + def content = newFile.text + // summary in version archive release pages only appears from 2.10.2b2 onwards + if (oldDate.format("yyyy") as Integer >= 2019 || JALVIEW_VERSION.startsWith("2.10")) { + def summaryFile = file("${jalviewDir}/${whatsnew_dir}/whatsnew-${JALVIEW_VERSION_UNDERSCORES}.md") + if (summaryFile.exists()) { + summary = summaryFile.text + } else { + summary = "" + } + content = content.replaceAll("__SUMMARY__", summary) + } else { + content = content.replaceAll("__SUMMARY__", "") + } + content = content.replaceAll("__CHANGES__", changes) + if (oldDate != null) { + if (oldDate[java.util.Calendar.YEAR] < 90) { + oldDate[java.util.Calendar.YEAR] += 2000 + } + content = content.replaceAll("__DATE__", oldDate.format("yyyy-MM-dd")) + } + newFile.text = hugoTemplateSubstitutions(content) + } + } + + inputs.dir(hugoTemplatesDir) + inputs.property("JALVIEW_VERSION", { JALVIEW_VERSION }) + inputs.property("CHANNEL", { CHANNEL }) +} + diff --git a/utils/hugo/templates/content/development/archive/Version-__VERSION_UNDERSCORES__/_index.md b/utils/hugo/templates/content/development/archive/Version-__VERSION_UNDERSCORES__/_index.md index dc418ad..54a11dc 100644 --- a/utils/hugo/templates/content/development/archive/Version-__VERSION_UNDERSCORES__/_index.md +++ b/utils/hugo/templates/content/development/archive/Version-__VERSION_UNDERSCORES__/_index.md @@ -11,6 +11,6 @@ cascade: jalviewjslink: false --- -__SUMMARY__ +__WHATSNEW__ __CHANGES__ -- 1.7.10.2