X-Git-Url: http://source.jalview.org/gitweb/?p=jalview.git;a=blobdiff_plain;f=build.gradle;h=4f9798cc05f9739e4e5f9b01587a9ad5d0068697;hp=79fb2d97cc7df85b8456de67721b6c712386f18b;hb=282aa9129a74235dc8d6c22b123d51eb1abd2f55;hpb=f2e585f8dee5aba8522bbe622f4d6573648073b4 diff --git a/build.gradle b/build.gradle index 79fb2d9..4f9798c 100644 --- a/build.gradle +++ b/build.gradle @@ -27,6 +27,7 @@ import com.google.common.hash.HashCode import com.google.common.hash.Hashing import com.google.common.io.Files import org.jsoup.Jsoup +import org.jsoup.nodes.Element buildscript { repositories { @@ -47,7 +48,7 @@ plugins { id "com.diffplug.gradle.spotless" version "3.28.0" id 'com.github.johnrengelman.shadow' version '4.0.3' id 'com.install4j.gradle' version '9.0.6' - id 'com.dorongold.task-tree' version '1.5' // only needed to display task dependency tree with gradle task1 [task2 ...] taskTree + id 'com.dorongold.task-tree' version '2.1.0' // only needed to display task dependency tree with gradle task1 [task2 ...] taskTree id 'com.palantir.git-version' version '0.13.0' apply false } @@ -137,6 +138,7 @@ ext { if (findProperty("JALVIEW_VERSION")==null || "".equals(JALVIEW_VERSION)) { JALVIEW_VERSION = releaseProps.get("jalview.version") } + println("JALVIEW_VERSION is set to '${JALVIEW_VERSION}'") // this property set when running Eclipse headlessly j2sHeadlessBuildProperty = string("net.sf.j2s.core.headlessbuild") @@ -1198,6 +1200,214 @@ task convertMdFiles { } +def hugoTemplateSubstitutions(String input, Map extras=null) { + def replacements = [ + DATE: getDate("yyyy-MM-dd"), + CHANNEL: propertiesChannelName, + APPLICATION_NAME: applicationName, + GIT_HASH: gitHash, + GIT_BRANCH: gitBranch, + VERSION: JALVIEW_VERSION, + JAVA_VERSION: JAVA_VERSION, + VERSION_UNDERSCORES: JALVIEW_VERSION_UNDERSCORES, + DRAFT: "false", + JVL_HEADER: "" + ] + def output = input + if (extras != null) { + extras.each{ k, v -> + output = output.replaceAll("__${k}__", ((v == null)?"":v)) + } + } + replacements.each{ k, v -> + output = output.replaceAll("__${k}__", ((v == null)?"":v)) + } + return output +} + +def mdFileComponents(File mdFile, def dateOnly=false) { + def map = [:] + def content = "" + if (mdFile.exists()) { + def inFrontMatter = false + def firstLine = true + mdFile.eachLine { line -> + if (line.matches("---")) { + def prev = inFrontMatter + inFrontMatter = firstLine + if (inFrontMatter != prev) + return false + } + if (inFrontMatter) { + def m = null + if (m = line =~ /^date:\s*(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2})/) { + map["date"] = new Date().parse("yyyy-MM-dd HH:mm:ss", m[0][1]) + } else if (m = line =~ /^date:\s*(\d{4}-\d{2}-\d{2})/) { + map["date"] = new Date().parse("yyyy-MM-dd", m[0][1]) + } else if (m = line =~ /^channel:\s*(\S+)/) { + map["channel"] = m[0][1] + } else if (m = line =~ /^version:\s*(\S+)/) { + map["version"] = m[0][1] + } else if (m = line =~ /^\s*([^:]+)\s*:\s*(\S.*)/) { + map[ m[0][1] ] = m[0][2] + } + if (dateOnly && map["date"] != null) { + return false + } + } else { + if (dateOnly) + return false + content += line+"\n" + } + firstLine = false + } + } + return dateOnly ? map["date"] : [map, content] +} + +task hugoTemplates { + group "website" + description "Create partially populated md pages for hugo website build" + + def hugoTemplatesDir = file("${jalviewDir}/${hugo_templates_dir}") + def hugoBuildDir = "${jalviewDir}/${hugo_build_dir}" + def templateFiles = fileTree(dir: hugoTemplatesDir) + def releaseMdFile = file("${jalviewDir}/${releases_dir}/release-${JALVIEW_VERSION_UNDERSCORES}.md") + def whatsnewMdFile = file("${jalviewDir}/${whatsnew_dir}/whatsnew-${JALVIEW_VERSION_UNDERSCORES}.md") + def oldJvlFile = file("${jalviewDir}/${hugo_old_jvl}") + def jalviewjsFile = file("${jalviewDir}/${hugo_jalviewjs}") + + doFirst { + // specific release template for version archive + def changes = "" + def whatsnew = null + def givenDate = null + def givenChannel = null + def givenVersion = null + if (CHANNEL == "RELEASE") { + def (map, content) = mdFileComponents(releaseMdFile) + givenDate = map.date + givenChannel = map.channel + givenVersion = map.version + changes = content + if (givenVersion != null && givenVersion != JALVIEW_VERSION) { + throw new GradleException("'version' header (${givenVersion}) found in ${releaseMdFile} does not match JALVIEW_VERSION (${JALVIEW_VERSION})") + } + + if (whatsnewMdFile.exists()) + whatsnew = whatsnewMdFile.text + } + + def oldJvl = oldJvlFile.exists() ? oldJvlFile.collect{it} : [] + def jalviewjsLink = jalviewjsFile.exists() ? jalviewjsFile.collect{it} : [] + + def changesHugo = null + if (changes != null) { + changesHugo = '
\n\n' + def inSection = false + changes.eachLine { line -> + def m = null + if (m = line =~ /^##([^#].*)$/) { + if (inSection) { + changesHugo += "
\n\n" + } + def section = m[0][1].trim() + section = section.toLowerCase() + section = section.replaceAll(/ +/, "_") + section = section.replaceAll(/[^a-z0-9_\-]/, "") + changesHugo += "
\n\n" + inSection = true + } else if (m = line =~ /^(\s*-\s*)(.*?)()?\s*$/) { + def comment = m[0][2].trim() + if (comment != "") { + comment = comment.replaceAll('"', """) + def issuekeys = [] + comment.eachMatch(/JAL-\d+/) { jal -> issuekeys += jal } + def newline = m[0][1] + if (comment.trim() != "") + newline += "{{}}${comment}{{}} " + newline += m[0][3].trim() + if (issuekeys.size() > 0) + newline += " {{< jal issue=\"${issuekeys.join(",")}\" alt=\"${comment}\" >}}" + if (m[0][4] != null) + newline += m[0][4] + line = newline + } + } + changesHugo += line+"\n" + } + if (inSection) { + changesHugo += "\n
\n\n" + } + changesHugo += '' + } + + templateFiles.each{ templateFile -> + def newFileName = string(hugoTemplateSubstitutions(templateFile.getName())) + def relPath = hugoTemplatesDir.toPath().relativize(templateFile.toPath()).getParent() + def newRelPathName = hugoTemplateSubstitutions( relPath.toString() ) + + def outPathName = string("${hugoBuildDir}/$newRelPathName") + + copy { + from templateFile + rename(templateFile.getName(), newFileName) + into outPathName + } + + def newFile = file("${outPathName}/${newFileName}".toString()) + def content = newFile.text + newFile.text = hugoTemplateSubstitutions(content, + [ + WHATSNEW: whatsnew, + CHANGES: changesHugo, + DATE: givenDate == null ? "" : givenDate.format("yyyy-MM-dd"), + DRAFT: givenDate == null ? "true" : "false", + JALVIEWJSLINK: jalviewjsLink.contains(JALVIEW_VERSION) ? "true" : "false", + JVL_HEADER: oldJvl.contains(JALVIEW_VERSION) ? "jvl: true" : "" + ] + ) + } + + } + + inputs.file(oldJvlFile) + inputs.dir(hugoTemplatesDir) + inputs.property("JALVIEW_VERSION", { JALVIEW_VERSION }) + inputs.property("CHANNEL", { CHANNEL }) +} + +def getMdDate(File mdFile) { + return mdFileComponents(mdFile, true) +} + +def getMdSections(String content) { + def sections = [:] + def sectionContent = "" + def sectionName = null + content.eachLine { line -> + def m = null + if (m = line =~ /^##([^#].*)$/) { + if (sectionName != null) { + sections[sectionName] = sectionContent + sectionName = null + sectionContent = "" + } + sectionName = m[0][1].trim() + sectionName = sectionName.toLowerCase() + sectionName = sectionName.replaceAll(/ +/, "_") + sectionName = sectionName.replaceAll(/[^a-z0-9_\-]/, "") + } else if (sectionName != null) { + sectionContent += line+"\n" + } + } + if (sectionContent != null) { + sections[sectionName] = sectionContent + } + return sections +} + + task copyHelp(type: Copy) { def inputDir = helpSourceDir def outputDir = "${helpBuildDir}/${help_dir}" @@ -1233,6 +1443,142 @@ task copyHelp(type: Copy) { } +task releasesTemplates { + group "help" + description "Recreate whatsNew.html and releases.html from markdown files and templates in help" + + dependsOn copyHelp + + def releasesTemplateFile = file("${jalviewDir}/${releases_template}") + def whatsnewTemplateFile = file("${jalviewDir}/${whatsnew_template}") + def releasesHtmlFile = file("${helpBuildDir}/${help_dir}/${releases_html}") + def whatsnewHtmlFile = file("${helpBuildDir}/${help_dir}/${whatsnew_html}") + def releasesMdDir = "${jalviewDir}/${releases_dir}" + def whatsnewMdDir = "${jalviewDir}/${whatsnew_dir}" + + doFirst { + def releaseMdFile = file("${releasesMdDir}/release-${JALVIEW_VERSION_UNDERSCORES}.md") + def whatsnewMdFile = file("${whatsnewMdDir}/whatsnew-${JALVIEW_VERSION_UNDERSCORES}.md") + + if (CHANNEL == "RELEASE") { + if (!releaseMdFile.exists()) { + throw new GradleException("File ${releaseMdFile} must be created for RELEASE") + } + if (!whatsnewMdFile.exists()) { + throw new GradleException("File ${whatsnewMdFile} must be created for RELEASE") + } + } + + def releaseFiles = fileTree(dir: releasesMdDir, include: "release-*.md") + def releaseFilesDates = releaseFiles.collectEntries { + [(it): getMdDate(it)] + } + releaseFiles = releaseFiles.sort { a,b -> releaseFilesDates[a].compareTo(releaseFilesDates[b]) } + + def releasesTemplate = releasesTemplateFile.text + def m = releasesTemplate =~ /(?s)__VERSION_LOOP_START__(.*)__VERSION_LOOP_END__/ + def versionTemplate = m[0][1] + + MutableDataSet options = new MutableDataSet() + + def extensions = new ArrayList<>() + options.set(Parser.EXTENSIONS, extensions) + options.set(Parser.HTML_BLOCK_COMMENT_ONLY_FULL_LINE, true) + + Parser parser = Parser.builder(options).build() + HtmlRenderer renderer = HtmlRenderer.builder(options).build() + + def actualVersions = releaseFiles.collect { rf -> + def (rfMap, rfContent) = mdFileComponents(rf) + return rfMap.version + } + def versionsHtml = "" + def linkedVersions = [] + releaseFiles.reverse().each { rFile -> + def (rMap, rContent) = mdFileComponents(rFile) + + def versionLink = "" + def partialVersion = "" + def firstPart = true + rMap.version.split("\\.").each { part -> + def displayPart = ( firstPart ? "" : "." ) + part + partialVersion += displayPart + if ( + linkedVersions.contains(partialVersion) + || ( actualVersions.contains(partialVersion) && partialVersion != rMap.version ) + ) { + versionLink += displayPart + } else { + versionLink += "${displayPart}" + linkedVersions += partialVersion + } + firstPart = false + } + def displayDate = releaseFilesDates[rFile].format("dd/MM/yyyy") + + def lm = null + def rContentProcessed = "" + rContent.eachLine { line -> + if (lm = line =~ /^(\s*-)(\s*)(.*)$/) { + line = "${lm[0][1]}${lm[0][3]}${lm[0][2]}" + } else if (lm = line =~ /^###([^#]+.*)$/) { + line = "_${lm[0][1].trim()}_" + } + rContentProcessed += line + "\n" + } + + def rContentSections = getMdSections(rContentProcessed) + def rVersion = versionTemplate + if (rVersion != "") { + def rNewFeatures = rContentSections["new_features"] + def rIssuesResolved = rContentSections["issues_resolved"] + Node newFeaturesNode = parser.parse(rNewFeatures) + String newFeaturesHtml = renderer.render(newFeaturesNode) + Node issuesResolvedNode = parser.parse(rIssuesResolved) + String issuesResolvedHtml = renderer.render(issuesResolvedNode) + rVersion = hugoTemplateSubstitutions(rVersion, + [ + VERSION: rMap.version, + VERSION_LINK: versionLink, + DISPLAY_DATE: displayDate, + NEW_FEATURES: newFeaturesHtml, + ISSUES_RESOLVED: issuesResolvedHtml + ] + ) + versionsHtml += rVersion + } + } + + releasesTemplate = releasesTemplate.replaceAll("(?s)__VERSION_LOOP_START__.*__VERSION_LOOP_END__", versionsHtml) + releasesTemplate = hugoTemplateSubstitutions(releasesTemplate) + releasesHtmlFile.text = releasesTemplate + + if (whatsnewMdFile.exists()) { + def wnDisplayDate = releaseFilesDates[releaseMdFile] != null ? releaseFilesDates[releaseMdFile].format("dd MMMM yyyy") : "" + def whatsnewMd = hugoTemplateSubstitutions(whatsnewMdFile.text) + Node whatsnewNode = parser.parse(whatsnewMd) + String whatsnewHtml = renderer.render(whatsnewNode) + whatsnewHtml = whatsnewTemplateFile.text.replaceAll("__WHATS_NEW__", whatsnewHtml) + whatsnewHtmlFile.text = hugoTemplateSubstitutions(whatsnewHtml, + [ + DISPLAY_DATE: wnDisplayDate + ] + ) + } else if (gradle.taskGraph.hasTask(":linkCheck")) { + whatsnewHtmlFile.text = "Development build " + getDate("yyyy-MM-dd HH:mm:ss") + } + + } + + inputs.file(releasesTemplateFile) + inputs.file(whatsnewTemplateFile) + inputs.dir(releasesMdDir) + inputs.dir(whatsnewMdDir) + outputs.file(releasesHtmlFile) + outputs.file(whatsnewHtmlFile) +} + + task copyResources(type: Copy) { group = "build" description = "Copy (and make text substitutions in) the resources dir to the build area" @@ -1330,6 +1676,7 @@ task prepare { dependsOn buildResources dependsOn copyDocs dependsOn copyHelp + dependsOn releasesTemplates dependsOn convertMdFiles dependsOn buildIndices } @@ -1402,7 +1749,7 @@ task linkCheck(type: JavaExec) { def helpLinksCheckerOutFile = file("${jalviewDir}/${utils_dir}/HelpLinksChecker.out") classpath = files("${jalviewDir}/${utils_dir}") main = "HelpLinksChecker" - workingDir = jalviewDir + workingDir = "${helpBuildDir}" args = [ "${helpBuildDir}/${help_dir}", "-nointernet" ] def outFOS = new FileOutputStream(helpLinksCheckerOutFile, false) // false == don't append @@ -2387,163 +2734,10 @@ task dataInstallersJson { } } -def hugoTemplateSubstitutions(String input) { - def output = input - output = output.replaceAll("__DATE__", getDate("yyyy-MM-dd")) - output = output.replaceAll("__CHANNEL__", propertiesChannelName) - output = output.replaceAll("__APPLICATION_NAME__", applicationName) - output = output.replaceAll("__GIT_HASH__", gitHash) - output = output.replaceAll("__GIT_BRANCH__", gitBranch) - output = output.replaceAll("__VERSION__", JALVIEW_VERSION) - output = output.replaceAll("__JAVA_VERSION__", JAVA_VERSION) - output = output.replaceAll("__VERSION_UNDERSCORES__", JALVIEW_VERSION_UNDERSCORES) - return output -} - -task hugoTemplates { - group "website" - description "Create partially populated md pages for hugo website build" - - def hugoTemplatesDir = file("${jalviewDir}/${hugo_templates_dir}") - def hugoBuildDir = "${jalviewDir}/${hugo_build_dir}" - def templateFiles = fileTree(dir: hugoTemplatesDir) - - doFirst { - // specific release template for version archive - def summary = "${applicationName} ${JALVIEW_VERSION}" - def changes = "" - def oldDate = null - if (CHANNEL == "RELEASE") { - def releasesHtmlFile = file("${helpSourceDir}/${releases_html}") - //HTMLPARSE - def html = releasesHtmlFile.text - def doc = Jsoup.parse(html) - def table = doc.select("table").first() - def headings = table.select("tr").first().select("th").collect { it.text() } - def releaseRow = null - def trs = table.select("tr") - trs.any { tr -> - def tds = tr.select("td") - if (tds.size() == 0) - return false - def releaseTd = tds.first() - if (releaseTd.text().startsWith("${JALVIEW_VERSION} ")) { - releaseRow = tr - return true - } - } - - if (releaseRow != null && headings != null && headings.size() == 3) { - def releaseTd = releaseRow.select("td").first() - def spaceIndex = releaseTd.text().indexOf(" ") - if (spaceIndex >= 0) { - oldDate = new Date().parse("dd/MM/yyyy", releaseTd.text().substring(spaceIndex+1)) - } - def releaseCells = releaseRow.select("td") - if (releaseCells.size() == 3) { - def title1 = headings[1] - def title2 = headings[2] - - def lastDotIndex = JALVIEW_VERSION.lastIndexOf(".") - if (lastDotIndex > 0) { - def patchRelease = JALVIEW_VERSION.substring(lastDotIndex+1) as Integer - def patchReleaseString = null - if (patchRelease == 0) { - patchReleaseString = "first minor" - } else if (patchRelease == 1) { - patchReleaseString = "first patch" - } else if (patchRelease == 2) { - patchReleaseString = "second patch" - } else if (patchRelease == 3) { - patchReleaseString = "third patch" - } else if (patchRelease == 4) { - patchReleaseString = "fourth patch" - } else if (patchRelease == 5) { - patchReleaseString = "fifth patch" - } else if (patchRelease == 6) { - patchReleaseString = "sixth patch" - } else if (patchRelease == 7) { - patchReleaseString = "seventh patch" - } else if (patchRelease > 13 && (patchRelease % 10 == 1)) { - patchReleaseString += "st" - } else if (patchRelease > 13 && (patchRelease % 10 == 2)) { - patchReleaseString += "nd" - } else if (patchRelease > 13 && (patchRelease % 10 == 3)) { - patchReleaseString += "rd" - } else if (patchRelease != null) { - patchReleaseString += "th" - } - summary += (patchReleaseString != null) ? " is the ${patchReleaseString} release in the ${JALVIEW_VERSION.substring(0,lastDotIndex)} series." : "" - } - - [1,2].each { col -> - if (headings[col] != null && headings[col].size() > 0) { - def noheadings = true - releaseCells[col].children().each { e -> - if (e.tagName().toLowerCase() == "ul") { - e.select("li").each { li -> - def issues = [] - def mdItem = "- " - li.childNodes().any {n -> - if (n.nodeName().equals("#comment")) { - mdItem += "${n} " - issues = n.getData().trim().split(/[,\s]+/) - return true - } - } - mdItem += li.text() - issues.each { jal -> - mdItem += " {{< jal issue=\"${jal}\" >}}" - } - if (noheadings) { - changes += "\n### ${headings[1]}\n\n" - noheadings = false - } - changes += "${mdItem}\n" - } - } else if (e.tag() == "em") { - changes += "\n#### ${e.text()}\n\n" - noheadings = false - } - } - } - } - - } - } - } - - templateFiles.each{ templateFile -> - def newFileName = string(hugoTemplateSubstitutions(templateFile.getName())) - def relPath = hugoTemplatesDir.toPath().relativize(templateFile.toPath()).getParent() - def newRelPathName = hugoTemplateSubstitutions( relPath.toString() ) - - def outPathName = string("${hugoBuildDir}/$newRelPathName") - - copy { - from templateFile - rename(templateFile.getName(), newFileName) - into outPathName - } - - def newFile = file("${outPathName}/${newFileName}".toString()) - def content = newFile.text - content = content.replaceAll("__SUMMARY__", summary) - content = content.replaceAll("__CHANGES__", changes) - if (oldDate != null) { - content = content.replaceAll("__DATE__", oldDate.format("yyyy-MM-dd")) - } - newFile.text = hugoTemplateSubstitutions(content) - } - } - - inputs.dir(hugoTemplatesDir) - inputs.property("JALVIEW_VERSION", { JALVIEW_VERSION }) - inputs.property("CHANNEL", { CHANNEL }) -} - - task helppages { + group "help" + description "Copies all help pages to build dir. Runs ant task 'pubhtmlhelp'." + dependsOn copyHelp dependsOn pubhtmlhelp