X-Git-Url: http://source.jalview.org/gitweb/?a=blobdiff_plain;f=build.gradle;h=3b81404290f0068e6827038e1b74f157cd43822b;hb=HEAD;hp=958343bfbcc2612e654f06fc81f50b1f244632db;hpb=a1ad718d48d84b35e1e5e69ff34fc0ca7f69790e;p=jalview.git diff --git a/build.gradle b/build.gradle index 958343b..3b81404 100644 --- a/build.gradle +++ b/build.gradle @@ -9,6 +9,11 @@ import org.gradle.util.ConfigureUtil import org.gradle.plugins.ide.eclipse.model.Output import org.gradle.plugins.ide.eclipse.model.Library import java.security.MessageDigest +import java.util.regex.Matcher +import java.util.concurrent.Executors +import java.util.concurrent.Future +import java.util.concurrent.ScheduledExecutorService +import java.util.concurrent.TimeUnit import groovy.transform.ExternalizeMethods import groovy.util.XmlParser import groovy.xml.XmlUtil @@ -37,6 +42,7 @@ buildscript { dependencies { classpath "com.vladsch.flexmark:flexmark-all:0.62.0" classpath "org.jsoup:jsoup:1.14.3" + classpath "com.eowise:gradle-imagemagick:0.5.1" } } @@ -46,9 +52,9 @@ plugins { id 'application' id 'eclipse' id "com.diffplug.gradle.spotless" version "3.28.0" - id 'com.github.johnrengelman.shadow' version '4.0.3' - id 'com.install4j.gradle' version '9.0.6' - id 'com.dorongold.task-tree' version '1.5' // only needed to display task dependency tree with gradle task1 [task2 ...] taskTree + id 'com.github.johnrengelman.shadow' version '6.0.0' + id 'com.install4j.gradle' version '10.0.3' + id 'com.dorongold.task-tree' version '2.1.1' // only needed to display task dependency tree with gradle task1 [task2 ...] taskTree id 'com.palantir.git-version' version '0.13.0' apply false } @@ -108,8 +114,12 @@ ext { getdownChannelName = CHANNEL.toLowerCase() // default to "default". Currently only has different cosmetics for "develop", "release", "default" propertiesChannelName = ["develop", "release", "test-release", "jalviewjs", "jalviewjs-release" ].contains(getdownChannelName) ? getdownChannelName : "default" + channelDirName = propertiesChannelName // Import channel_properties - channelDir = string("${jalviewDir}/${channel_properties_dir}/${propertiesChannelName}") + if (getdownChannelName.startsWith("develop-")) { + channelDirName = "develop-SUFFIX" + } + channelDir = string("${jalviewDir}/${channel_properties_dir}/${channelDirName}") channelGradleProperties = string("${channelDir}/channel_gradle.properties") channelPropsFile = string("${channelDir}/${resource_dir}/${channel_props}") overrideProperties(channelGradleProperties, false) @@ -138,6 +148,7 @@ ext { if (findProperty("JALVIEW_VERSION")==null || "".equals(JALVIEW_VERSION)) { JALVIEW_VERSION = releaseProps.get("jalview.version") } + println("JALVIEW_VERSION is set to '${JALVIEW_VERSION}'") // this property set when running Eclipse headlessly j2sHeadlessBuildProperty = string("net.sf.j2s.core.headlessbuild") @@ -197,6 +208,8 @@ ext { testSourceDir = useClover ? cloverTestInstrDir : testDir testClassesDir = useClover ? cloverTestClassesDir : "${jalviewDir}/${test_output_dir}" + channelSuffix = "" + backgroundImageText = BACKGROUNDIMAGETEXT getdownChannelDir = string("${getdown_website_dir}/${propertiesChannelName}") getdownAppBaseDir = string("${jalviewDir}/${getdownChannelDir}/${JAVA_VERSION}") getdownArchiveDir = string("${jalviewDir}/${getdown_archive_dir}") @@ -214,12 +227,15 @@ ext { getdownLauncher = string("${jalviewDir}/${getdown_lib_dir}/${getdown_launcher}") getdownAppDistDir = getdown_app_dir_alt getdownImagesDir = string("${jalviewDir}/${getdown_images_dir}") + getdownImagesBuildDir = string("${buildDir}/imagemagick/getdown") getdownSetAppBaseProperty = false // whether to pass the appbase and appdistdir to the application reportRsyncCommand = false jvlChannelName = CHANNEL.toLowerCase() install4jSuffix = CHANNEL.substring(0, 1).toUpperCase() + CHANNEL.substring(1).toLowerCase(); // BUILD -> Build install4jDMGDSStore = "${install4j_images_dir}/${install4j_dmg_ds_store}" - install4jDMGBackgroundImage = "${install4j_images_dir}/${install4j_dmg_background}" + install4jDMGBackgroundImageDir = "${install4j_images_dir}" + install4jDMGBackgroundImageBuildDir = "build/imagemagick/install4j" + install4jDMGBackgroundImageFile = "${install4j_dmg_background}" install4jInstallerName = "${jalview_name} Non-Release Installer" install4jExecutableName = install4j_executable_name install4jExtraScheme = "jalviewx" @@ -244,6 +260,7 @@ ext { testng_excluded_groups = "Not-bamboo" } install4jExtraScheme = "jalviewb" + backgroundImageText = true break case [ "RELEASE", "JALVIEWJS-RELEASE" ]: @@ -274,7 +291,7 @@ ext { getdownDir = string("${getdownChannelName}/${JAVA_VERSION}") getdownAppBase = file(getdownAppBaseDir).toURI().toString() if (!file("${ARCHIVEDIR}/${package_dir}").exists()) { - throw new GradleException("Must provide an ARCHIVEDIR value to produce an archive distribution") + throw new GradleException("Must provide an ARCHIVEDIR value to produce an archive distribution [did not find '${ARCHIVEDIR}/${package_dir}']") } else { package_dir = string("${ARCHIVEDIR}/${package_dir}") buildProperties = string("${ARCHIVEDIR}/${classes_dir}/${build_properties_file}") @@ -286,6 +303,23 @@ ext { install4jExtraScheme = "jalviewa" break + case ~/^DEVELOP-([\.\-\w]*)$/: + def suffix = Matcher.lastMatcher[0][1] + reportRsyncCommand = true + getdownSetAppBaseProperty = true + JALVIEW_VERSION=JALVIEW_VERSION+"-d${suffix}-${buildDate}" + install4jSuffix = "Develop ${suffix}" + install4jExtraScheme = "jalviewd" + install4jInstallerName = "${jalview_name} Develop ${suffix} Installer" + getdownChannelName = string("develop-${suffix}") + getdownChannelDir = string("${getdown_website_dir}/${getdownChannelName}") + getdownAppBaseDir = string("${jalviewDir}/${getdownChannelDir}/${JAVA_VERSION}") + getdownDir = string("${getdownChannelName}/${JAVA_VERSION}") + getdownAppBase = string("${getdown_channel_base}/${getdownDir}") + channelSuffix = string(suffix) + backgroundImageText = true + break + case "DEVELOP": reportRsyncCommand = true getdownSetAppBaseProperty = true @@ -295,6 +329,7 @@ ext { install4jSuffix = "Develop" install4jExtraScheme = "jalviewd" install4jInstallerName = "${jalview_name} Develop Installer" + backgroundImageText = true break case "TEST-RELEASE": @@ -309,6 +344,7 @@ ext { install4jSuffix = "Test" install4jExtraScheme = "jalviewt" install4jInstallerName = "${jalview_name} Test Installer" + backgroundImageText = true break case ~/^SCRATCH(|-[-\w]*)$/: @@ -332,6 +368,7 @@ ext { install4jSuffix = "Test-Local" install4jExtraScheme = "jalviewt" install4jInstallerName = "${jalview_name} Test Installer" + backgroundImageText = true break case [ "LOCAL", "JALVIEWJS" ]: @@ -482,16 +519,10 @@ ext { // for install4j JAVA_MIN_VERSION = JAVA_VERSION JAVA_MAX_VERSION = JAVA_VERSION - def jreInstallsDir = string(jre_installs_dir) + jreInstallsDir = string(jre_installs_dir) if (jreInstallsDir.startsWith("~/")) { jreInstallsDir = System.getProperty("user.home") + jreInstallsDir.substring(1) } - macosJavaVMDir = string("${jreInstallsDir}/jre-${JAVA_INTEGER_VERSION}-mac-x64/jre") - windowsJavaVMDir = string("${jreInstallsDir}/jre-${JAVA_INTEGER_VERSION}-windows-x64/jre") - linuxJavaVMDir = string("${jreInstallsDir}/jre-${JAVA_INTEGER_VERSION}-linux-x64/jre") - macosJavaVMTgz = string("${jreInstallsDir}/tgz/jre_${JAVA_INTEGER_VERSION}_mac_x64.tar.gz") - windowsJavaVMTgz = string("${jreInstallsDir}/tgz/jre_${JAVA_INTEGER_VERSION}_windows_x64.tar.gz") - linuxJavaVMTgz = string("${jreInstallsDir}/tgz/jre_${JAVA_INTEGER_VERSION}_linux_x64.tar.gz") install4jDir = string("${jalviewDir}/${install4j_utils_dir}") install4jConfFileName = string("jalview-install4j-conf.install4j") install4jConfFile = file("${install4jDir}/${install4jConfFileName}") @@ -513,6 +544,14 @@ ext { helpSourceDir = string("${helpParentDir}/${help_dir}") helpFile = string("${helpBuildDir}/${help_dir}/help.jhm") + convertBinary = null + convertBinaryExpectedLocation = imagemagick_convert + if (convertBinaryExpectedLocation.startsWith("~/")) { + convertBinaryExpectedLocation = System.getProperty("user.home") + convertBinaryExpectedLocation.substring(1) + } + if (file(convertBinaryExpectedLocation).exists()) { + convertBinary = convertBinaryExpectedLocation + } relativeBuildDir = file(jalviewDirAbsolutePath).toPath().relativize(buildDir.toPath()) jalviewjsBuildDir = string("${relativeBuildDir}/jalviewjs") @@ -533,11 +572,16 @@ ext { jalviewjsJ2sAltSettingsFileName = string("${jalviewDir}/${jalviewjs_j2s_alt_settings}") jalviewjsJ2sProps = null jalviewjsJ2sPlugin = jalviewjs_j2s_plugin + jalviewjsStderrLaunchFilename = "${jalviewjsSiteDir}/"+(file(jalviewjs_stderr_launch).getName()) eclipseWorkspace = null eclipseBinary = string("") eclipseVersion = string("") eclipseDebug = false + + jalviewjsChromiumUserDir = "${jalviewjsBuildDir}/${jalviewjs_chromium_user_dir}" + jalviewjsChromiumProfileDir = "${ext.jalviewjsChromiumUserDir}/${jalviewjs_chromium_profile_name}" + // ENDEXT } @@ -1029,7 +1073,7 @@ compileJava { // JBP->BS should the print statement in doFirst refer to compile_target_compatibility ? sourceCompatibility = compile_source_compatibility targetCompatibility = compile_target_compatibility - options.compilerArgs = additional_compiler_args + options.compilerArgs += additional_compiler_args options.encoding = "UTF-8" doFirst { print ("Setting target compatibility to "+compile_target_compatibility+"\n") @@ -1041,7 +1085,7 @@ compileJava { compileTestJava { sourceCompatibility = compile_source_compatibility targetCompatibility = compile_target_compatibility - options.compilerArgs = additional_compiler_args + options.compilerArgs += additional_compiler_args doFirst { print ("Setting target compatibility to "+targetCompatibility+"\n") } @@ -1199,6 +1243,214 @@ task convertMdFiles { } +def hugoTemplateSubstitutions(String input, Map extras=null) { + def replacements = [ + DATE: getDate("yyyy-MM-dd"), + CHANNEL: propertiesChannelName, + APPLICATION_NAME: applicationName, + GIT_HASH: gitHash, + GIT_BRANCH: gitBranch, + VERSION: JALVIEW_VERSION, + JAVA_VERSION: JAVA_VERSION, + VERSION_UNDERSCORES: JALVIEW_VERSION_UNDERSCORES, + DRAFT: "false", + JVL_HEADER: "" + ] + def output = input + if (extras != null) { + extras.each{ k, v -> + output = output.replaceAll("__${k}__", ((v == null)?"":v)) + } + } + replacements.each{ k, v -> + output = output.replaceAll("__${k}__", ((v == null)?"":v)) + } + return output +} + +def mdFileComponents(File mdFile, def dateOnly=false) { + def map = [:] + def content = "" + if (mdFile.exists()) { + def inFrontMatter = false + def firstLine = true + mdFile.eachLine { line -> + if (line.matches("---")) { + def prev = inFrontMatter + inFrontMatter = firstLine + if (inFrontMatter != prev) + return false + } + if (inFrontMatter) { + def m = null + if (m = line =~ /^date:\s*(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2})/) { + map["date"] = new Date().parse("yyyy-MM-dd HH:mm:ss", m[0][1]) + } else if (m = line =~ /^date:\s*(\d{4}-\d{2}-\d{2})/) { + map["date"] = new Date().parse("yyyy-MM-dd", m[0][1]) + } else if (m = line =~ /^channel:\s*(\S+)/) { + map["channel"] = m[0][1] + } else if (m = line =~ /^version:\s*(\S+)/) { + map["version"] = m[0][1] + } else if (m = line =~ /^\s*([^:]+)\s*:\s*(\S.*)/) { + map[ m[0][1] ] = m[0][2] + } + if (dateOnly && map["date"] != null) { + return false + } + } else { + if (dateOnly) + return false + content += line+"\n" + } + firstLine = false + } + } + return dateOnly ? map["date"] : [map, content] +} + +task hugoTemplates { + group "website" + description "Create partially populated md pages for hugo website build" + + def hugoTemplatesDir = file("${jalviewDir}/${hugo_templates_dir}") + def hugoBuildDir = "${jalviewDir}/${hugo_build_dir}" + def templateFiles = fileTree(dir: hugoTemplatesDir) + def releaseMdFile = file("${jalviewDir}/${releases_dir}/release-${JALVIEW_VERSION_UNDERSCORES}.md") + def whatsnewMdFile = file("${jalviewDir}/${whatsnew_dir}/whatsnew-${JALVIEW_VERSION_UNDERSCORES}.md") + def oldJvlFile = file("${jalviewDir}/${hugo_old_jvl}") + def jalviewjsFile = file("${jalviewDir}/${hugo_jalviewjs}") + + doFirst { + // specific release template for version archive + def changes = "" + def whatsnew = null + def givenDate = null + def givenChannel = null + def givenVersion = null + if (CHANNEL == "RELEASE") { + def (map, content) = mdFileComponents(releaseMdFile) + givenDate = map.date + givenChannel = map.channel + givenVersion = map.version + changes = content + if (givenVersion != null && givenVersion != JALVIEW_VERSION) { + throw new GradleException("'version' header (${givenVersion}) found in ${releaseMdFile} does not match JALVIEW_VERSION (${JALVIEW_VERSION})") + } + + if (whatsnewMdFile.exists()) + whatsnew = whatsnewMdFile.text + } + + def oldJvl = oldJvlFile.exists() ? oldJvlFile.collect{it} : [] + def jalviewjsLink = jalviewjsFile.exists() ? jalviewjsFile.collect{it} : [] + + def changesHugo = null + if (changes != null) { + changesHugo = '
\n\n' + def inSection = false + changes.eachLine { line -> + def m = null + if (m = line =~ /^##([^#].*)$/) { + if (inSection) { + changesHugo += "
\n\n" + } + def section = m[0][1].trim() + section = section.toLowerCase() + section = section.replaceAll(/ +/, "_") + section = section.replaceAll(/[^a-z0-9_\-]/, "") + changesHugo += "
\n\n" + inSection = true + } else if (m = line =~ /^(\s*-\s*)(.*?)()?\s*$/) { + def comment = m[0][2].trim() + if (comment != "") { + comment = comment.replaceAll('"', """) + def issuekeys = [] + comment.eachMatch(/JAL-\d+/) { jal -> issuekeys += jal } + def newline = m[0][1] + if (comment.trim() != "") + newline += "{{}}${comment}{{}} " + newline += m[0][3].trim() + if (issuekeys.size() > 0) + newline += " {{< jal issue=\"${issuekeys.join(",")}\" alt=\"${comment}\" >}}" + if (m[0][4] != null) + newline += m[0][4] + line = newline + } + } + changesHugo += line+"\n" + } + if (inSection) { + changesHugo += "\n
\n\n" + } + changesHugo += '' + } + + templateFiles.each{ templateFile -> + def newFileName = string(hugoTemplateSubstitutions(templateFile.getName())) + def relPath = hugoTemplatesDir.toPath().relativize(templateFile.toPath()).getParent() + def newRelPathName = hugoTemplateSubstitutions( relPath.toString() ) + + def outPathName = string("${hugoBuildDir}/$newRelPathName") + + copy { + from templateFile + rename(templateFile.getName(), newFileName) + into outPathName + } + + def newFile = file("${outPathName}/${newFileName}".toString()) + def content = newFile.text + newFile.text = hugoTemplateSubstitutions(content, + [ + WHATSNEW: whatsnew, + CHANGES: changesHugo, + DATE: givenDate == null ? "" : givenDate.format("yyyy-MM-dd"), + DRAFT: givenDate == null ? "true" : "false", + JALVIEWJSLINK: jalviewjsLink.contains(JALVIEW_VERSION) ? "true" : "false", + JVL_HEADER: oldJvl.contains(JALVIEW_VERSION) ? "jvl: true" : "" + ] + ) + } + + } + + inputs.file(oldJvlFile) + inputs.dir(hugoTemplatesDir) + inputs.property("JALVIEW_VERSION", { JALVIEW_VERSION }) + inputs.property("CHANNEL", { CHANNEL }) +} + +def getMdDate(File mdFile) { + return mdFileComponents(mdFile, true) +} + +def getMdSections(String content) { + def sections = [:] + def sectionContent = "" + def sectionName = null + content.eachLine { line -> + def m = null + if (m = line =~ /^##([^#].*)$/) { + if (sectionName != null) { + sections[sectionName] = sectionContent + sectionName = null + sectionContent = "" + } + sectionName = m[0][1].trim() + sectionName = sectionName.toLowerCase() + sectionName = sectionName.replaceAll(/ +/, "_") + sectionName = sectionName.replaceAll(/[^a-z0-9_\-]/, "") + } else if (sectionName != null) { + sectionContent += line+"\n" + } + } + if (sectionContent != null) { + sections[sectionName] = sectionContent + } + return sections +} + + task copyHelp(type: Copy) { def inputDir = helpSourceDir def outputDir = "${helpBuildDir}/${help_dir}" @@ -1234,37 +1486,174 @@ task copyHelp(type: Copy) { } -task copyResources(type: Copy) { - group = "build" - description = "Copy (and make text substitutions in) the resources dir to the build area" +task releasesTemplates { + group "help" + description "Recreate whatsNew.html and releases.html from markdown files and templates in help" - def inputDir = resourceDir - def outputDir = resourcesBuildDir - from(inputDir) { - include('**/*.txt') - include('**/*.md') - include('**/*.html') - include('**/*.xml') - filter(ReplaceTokens, - beginToken: '$$', - endToken: '$$', - tokens: [ - 'Version-Rel': JALVIEW_VERSION, - 'Year-Rel': getDate("yyyy") - ] - ) - } - from(inputDir) { - exclude('**/*.txt') - exclude('**/*.md') - exclude('**/*.html') - exclude('**/*.xml') - } - into outputDir + dependsOn copyHelp - inputs.dir(inputDir) - outputs.dir(outputDir) -} + def releasesTemplateFile = file("${jalviewDir}/${releases_template}") + def whatsnewTemplateFile = file("${jalviewDir}/${whatsnew_template}") + def releasesHtmlFile = file("${helpBuildDir}/${help_dir}/${releases_html}") + def whatsnewHtmlFile = file("${helpBuildDir}/${help_dir}/${whatsnew_html}") + def releasesMdDir = "${jalviewDir}/${releases_dir}" + def whatsnewMdDir = "${jalviewDir}/${whatsnew_dir}" + + doFirst { + def releaseMdFile = file("${releasesMdDir}/release-${JALVIEW_VERSION_UNDERSCORES}.md") + def whatsnewMdFile = file("${whatsnewMdDir}/whatsnew-${JALVIEW_VERSION_UNDERSCORES}.md") + + if (CHANNEL == "RELEASE") { + if (!releaseMdFile.exists()) { + throw new GradleException("File ${releaseMdFile} must be created for RELEASE") + } + if (!whatsnewMdFile.exists()) { + throw new GradleException("File ${whatsnewMdFile} must be created for RELEASE") + } + } + + def releaseFiles = fileTree(dir: releasesMdDir, include: "release-*.md") + def releaseFilesDates = releaseFiles.collectEntries { + [(it): getMdDate(it)] + } + releaseFiles = releaseFiles.sort { a,b -> releaseFilesDates[a].compareTo(releaseFilesDates[b]) } + + def releasesTemplate = releasesTemplateFile.text + def m = releasesTemplate =~ /(?s)__VERSION_LOOP_START__(.*)__VERSION_LOOP_END__/ + def versionTemplate = m[0][1] + + MutableDataSet options = new MutableDataSet() + + def extensions = new ArrayList<>() + options.set(Parser.EXTENSIONS, extensions) + options.set(Parser.HTML_BLOCK_COMMENT_ONLY_FULL_LINE, true) + + Parser parser = Parser.builder(options).build() + HtmlRenderer renderer = HtmlRenderer.builder(options).build() + + def actualVersions = releaseFiles.collect { rf -> + def (rfMap, rfContent) = mdFileComponents(rf) + return rfMap.version + } + def versionsHtml = "" + def linkedVersions = [] + releaseFiles.reverse().each { rFile -> + def (rMap, rContent) = mdFileComponents(rFile) + + def versionLink = "" + def partialVersion = "" + def firstPart = true + rMap.version.split("\\.").each { part -> + def displayPart = ( firstPart ? "" : "." ) + part + partialVersion += displayPart + if ( + linkedVersions.contains(partialVersion) + || ( actualVersions.contains(partialVersion) && partialVersion != rMap.version ) + ) { + versionLink += displayPart + } else { + versionLink += "${displayPart}" + linkedVersions += partialVersion + } + firstPart = false + } + def displayDate = releaseFilesDates[rFile].format("dd/MM/yyyy") + + def lm = null + def rContentProcessed = "" + rContent.eachLine { line -> + if (lm = line =~ /^(\s*-)(\s*)(.*)$/) { + line = "${lm[0][1]}${lm[0][3]}${lm[0][2]}" + } else if (lm = line =~ /^###([^#]+.*)$/) { + line = "_${lm[0][1].trim()}_" + } + rContentProcessed += line + "\n" + } + + def rContentSections = getMdSections(rContentProcessed) + def rVersion = versionTemplate + if (rVersion != "") { + def rNewFeatures = rContentSections["new_features"] + def rIssuesResolved = rContentSections["issues_resolved"] + Node newFeaturesNode = parser.parse(rNewFeatures) + String newFeaturesHtml = renderer.render(newFeaturesNode) + Node issuesResolvedNode = parser.parse(rIssuesResolved) + String issuesResolvedHtml = renderer.render(issuesResolvedNode) + rVersion = hugoTemplateSubstitutions(rVersion, + [ + VERSION: rMap.version, + VERSION_LINK: versionLink, + DISPLAY_DATE: displayDate, + NEW_FEATURES: newFeaturesHtml, + ISSUES_RESOLVED: issuesResolvedHtml + ] + ) + versionsHtml += rVersion + } + } + + releasesTemplate = releasesTemplate.replaceAll("(?s)__VERSION_LOOP_START__.*__VERSION_LOOP_END__", versionsHtml) + releasesTemplate = hugoTemplateSubstitutions(releasesTemplate) + releasesHtmlFile.text = releasesTemplate + + if (whatsnewMdFile.exists()) { + def wnDisplayDate = releaseFilesDates[releaseMdFile] != null ? releaseFilesDates[releaseMdFile].format("dd MMMM yyyy") : "" + def whatsnewMd = hugoTemplateSubstitutions(whatsnewMdFile.text) + Node whatsnewNode = parser.parse(whatsnewMd) + String whatsnewHtml = renderer.render(whatsnewNode) + whatsnewHtml = whatsnewTemplateFile.text.replaceAll("__WHATS_NEW__", whatsnewHtml) + whatsnewHtmlFile.text = hugoTemplateSubstitutions(whatsnewHtml, + [ + VERSION: JALVIEW_VERSION, + DISPLAY_DATE: wnDisplayDate + ] + ) + } else if (gradle.taskGraph.hasTask(":linkCheck")) { + whatsnewHtmlFile.text = "Development build " + getDate("yyyy-MM-dd HH:mm:ss") + } + + } + + inputs.file(releasesTemplateFile) + inputs.file(whatsnewTemplateFile) + inputs.dir(releasesMdDir) + inputs.dir(whatsnewMdDir) + outputs.file(releasesHtmlFile) + outputs.file(whatsnewHtmlFile) +} + + +task copyResources(type: Copy) { + group = "build" + description = "Copy (and make text substitutions in) the resources dir to the build area" + + def inputDir = resourceDir + def outputDir = resourcesBuildDir + from(inputDir) { + include('**/*.txt') + include('**/*.md') + include('**/*.html') + include('**/*.xml') + filter(ReplaceTokens, + beginToken: '$$', + endToken: '$$', + tokens: [ + 'Version-Rel': JALVIEW_VERSION, + 'Year-Rel': getDate("yyyy") + ] + ) + } + from(inputDir) { + exclude('**/*.txt') + exclude('**/*.md') + exclude('**/*.html') + exclude('**/*.xml') + } + into outputDir + + inputs.dir(inputDir) + outputs.dir(outputDir) +} task copyChannelResources(type: Copy) { dependsOn copyResources @@ -1273,7 +1662,19 @@ task copyChannelResources(type: Copy) { def inputDir = "${channelDir}/${resource_dir}" def outputDir = resourcesBuildDir - from inputDir + from(inputDir) { + include(channel_props) + filter(ReplaceTokens, + beginToken: '__', + endToken: '__', + tokens: [ + 'SUFFIX': channelSuffix + ] + ) + } + from(inputDir) { + exclude(channel_props) + } into outputDir inputs.dir(inputDir) @@ -1294,6 +1695,7 @@ task createBuildProperties(type: WriteProperties) { property "BUILD_DATE", getDate("HH:mm:ss dd MMMM yyyy") property "VERSION", JALVIEW_VERSION property "INSTALLATION", INSTALLATION+" git-commit:"+gitHash+" ["+gitBranch+"]" + property "JAVA_COMPILE_VERSION", JAVA_INTEGER_VERSION if (getdownSetAppBaseProperty) { property "GETDOWNAPPBASE", getdownAppBase property "GETDOWNAPPDISTDIR", getdownAppDistDir @@ -1331,6 +1733,7 @@ task prepare { dependsOn buildResources dependsOn copyDocs dependsOn copyHelp + dependsOn releasesTemplates dependsOn convertMdFiles dependsOn buildIndices } @@ -1338,26 +1741,140 @@ task prepare { compileJava.dependsOn prepare run.dependsOn compileJava -//run.dependsOn prepare +compileTestJava.dependsOn compileJava + -//testReportDirName = "test-reports" // note that test workingDir will be $jalviewDir test { - dependsOn prepare + group = "Verification" + description = "Runs all testTaskN tasks)" if (useClover) { dependsOn cloverClasses - } else { //? - dependsOn compileJava //? + } else { //? + dependsOn testClasses + } + + // not running tests in this task + exclude "**/*" +} +/* testTask0 is the main test task */ +task testTask0(type: Test) { + group = "Verification" + description = "The main test task. Runs all non-testTaskN-labelled tests (unless excluded)" + useTestNG() { + includeGroups testng_groups.split(",") + excludeGroups testng_excluded_groups.split(",") + tasks.withType(Test).matching {it.name.startsWith("testTask") && it.name != name}.all {t -> excludeGroups t.name} + preserveOrder true + useDefaultListeners=true + } +} + +/* separated tests */ +task testTask1(type: Test) { + group = "Verification" + description = "Tests that need to be isolated from the main test run" + useTestNG() { + includeGroups name + excludeGroups testng_excluded_groups.split(",") + preserveOrder true + useDefaultListeners=true + } +} + +task testTask2(type: Test) { + group = "Verification" + description = "Tests that need to be isolated from the main test run" + useTestNG() { + includeGroups name + excludeGroups testng_excluded_groups.split(",") + preserveOrder true + useDefaultListeners=true + } +} +task testTask3(type: Test) { + group = "Verification" + description = "Tests that need to be isolated from the main test run" + useTestNG() { + includeGroups name + excludeGroups testng_excluded_groups.split(",") + preserveOrder true + useDefaultListeners=true } +} +/* insert more testTaskNs here -- change N to next digit or other string */ +/* +task testTaskN(type: Test) { + group = "Verification" + description = "Tests that need to be isolated from the main test run" useTestNG() { - includeGroups testng_groups - excludeGroups testng_excluded_groups + includeGroups name + excludeGroups testng_excluded_groups.split(",") preserveOrder true useDefaultListeners=true } +} +*/ + +/* + * adapted from https://medium.com/@wasyl/pretty-tests-summary-in-gradle-744804dd676c + * to summarise test results from all Test tasks + */ +/* START of test tasks results summary */ +import groovy.time.TimeCategory +import org.gradle.api.tasks.testing.logging.TestExceptionFormat +import org.gradle.api.tasks.testing.logging.TestLogEvent +rootProject.ext.testsResults = [] // Container for tests summaries + +tasks.withType(Test).matching {t -> t.getName().startsWith("testTask")}.all { testTask -> + + // from original test task + if (useClover) { + dependsOn cloverClasses + } else { //? + dependsOn testClasses //? + } + + // run main tests first + if (!testTask.name.equals("testTask0")) + testTask.mustRunAfter "testTask0" + + testTask.testLogging { logging -> + events TestLogEvent.FAILED +// TestLogEvent.SKIPPED, +// TestLogEvent.STANDARD_OUT, +// TestLogEvent.STANDARD_ERROR + + exceptionFormat TestExceptionFormat.FULL + showExceptions true + showCauses true + showStackTraces true + if (test_output) { + showStandardStreams true + } + info.events = [ TestLogEvent.FAILED ] + } + + if (OperatingSystem.current().isMacOsX()) { + testTask.systemProperty "apple.awt.UIElement", "true" + testTask.environment "JAVA_TOOL_OPTIONS", "-Dapple.awt.UIElement=true" + } + + + ignoreFailures = true // Always try to run all tests for all modules + + afterSuite { desc, result -> + if (desc.parent) + return // Only summarize results for whole modules + + def resultsInfo = [testTask.project.name, testTask.name, result, TimeCategory.minus(new Date(result.endTime), new Date(result.startTime)), testTask.reports.html.entryPoint] + + rootProject.ext.testsResults.add(resultsInfo) + } + // from original test task maxHeapSize = "1024m" workingDir = jalviewDir @@ -1376,12 +1893,144 @@ test { jvmArgs += additional_compiler_args doFirst { + // this is not perfect yet -- we should only add the commandLineIncludePatterns to the + // testTasks that include the tests, and exclude all from the others. + // get --test argument + filter.commandLineIncludePatterns = test.filter.commandLineIncludePatterns + // do something with testTask.getCandidateClassFiles() to see if the test should silently finish because of the + // commandLineIncludePatterns not matching anything. Instead we are doing setFailOnNoMatchingTests(false) below + + if (useClover) { println("Running tests " + (useClover?"WITH":"WITHOUT") + " clover") } } + + + /* don't fail on no matching tests (so --tests will run across all testTasks) */ + testTask.filter.setFailOnNoMatchingTests(false) + + /* ensure the "test" task dependsOn all the testTasks */ + test.dependsOn testTask +} + +gradle.buildFinished { + def allResults = rootProject.ext.testsResults + + if (!allResults.isEmpty()) { + printResults allResults + allResults.each {r -> + if (r[2].resultType == TestResult.ResultType.FAILURE) + throw new GradleException("Failed tests!") + } + } +} + +private static String colString(styler, col, colour, text) { + return col?"${styler[colour](text)}":text +} + +private static String getSummaryLine(s, pn, tn, rt, rc, rs, rf, rsk, t, col) { + def colour = 'black' + def text = rt + def nocol = false + if (rc == 0) { + text = "-----" + nocol = true + } else { + switch(rt) { + case TestResult.ResultType.SUCCESS: + colour = 'green' + break; + case TestResult.ResultType.FAILURE: + colour = 'red' + break; + default: + nocol = true + break; + } + } + StringBuilder sb = new StringBuilder() + sb.append("${pn}") + if (tn != null) + sb.append(":${tn}") + sb.append(" results: ") + sb.append(colString(s, col && !nocol, colour, text)) + sb.append(" (") + sb.append("${rc} tests, ") + sb.append(colString(s, col && rs > 0, 'green', rs)) + sb.append(" successes, ") + sb.append(colString(s, col && rf > 0, 'red', rf)) + sb.append(" failures, ") + sb.append("${rsk} skipped) in ${t}") + return sb.toString() } +private static void printResults(allResults) { + + // styler from https://stackoverflow.com/a/56139852 + def styler = 'black red green yellow blue magenta cyan white'.split().toList().withIndex(30).collectEntries { key, val -> [(key) : { "\033[${val}m${it}\033[0m" }] } + + def maxLength = 0 + def failedTests = false + def summaryLines = [] + def totalcount = 0 + def totalsuccess = 0 + def totalfail = 0 + def totalskip = 0 + def totaltime = TimeCategory.getSeconds(0) + // sort on project name then task name + allResults.sort {a, b -> a[0] == b[0]? a[1]<=>b[1]:a[0] <=> b[0]}.each { + def projectName = it[0] + def taskName = it[1] + def result = it[2] + def time = it[3] + def report = it[4] + def summaryCol = getSummaryLine(styler, projectName, taskName, result.resultType, result.testCount, result.successfulTestCount, result.failedTestCount, result.skippedTestCount, time, true) + def summaryPlain = getSummaryLine(styler, projectName, taskName, result.resultType, result.testCount, result.successfulTestCount, result.failedTestCount, result.skippedTestCount, time, false) + def reportLine = "Report file: ${report}" + def ls = summaryPlain.length() + def lr = reportLine.length() + def m = [ls, lr].max() + if (m > maxLength) + maxLength = m + def info = [ls, summaryCol, reportLine] + summaryLines.add(info) + failedTests |= result.resultType == TestResult.ResultType.FAILURE + totalcount += result.testCount + totalsuccess += result.successfulTestCount + totalfail += result.failedTestCount + totalskip += result.skippedTestCount + totaltime += time + } + def totalSummaryCol = getSummaryLine(styler, "OVERALL", "", failedTests?TestResult.ResultType.FAILURE:TestResult.ResultType.SUCCESS, totalcount, totalsuccess, totalfail, totalskip, totaltime, true) + def totalSummaryPlain = getSummaryLine(styler, "OVERALL", "", failedTests?TestResult.ResultType.FAILURE:TestResult.ResultType.SUCCESS, totalcount, totalsuccess, totalfail, totalskip, totaltime, false) + def tls = totalSummaryPlain.length() + if (tls > maxLength) + maxLength = tls + def info = [tls, totalSummaryCol, null] + summaryLines.add(info) + + def allSummaries = [] + for(sInfo : summaryLines) { + def ls = sInfo[0] + def summary = sInfo[1] + def report = sInfo[2] + + StringBuilder sb = new StringBuilder() + sb.append("│" + summary + " " * (maxLength - ls) + "│") + if (report != null) { + sb.append("\n│" + report + " " * (maxLength - report.length()) + "│") + } + allSummaries += sb.toString() + } + + println "┌${"${"─" * maxLength}"}┐" + println allSummaries.join("\n├${"${"─" * maxLength}"}┤\n") + println "└${"${"─" * maxLength}"}┘" +} +/* END of test tasks results summary */ + task compileLinkCheck(type: JavaCompile) { options.fork = true @@ -1403,7 +2052,7 @@ task linkCheck(type: JavaExec) { def helpLinksCheckerOutFile = file("${jalviewDir}/${utils_dir}/HelpLinksChecker.out") classpath = files("${jalviewDir}/${utils_dir}") main = "HelpLinksChecker" - workingDir = jalviewDir + workingDir = "${helpBuildDir}" args = [ "${helpBuildDir}/${help_dir}", "-nointernet" ] def outFOS = new FileOutputStream(helpLinksCheckerOutFile, false) // false == don't append @@ -1507,12 +2156,33 @@ shadowJar { if (buildDist) { dependsOn makeDist } - from ("${jalviewDir}/${libDistDir}") { - include("*.jar") - } - manifest { - attributes "Implementation-Version": JALVIEW_VERSION, - "Application-Name": applicationName + + def jarFiles = fileTree(dir: "${jalviewDir}/${libDistDir}", include: "*.jar", exclude: "regex.jar").getFiles() + def groovyJars = jarFiles.findAll {it1 -> file(it1).getName().startsWith("groovy-swing")} + def otherJars = jarFiles.findAll {it2 -> !file(it2).getName().startsWith("groovy-swing")} + from groovyJars + from otherJars + + // we need to include the groovy-swing Include-Package for it to run in the shadowJar + doFirst { + def jarFileManifests = [] + groovyJars.each { jarFile -> + def mf = zipTree(jarFile).getFiles().find { it.getName().equals("MANIFEST.MF") } + if (mf != null) { + jarFileManifests += mf + } + } + + manifest { + attributes "Implementation-Version": JALVIEW_VERSION, "Application-Name": applicationName + from (jarFileManifests) { + eachEntry { details -> + if (!details.key.equals("Import-Package")) { + details.exclude() + } + } + } + } } duplicatesStrategy "INCLUDE" @@ -1523,10 +2193,58 @@ shadowJar { minimize() } +task getdownImagesCopy() { + inputs.dir getdownImagesDir + outputs.dir getdownImagesBuildDir + + doFirst { + copy { + from(getdownImagesDir) { + include("*getdown*.png") + } + into getdownImagesBuildDir + } + } +} + +task getdownImagesProcess() { + dependsOn getdownImagesCopy + + doFirst { + if (backgroundImageText) { + if (convertBinary == null) { + throw new StopExecutionException("No ImageMagick convert binary installed at '${convertBinaryExpectedLocation}'") + } + if (!project.hasProperty("getdown_background_image_text_suffix_cmd")) { + throw new StopExecutionException("No property 'getdown_background_image_text_suffix_cmd' defined. See channel_gradle.properties for channel ${CHANNEL}") + } + fileTree(dir: getdownImagesBuildDir, include: "*background*.png").getFiles().each { file -> + exec { + executable convertBinary + args = [ + file.getPath(), + '-font', getdown_background_image_text_font, + '-fill', getdown_background_image_text_colour, + '-draw', sprintf(getdown_background_image_text_suffix_cmd, channelSuffix), + '-draw', sprintf(getdown_background_image_text_commit_cmd, "git-commit: ${gitHash}"), + '-draw', sprintf(getdown_background_image_text_date_cmd, getDate("yyyy-MM-dd HH:mm:ss")), + file.getPath() + ] + } + } + } + } +} + +task getdownImages() { + dependsOn getdownImagesProcess +} -task getdownWebsite() { +task getdownWebsiteBuild() { group = "distribution" - description = "Create the getdown minimal app folder, and website folder for this version of jalview. Website folder also used for offline app installer" + description = "Create the getdown minimal app folder, and website folder for this version of jalview. Website folder also used for offline app installer. No digest is created." + + dependsOn getdownImages if (buildDist) { dependsOn makeDist } @@ -1549,6 +2267,13 @@ task getdownWebsite() { copy { from channelPropsFile + filter(ReplaceTokens, + beginToken: '__', + endToken: '__', + tokens: [ + 'SUFFIX': channelSuffix + ] + ) into getdownAppBaseDir } getdownWebsiteResourceFilenames += file(channelPropsFile).getName() @@ -1564,11 +2289,11 @@ task getdownWebsite() { if (getdownAltMultiJavaLocation != null && getdownAltMultiJavaLocation.length() > 0) { props.put("getdown_txt_multi_java_location", getdownAltMultiJavaLocation) } - if (getdownImagesDir != null && file(getdownImagesDir).exists()) { - props.put("getdown_txt_ui.background_image", "${getdownImagesDir}/${getdown_background_image}") - props.put("getdown_txt_ui.instant_background_image", "${getdownImagesDir}/${getdown_instant_background_image}") - props.put("getdown_txt_ui.error_background", "${getdownImagesDir}/${getdown_error_background}") - props.put("getdown_txt_ui.progress_image", "${getdownImagesDir}/${getdown_progress_image}") + if (getdownImagesBuildDir != null && file(getdownImagesBuildDir).exists()) { + props.put("getdown_txt_ui.background_image", "${getdownImagesBuildDir}/${getdown_background_image}") + props.put("getdown_txt_ui.instant_background_image", "${getdownImagesBuildDir}/${getdown_instant_background_image}") + props.put("getdown_txt_ui.error_background", "${getdownImagesBuildDir}/${getdown_error_background}") + props.put("getdown_txt_ui.progress_image", "${getdownImagesBuildDir}/${getdown_progress_image}") props.put("getdown_txt_ui.icon", "${getdownImagesDir}/${getdown_icon}") props.put("getdown_txt_ui.mac_dock_icon", "${getdownImagesDir}/${getdown_mac_dock_icon}") } @@ -1629,7 +2354,7 @@ task getdownWebsite() { from s into "${getdownAppBaseDir}/${getdown_wrapper_script_dir}" } - getdownTextLines += "resource = ${getdown_wrapper_script_dir}/${script}" + getdownTextLines += "xresource = ${getdown_wrapper_script_dir}/${script}" } } @@ -1771,7 +2496,9 @@ task getdownDigestDir(type: JavaExec) { task getdownDigest(type: JavaExec) { group = "distribution" description = "Digest the getdown website folder" - dependsOn getdownWebsite + + dependsOn getdownWebsiteBuild + doFirst { classpath = files(getdownLauncher) } @@ -1804,12 +2531,19 @@ task getdown() { } } +task getdownWebsite { + group = "distribution" + description = "A task to create the whole getdown channel website dir including digest file" + + dependsOn getdownWebsiteBuild + dependsOn getdownDigest +} task getdownArchiveBuild() { group = "distribution" description = "Put files in the archive dir to go on the website" - dependsOn getdownWebsite + dependsOn getdownWebsiteBuild def v = "v${JALVIEW_VERSION_UNDERSCORES}" def vDir = "${getdownArchiveDir}/${v}" @@ -2026,13 +2760,60 @@ task cleanInstallersDataFiles { } } -task installerFiles(type: com.install4j.gradle.Install4jTask) { - group = "distribution" - description = "Create the install4j installers" - dependsOn getdown - dependsOn copyInstall4jTemplate - dependsOn cleanInstallersDataFiles - +task install4jDMGBackgroundImageCopy { + inputs.file "${install4jDMGBackgroundImageDir}/${install4jDMGBackgroundImageFile}" + outputs.dir "${install4jDMGBackgroundImageBuildDir}" + doFirst { + copy { + from(install4jDMGBackgroundImageDir) { + include(install4jDMGBackgroundImageFile) + } + into install4jDMGBackgroundImageBuildDir + } + } +} + +task install4jDMGBackgroundImageProcess { + dependsOn install4jDMGBackgroundImageCopy + + doFirst { + if (backgroundImageText) { + if (convertBinary == null) { + throw new StopExecutionException("No ImageMagick convert binary installed at '${convertBinaryExpectedLocation}'") + } + if (!project.hasProperty("install4j_background_image_text_suffix_cmd")) { + throw new StopExecutionException("No property 'install4j_background_image_text_suffix_cmd' defined. See channel_gradle.properties for channel ${CHANNEL}") + } + fileTree(dir: install4jDMGBackgroundImageBuildDir, include: "*.png").getFiles().each { file -> + exec { + executable convertBinary + args = [ + file.getPath(), + '-font', install4j_background_image_text_font, + '-fill', install4j_background_image_text_colour, + '-draw', sprintf(install4j_background_image_text_suffix_cmd, channelSuffix), + '-draw', sprintf(install4j_background_image_text_commit_cmd, "git-commit: ${gitHash}"), + '-draw', sprintf(install4j_background_image_text_date_cmd, getDate("yyyy-MM-dd HH:mm:ss")), + file.getPath() + ] + } + } + } + } +} + +task install4jDMGBackgroundImage { + dependsOn install4jDMGBackgroundImageProcess +} + +task installerFiles(type: com.install4j.gradle.Install4jTask) { + group = "distribution" + description = "Create the install4j installers" + dependsOn getdown + dependsOn copyInstall4jTemplate + dependsOn cleanInstallersDataFiles + dependsOn install4jDMGBackgroundImage + projectFile = install4jConfFile // create an md5 for the input files to use as version for install4j conf file @@ -2063,21 +2844,16 @@ task installerFiles(type: com.install4j.gradle.Install4jTask) { 'JAVA_VERSION': JAVA_VERSION, 'JAVA_INTEGER_VERSION': JAVA_INTEGER_VERSION, 'VERSION': JALVIEW_VERSION, - 'MACOS_JAVA_VM_DIR': macosJavaVMDir, - 'WINDOWS_JAVA_VM_DIR': windowsJavaVMDir, - 'LINUX_JAVA_VM_DIR': linuxJavaVMDir, - 'MACOS_JAVA_VM_TGZ': macosJavaVMTgz, - 'WINDOWS_JAVA_VM_TGZ': windowsJavaVMTgz, - 'LINUX_JAVA_VM_TGZ': linuxJavaVMTgz, 'COPYRIGHT_MESSAGE': install4j_copyright_message, 'BUNDLE_ID': install4jBundleId, 'INTERNAL_ID': install4jInternalId, 'WINDOWS_APPLICATION_ID': install4jWinApplicationId, 'MACOS_DMG_DS_STORE': install4jDMGDSStore, - 'MACOS_DMG_BG_IMAGE': install4jDMGBackgroundImage, + 'MACOS_DMG_BG_IMAGE': "${install4jDMGBackgroundImageBuildDir}/${install4jDMGBackgroundImageFile}", 'WRAPPER_LINK': getdownWrapperLink, 'BASH_WRAPPER_SCRIPT': getdown_bash_wrapper_script, 'POWERSHELL_WRAPPER_SCRIPT': getdown_powershell_wrapper_script, + 'BATCH_WRAPPER_SCRIPT': getdown_batch_wrapper_script, 'WRAPPER_SCRIPT_BIN_DIR': getdown_wrapper_script_dir, 'INSTALLER_NAME': install4jInstallerName, 'INSTALL4J_UTILS_DIR': install4j_utils_dir, @@ -2098,8 +2874,29 @@ task installerFiles(type: com.install4j.gradle.Install4jTask) { 'WINDOWS_ICONS_FILE': install4jWindowsIconsFile, 'PNG_ICON_FILE': install4jPngIconFile, 'BACKGROUND': install4jBackground, + ] + def varNameMap = [ + 'mac': 'MACOS', + 'windows': 'WINDOWS', + 'linux': 'LINUX' + ] + + // these are the bundled OS/architecture VMs needed by install4j + def osArch = [ + [ "mac", "x64" ], + [ "mac", "aarch64" ], + [ "windows", "x64" ], + [ "linux", "x64" ], + [ "linux", "aarch64" ] ] + osArch.forEach { os, arch -> + variables[ sprintf("%s_%s_JAVA_VM_DIR", varNameMap[os], arch.toUpperCase(Locale.ROOT)) ] = sprintf("%s/jre-%s-%s-%s/jre", jreInstallsDir, JAVA_INTEGER_VERSION, os, arch) + // N.B. For some reason install4j requires the below filename to have underscores and not hyphens + // otherwise running `gradle installers` generates a non-useful error: + // `install4j: compilation failed. Reason: java.lang.NumberFormatException: For input string: "windows"` + variables[ sprintf("%s_%s_JAVA_VM_TGZ", varNameMap[os], arch.toUpperCase(Locale.ROOT)) ] = sprintf("%s/tgz/jre_%s_%s_%s.tar.gz", jreInstallsDir, JAVA_INTEGER_VERSION, os, arch) + } //println("INSTALL4J VARIABLES:") //variables.each{k,v->println("${k}=${v}")} @@ -2133,8 +2930,6 @@ task installerFiles(type: com.install4j.gradle.Install4jTask) { inputs.dir(getdownAppBaseDir) inputs.file(install4jConfFile) inputs.file("${install4jDir}/${install4j_info_plist_file_associations}") - inputs.dir(macosJavaVMDir) - inputs.dir(windowsJavaVMDir) outputs.dir("${jalviewDir}/${install4j_build_dir}/${JAVA_VERSION}") } @@ -2258,6 +3053,7 @@ task sourceDist(type: Tar) { into project.name def EXCLUDE_FILES=[ + "dist/*", "build/*", "bin/*", "test-output/", @@ -2274,418 +3070,124 @@ task sourceDist(type: Tar) { "**/*.log", "RELEASE", ] - def PROCESS_FILES=[ - "AUTHORS", - "CITATION", - "FEATURETODO", - "JAVA-11-README", - "FEATURETODO", - "LICENSE", - "**/README", - "THIRDPARTYLIBS", - "TESTNG", - "build.gradle", - "gradle.properties", - "**/*.java", - "**/*.html", - "**/*.xml", - "**/*.gradle", - "**/*.groovy", - "**/*.properties", - "**/*.perl", - "**/*.sh", - ] - def INCLUDE_FILES=[ - ".classpath", - ".settings/org.eclipse.buildship.core.prefs", - ".settings/org.eclipse.jdt.core.prefs" - ] - - from(jalviewDir) { - exclude (EXCLUDE_FILES) - include (PROCESS_FILES) - filter(ReplaceTokens, - beginToken: '$$', - endToken: '$$', - tokens: [ - 'Version-Rel': JALVIEW_VERSION, - 'Year-Rel': getDate("yyyy") - ] - ) - } - from(jalviewDir) { - exclude (EXCLUDE_FILES) - exclude (PROCESS_FILES) - exclude ("appletlib") - exclude ("**/*locales") - exclude ("*locales/**") - exclude ("utils/InstallAnywhere") - - exclude (getdown_files_dir) - // getdown_website_dir and getdown_archive_dir moved to build/website/docroot/getdown - //exclude (getdown_website_dir) - //exclude (getdown_archive_dir) - - // exluding these as not using jars as modules yet - exclude ("${j11modDir}/**/*.jar") - } - from(jalviewDir) { - include(INCLUDE_FILES) - } -// from (jalviewDir) { -// // explicit includes for stuff that seemed to not get included -// include(fileTree("test/**/*.")) -// exclude(EXCLUDE_FILES) -// exclude(PROCESS_FILES) -// } - - from(file(buildProperties).getParent()) { - include(file(buildProperties).getName()) - rename(file(buildProperties).getName(), "build_properties") - filter({ line -> - line.replaceAll("^INSTALLATION=.*\$","INSTALLATION=Source Release"+" git-commit\\\\:"+gitHash+" ["+gitBranch+"]") - }) - } - - def sourceTarBuildDir = "${buildDir}/sourceTar" - from(sourceTarBuildDir) { - // this includes the appended RELEASE properties file - } -} - -task dataInstallersJson { - group "website" - description "Create the installers-VERSION.json data file for installer files created" - - mustRunAfter installers - mustRunAfter shadowJar - mustRunAfter sourceDist - mustRunAfter getdownArchive - - def installersOutputTxt = file("${jalviewDir}/${install4jBuildDir}/output.txt") - def installersSha256 = file("${jalviewDir}/${install4jBuildDir}/sha256sums") - - if (installersOutputTxt.exists()) { - inputs.file(installersOutputTxt) - } - if (install4jCheckSums && installersSha256.exists()) { - inputs.file(installersSha256) - } - [ - shadowJar.archiveFile, // executable JAR - getdownVersionLaunchJvl, // version JVL - sourceDist.archiveFile // source TGZ - ].each { fileName -> - if (file(fileName).exists()) { - inputs.file(fileName) - } - } - - outputs.file(hugoDataJsonFile) - - doFirst { - writeDataJsonFile(installersOutputTxt, installersSha256, hugoDataJsonFile) - } -} - -def hugoTemplateSubstitutions(String input, Map extras=null) { - def replacements = [ - DATE: getDate("yyyy-MM-dd"), - CHANNEL: propertiesChannelName, - APPLICATION_NAME: applicationName, - GIT_HASH: gitHash, - GIT_BRANCH: gitBranch, - VERSION: JALVIEW_VERSION, - JAVA_VERSION: JAVA_VERSION, - VERSION_UNDERSCORES: JALVIEW_VERSION_UNDERSCORES, - DRAFT: "false", - JVL_HEADER: "" - ] - def output = input - if (extras != null) { - extras.each{ k, v -> - output = output.replaceAll("__${k}__", ((v == null)?"":v)) - } - } - replacements.each{ k, v -> - output = output.replaceAll("__${k}__", ((v == null)?"":v)) - } - return output -} - -def mdFileComponents(File mdFile, def dateOnly=false) { - def map = [:] - def content = "" - if (mdFile.exists()) { - def inFrontMatter = false - def firstLine = true - mdFile.eachLine { line -> - if (line.matches("---")) { - def prev = inFrontMatter - inFrontMatter = firstLine - if (inFrontMatter != prev) - return false - } - if (inFrontMatter) { - def m = null - if (m = line =~ /^date:\s*(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2})/) { - map["date"] = new Date().parse("yyyy-MM-dd HH:mm:ss", m[0][1]) - } else if (m = line =~ /^date:\s*(\d{4}-\d{2}-\d{2})/) { - map["date"] = new Date().parse("yyyy-MM-dd", m[0][1]) - } else if (m = line =~ /^channel:\s*(\S+)/) { - map["channel"] = m[0][1] - } else if (m = line =~ /^version:\s*(\S+)/) { - map["version"] = m[0][1] - } else if (m = line =~ /^\s*([^:]+)\s*:\s*(\S.*)/) { - map[ m[0][1] ] = m[0][2] - } - if (dateOnly && map["date"] != null) { - return false - } - } else { - if (dateOnly) - return false - content += line+"\n" - } - firstLine = false - } - } - return dateOnly ? map["date"] : [map, content] -} - -task hugoTemplates { - group "website" - description "Create partially populated md pages for hugo website build" - - def hugoTemplatesDir = file("${jalviewDir}/${hugo_templates_dir}") - def hugoBuildDir = "${jalviewDir}/${hugo_build_dir}" - def templateFiles = fileTree(dir: hugoTemplatesDir) - def releaseMdFile = file("${jalviewDir}/${releases_dir}/release-${JALVIEW_VERSION_UNDERSCORES}.md") - def whatsnewMdFile = file("${jalviewDir}/${whatsnew_dir}/whatsnew-${JALVIEW_VERSION_UNDERSCORES}.md") - def oldJvlFile = file("${jalviewDir}/${hugo_old_jvl}") - def jalviewjsFile = file("${jalviewDir}/${hugo_jalviewjs}") - - doFirst { - // specific release template for version archive - def changes = "" - def whatsnew = null - def givenDate = null - def givenChannel = null - def givenVersion = null - if (CHANNEL == "RELEASE") { - def (map, content) = mdFileComponents(releaseMdFile) - givenDate = map.date - givenChannel = map.channel - givenVersion = map.version - changes = content - if (givenVersion != null && givenVersion != JALVIEW_VERSION) { - throw new GradleException("'version' header (${givenVersion}) found in ${releaseMdFile} does not match JALVIEW_VERSION (${JALVIEW_VERSION})") - } - - if (whatsnewMdFile.exists()) - whatsnew = whatsnewMdFile.text - } - - def oldJvl = oldJvlFile.exists() ? oldJvlFile.collect{it} : [] - def jalviewjsLink = jalviewjsFile.exists() ? jalviewjsFile.collect{it} : [] - - def changesHugo = null - if (changes != null) { - changesHugo = '
\n\n' - def inSection = false - changes.eachLine { line -> - def m = null - if (m = line =~ /^##([^#].*)$/) { - if (inSection) { - changesHugo += "
\n\n" - } - def section = m[0][1].trim() - section = section.toLowerCase() - section = section.replaceAll(/ +/, "_") - section = section.replaceAll(/[^a-z0-9_\-]/, "") - changesHugo += "
\n\n" - inSection = true - } else if (m = line =~ /^(\s*-\s*)(.*?)()?\s*$/) { - def comment = m[0][2].trim() - if (comment != "") { - comment = comment.replaceAll('"', """) - def issuekeys = [] - comment.eachMatch(/JAL-\d+/) { jal -> issuekeys += jal } - def newline = m[0][1] - if (comment.trim() != "") - newline += "{{}}${comment}{{}} " - newline += m[0][3].trim() - if (issuekeys.size() > 0) - newline += " {{< jal issue=\"${issuekeys.join(",")}\" alt=\"${comment}\" >}}" - if (m[0][4] != null) - newline += m[0][4] - line = newline - } - } - changesHugo += line+"\n" - } - if (inSection) { - changesHugo += "\n
\n\n" - } - changesHugo += '' - } - - templateFiles.each{ templateFile -> - def newFileName = string(hugoTemplateSubstitutions(templateFile.getName())) - def relPath = hugoTemplatesDir.toPath().relativize(templateFile.toPath()).getParent() - def newRelPathName = hugoTemplateSubstitutions( relPath.toString() ) - - def outPathName = string("${hugoBuildDir}/$newRelPathName") - - copy { - from templateFile - rename(templateFile.getName(), newFileName) - into outPathName - } - - def newFile = file("${outPathName}/${newFileName}".toString()) - def content = newFile.text - newFile.text = hugoTemplateSubstitutions(content, - [ - WHATSNEW: whatsnew, - CHANGES: changesHugo, - DATE: givenDate == null ? "" : givenDate.format("yyyy-MM-dd"), - DRAFT: givenDate == null ? "true" : "false", - JALVIEWJSLINK: jalviewjsLink.contains(JALVIEW_VERSION) ? "true" : "false", - JVL_HEADER: oldJvl.contains(JALVIEW_VERSION) ? "jvl: true" : "" - ] - ) - } + def PROCESS_FILES=[ + "AUTHORS", + "CITATION", + "FEATURETODO", + "JAVA-11-README", + "FEATURETODO", + "LICENSE", + "**/README", + "THIRDPARTYLIBS", + "TESTNG", + "build.gradle", + "gradle.properties", + "**/*.java", + "**/*.html", + "**/*.xml", + "**/*.gradle", + "**/*.groovy", + "**/*.properties", + "**/*.perl", + "**/*.sh", + ] + def INCLUDE_FILES=[ + ".classpath", + ".settings/org.eclipse.buildship.core.prefs", + ".settings/org.eclipse.jdt.core.prefs" + ] + from(jalviewDir) { + exclude (EXCLUDE_FILES) + include (PROCESS_FILES) + filter(ReplaceTokens, + beginToken: '$$', + endToken: '$$', + tokens: [ + 'Version-Rel': JALVIEW_VERSION, + 'Year-Rel': getDate("yyyy") + ] + ) } + from(jalviewDir) { + exclude (EXCLUDE_FILES) + exclude (PROCESS_FILES) + exclude ("appletlib") + exclude ("**/*locales") + exclude ("*locales/**") + exclude ("utils/InstallAnywhere") - inputs.file(oldJvlFile) - inputs.dir(hugoTemplatesDir) - inputs.property("JALVIEW_VERSION", { JALVIEW_VERSION }) - inputs.property("CHANNEL", { CHANNEL }) -} - -def getMdDate(File mdFile) { - return mdFileComponents(mdFile, true) -} + exclude (getdown_files_dir) + // getdown_website_dir and getdown_archive_dir moved to build/website/docroot/getdown + //exclude (getdown_website_dir) + //exclude (getdown_archive_dir) -def getMdSections(String content) { - def sections = [:] - def sectionContent = "" - def sectionName = null - content.eachLine { line -> - def m = null - if (m = line =~ /^##([^#].*)$/) { - if (sectionName != null) { - sections[sectionName] = sectionContent - sectionName = null - sectionContent = "" - } - sectionName = m[0][1].trim() - sectionName = sectionName.toLowerCase() - sectionName = sectionName.replaceAll(/ +/, "_") - sectionName = sectionName.replaceAll(/[^a-z0-9_\-]/, "") - } else if (sectionName != null) { - sectionContent += line+"\n" - } + // exluding these as not using jars as modules yet + exclude ("${j11modDir}/**/*.jar") } - if (sectionContent != null) { - sections[sectionName] = sectionContent + from(jalviewDir) { + include(INCLUDE_FILES) } - return sections -} - -task releasesTemplates { - def releasesTemplateFile = file("${jalviewDir}/${releases_template}") - def whatsnewTemplateFile = file("${jalviewDir}/${whatsnew_template}") - def releasesHtmlFile = file("${helpSourceDir}/${releases_html}") - def whatsnewHtmlFile = file("${helpSourceDir}/${whatsnew_html}") - def releasesMdDir = "${jalviewDir}/${releases_dir}" - def whatsnewMdDir = "${jalviewDir}/${whatsnew_dir}" +// from (jalviewDir) { +// // explicit includes for stuff that seemed to not get included +// include(fileTree("test/**/*.")) +// exclude(EXCLUDE_FILES) +// exclude(PROCESS_FILES) +// } - doFirst { - def releaseFiles = fileTree(dir: releasesMdDir, include: "release-*.md") - def releaseFilesDates = releaseFiles.collectEntries { - [(it): getMdDate(it)] - } - releaseFiles = releaseFiles.sort { a,b -> releaseFilesDates[a].compareTo(releaseFilesDates[b]) } + from(file(buildProperties).getParent()) { + include(file(buildProperties).getName()) + rename(file(buildProperties).getName(), "build_properties") + filter({ line -> + line.replaceAll("^INSTALLATION=.*\$","INSTALLATION=Source Release"+" git-commit\\\\:"+gitHash+" ["+gitBranch+"]") + }) + } - def releasesTemplate = releasesTemplateFile.text - def m = releasesTemplate =~ /(?s)__VERSION_LOOP_START__(.*)__VERSION_LOOP_END__/ - def versionTemplate = m[0][1] + def sourceTarBuildDir = "${buildDir}/sourceTar" + from(sourceTarBuildDir) { + // this includes the appended RELEASE properties file + } +} - MutableDataSet options = new MutableDataSet() +task dataInstallersJson { + group "website" + description "Create the installers-VERSION.json data file for installer files created" - def extensions = new ArrayList<>() - options.set(Parser.EXTENSIONS, extensions) - options.set(Parser.HTML_BLOCK_COMMENT_ONLY_FULL_LINE, true) + mustRunAfter installers + mustRunAfter shadowJar + mustRunAfter sourceDist + mustRunAfter getdownArchive - Parser parser = Parser.builder(options).build() - HtmlRenderer renderer = HtmlRenderer.builder(options).build() + def installersOutputTxt = file("${jalviewDir}/${install4jBuildDir}/output.txt") + def installersSha256 = file("${jalviewDir}/${install4jBuildDir}/sha256sums") - def actualVersions = releaseFiles.collect { rf -> - def (rfMap, rfContent) = mdFileComponents(rf) - return rfMap.version + if (installersOutputTxt.exists()) { + inputs.file(installersOutputTxt) + } + if (install4jCheckSums && installersSha256.exists()) { + inputs.file(installersSha256) + } + [ + shadowJar.archiveFile, // executable JAR + getdownVersionLaunchJvl, // version JVL + sourceDist.archiveFile // source TGZ + ].each { fileName -> + if (file(fileName).exists()) { + inputs.file(fileName) } - def versionsHtml = "" - def linkedVersions = [] - releaseFiles.reverse().each { rFile -> - def (rMap, rContent) = mdFileComponents(rFile) - - def versionLink = "" - def partialVersion = "" - def firstPart = true - rMap.version.split("\\.").each { part -> - def displayPart = ( firstPart ? "" : "." ) + part - partialVersion += displayPart - if ( - linkedVersions.contains(partialVersion) - || ( actualVersions.contains(partialVersion) && partialVersion != rMap.version ) - ) { - versionLink += displayPart - } else { - versionLink += "${displayPart}" - linkedVersions += partialVersion - } - firstPart = false - } - def displayDate = releaseFilesDates[rFile].format("d MMMMM yyyy") + } - def rContentSections = getMdSections(rContent) - def rVersion = versionTemplate - if (rVersion != "") { - Node newFeaturesNode = parser.parse(rContentSections["new_features"]) - String newFeaturesHtml = renderer.render(newFeaturesNode) - Node issuesResolvedNode = parser.parse(rContentSections["issues_resolved"]) - String issuesResolvedHtml = renderer.render(issuesResolvedNode) - rVersion = hugoTemplateSubstitutions(rVersion, - [ - VERSION: rMap.version, - VERSION_LINK: versionLink, - DISPLAY_DATE: displayDate, - NEW_FEATURES: newFeaturesHtml, - ISSUES_RESOLVED: issuesResolvedHtml - ] - ) - versionsHtml += rVersion - } - } + outputs.file(hugoDataJsonFile) - releasesTemplate = releasesTemplate.replaceAll("(?s)__VERSION_LOOP_START__.*__VERSION_LOOP_END__", versionsHtml) - releasesTemplate = hugoTemplateSubstitutions(releasesTemplate) - releasesHtmlFile.text = releasesTemplate + doFirst { + writeDataJsonFile(installersOutputTxt, installersSha256, hugoDataJsonFile) } - - inputs.file(releasesTemplateFile) - inputs.file(whatsnewTemplateFile) - inputs.dir(releasesMdDir) - inputs.dir(whatsnewMdDir) - outputs.file(releasesHtmlFile) - outputs.file(whatsnewHtmlFile) } - task helppages { + group "help" + description "Copies all help pages to build dir. Runs ant task 'pubhtmlhelp'." + dependsOn copyHelp dependsOn pubhtmlhelp @@ -3766,8 +4268,157 @@ task eclipseAutoBuildTask { } +task jalviewjsCopyStderrLaunchFile(type: Copy) { + from file(jalviewjs_stderr_launch) + into jalviewjsSiteDir + + inputs.file jalviewjs_stderr_launch + outputs.file jalviewjsStderrLaunchFilename +} + +task cleanJalviewjsChromiumUserDir { + doFirst { + delete jalviewjsChromiumUserDir + } + outputs.dir jalviewjsChromiumUserDir + // always run when depended on + outputs.upToDateWhen { !file(jalviewjsChromiumUserDir).exists() } +} + +task jalviewjsChromiumProfile { + dependsOn cleanJalviewjsChromiumUserDir + mustRunAfter cleanJalviewjsChromiumUserDir + + def firstRun = file("${jalviewjsChromiumUserDir}/First Run") + + doFirst { + mkdir jalviewjsChromiumProfileDir + firstRun.text = "" + } + outputs.file firstRun +} + +task jalviewjsLaunchTest { + group "Test" + description "Check JalviewJS opens in a browser" + dependsOn jalviewjsBuildSite + dependsOn jalviewjsCopyStderrLaunchFile + dependsOn jalviewjsChromiumProfile + + def macOS = OperatingSystem.current().isMacOsX() + def chromiumBinary = macOS ? jalviewjs_macos_chromium_binary : jalviewjs_chromium_binary + if (chromiumBinary.startsWith("~/")) { + chromiumBinary = System.getProperty("user.home") + chromiumBinary.substring(1) + } + + def stdout + def stderr + doFirst { + def timeoutms = Integer.valueOf(jalviewjs_chromium_overall_timeout) * 1000 + + def binary = file(chromiumBinary) + if (!binary.exists()) { + throw new StopExecutionException("Could not find chromium binary '${chromiumBinary}'. Cannot run task ${name}.") + } + stdout = new ByteArrayOutputStream() + stderr = new ByteArrayOutputStream() + def execStdout + def execStderr + if (jalviewjs_j2s_to_console.equals("true")) { + execStdout = new org.apache.tools.ant.util.TeeOutputStream( + stdout, + System.out) + execStderr = new org.apache.tools.ant.util.TeeOutputStream( + stderr, + System.err) + } else { + execStdout = stdout + execStderr = stderr + } + def execArgs = [ + "--no-sandbox", // --no-sandbox IS USED BY THE THORIUM APPIMAGE ON THE BUILDSERVER + "--headless=new", + "--disable-gpu", + "--timeout=${timeoutms}", + "--virtual-time-budget=${timeoutms}", + "--user-data-dir=${jalviewDirAbsolutePath}/${jalviewjsBuildDir}/${jalviewjs_chromium_user_dir}", + "--profile-directory=${jalviewjs_chromium_profile_name}", + "--allow-file-access-from-files", + "--enable-logging=stderr", + "file://${jalviewDirAbsolutePath}/${jalviewjsStderrLaunchFilename}" + ] + + if (true || macOS) { + ScheduledExecutorService executor = Executors.newScheduledThreadPool(3); + Future f1 = executor.submit( + () -> { + exec { + standardOutput = execStdout + errorOutput = execStderr + executable(chromiumBinary) + args(execArgs) + println "COMMAND: '"+commandLine.join(" ")+"'" + } + executor.shutdownNow() + } + ) + + def noChangeBytes = 0 + def noChangeIterations = 0 + executor.scheduleAtFixedRate( + () -> { + String stderrString = stderr.toString() + // shutdown the task if we have a success string + if (stderrString.contains(jalviewjs_desktop_init_string)) { + f1.cancel() + Thread.sleep(1000) + executor.shutdownNow() + } + // if no change in stderr for 10s then also end + if (noChangeIterations >= jalviewjs_chromium_idle_timeout) { + executor.shutdownNow() + } + if (stderrString.length() == noChangeBytes) { + noChangeIterations++ + } else { + noChangeBytes = stderrString.length() + noChangeIterations = 0 + } + }, + 1, 1, TimeUnit.SECONDS) + + executor.schedule(new Runnable(){ + public void run(){ + f1.cancel() + executor.shutdownNow() + } + }, timeoutms, TimeUnit.MILLISECONDS) + + executor.awaitTermination(timeoutms+10000, TimeUnit.MILLISECONDS) + executor.shutdownNow() + } + + } + + doLast { + def found = false + stderr.toString().eachLine { line -> + if (line.contains(jalviewjs_desktop_init_string)) { + println("Found line '"+line+"'") + found = true + return + } + } + if (!found) { + throw new GradleException("Could not find evidence of Desktop launch in JalviewJS.") + } + } +} + + task jalviewjs { group "JalviewJS" - description "Build the site" + description "Build the JalviewJS site and run the launch test" dependsOn jalviewjsBuildSite + dependsOn jalviewjsLaunchTest }