X-Git-Url: http://source.jalview.org/gitweb/?a=blobdiff_plain;f=build.gradle;h=bde50b57b367abbc8de6286fd68e0749ad97494f;hb=refs%2Fheads%2Fdevelop;hp=bd0ae207edd798ab27cf287eb36a174e5c5c705f;hpb=a02114f9eb9665810b4c1d89c9806dabcb2b4fd7;p=jalview.git diff --git a/build.gradle b/build.gradle index bd0ae20..d057994 100644 --- a/build.gradle +++ b/build.gradle @@ -9,16 +9,42 @@ import org.gradle.util.ConfigureUtil import org.gradle.plugins.ide.eclipse.model.Output import org.gradle.plugins.ide.eclipse.model.Library import java.security.MessageDigest +import java.util.regex.Matcher +import java.util.concurrent.Executors +import java.util.concurrent.Future +import java.util.concurrent.ScheduledExecutorService +import java.util.concurrent.TimeUnit import groovy.transform.ExternalizeMethods import groovy.util.XmlParser import groovy.xml.XmlUtil - +import groovy.json.JsonBuilder +import com.vladsch.flexmark.util.ast.Node +import com.vladsch.flexmark.html.HtmlRenderer +import com.vladsch.flexmark.parser.Parser +import com.vladsch.flexmark.util.data.MutableDataSet +import com.vladsch.flexmark.ext.gfm.tasklist.TaskListExtension +import com.vladsch.flexmark.ext.tables.TablesExtension +import com.vladsch.flexmark.ext.gfm.strikethrough.StrikethroughExtension +import com.vladsch.flexmark.ext.autolink.AutolinkExtension +import com.vladsch.flexmark.ext.anchorlink.AnchorLinkExtension +import com.vladsch.flexmark.ext.toc.TocExtension +import com.google.common.hash.HashCode +import com.google.common.hash.Hashing +import com.google.common.io.Files +import org.jsoup.Jsoup +import org.jsoup.nodes.Element buildscript { repositories { mavenCentral() mavenLocal() } + dependencies { + classpath "com.vladsch.flexmark:flexmark-all:0.62.0" + classpath "org.jsoup:jsoup:1.14.3" + classpath "com.eowise:gradle-imagemagick:0.5.1" + classpath 'ru.vyarus:gradle-use-python-plugin:4.0.0' + } } @@ -27,9 +53,11 @@ plugins { id 'application' id 'eclipse' id "com.diffplug.gradle.spotless" version "3.28.0" - id 'com.github.johnrengelman.shadow' version '4.0.3' - id 'com.install4j.gradle' version '8.0.4' - id 'com.dorongold.task-tree' version '1.5' // only needed to display task dependency tree with gradle task1 [task2 ...] taskTree + id 'com.github.johnrengelman.shadow' version '6.0.0' + id 'com.install4j.gradle' version '10.0.3' + id 'com.dorongold.task-tree' version '2.1.1' // only needed to display task dependency tree with gradle task1 [task2 ...] taskTree + id 'com.palantir.git-version' version '0.13.0' apply false + id 'ru.vyarus.use-python' version '4.0.0' } repositories { @@ -39,29 +67,19 @@ repositories { } + // in ext the values are cast to Object. Ensure string values are cast as String (and not GStringImpl) for later use def string(Object o) { return o == null ? "" : o.toString() } - -ext { - jalviewDirAbsolutePath = file(jalviewDir).getAbsolutePath() - jalviewDirRelativePath = jalviewDir - - // local build environment properties - // can be "projectDir/local.properties" - def localProps = "${projectDir}/local.properties" - def propsFile = null; - if (file(localProps).exists()) { - propsFile = localProps - } - // or "../projectDir_local.properties" - def dirLocalProps = projectDir.getParent() + "/" + projectDir.getName() + "_local.properties" - if (file(dirLocalProps).exists()) { - propsFile = dirLocalProps +def overrideProperties(String propsFileName, boolean output = false) { + if (propsFileName == null) { + return } - if (propsFile != null) { + def propsFile = file(propsFileName) + if (propsFile != null && propsFile.exists()) { + println("Using properties from file '${propsFileName}'") try { def p = new Properties() def localPropsFIS = new FileInputStream(propsFile) @@ -69,24 +87,55 @@ ext { localPropsFIS.close() p.each { key, val -> - def oldval = findProperty(key) - setProperty(key, val) - if (oldval != null) { - println("Overriding property '${key}' ('${oldval}') with ${file(propsFile).getName()} value '${val}'") + def oldval + if (project.hasProperty(key)) { + oldval = project.findProperty(key) + project.setProperty(key, val) + if (output) { + println("Overriding property '${key}' ('${oldval}') with ${file(propsFile).getName()} value '${val}'") + } } else { - println("Setting unknown property '${key}' with ${file(propsFile).getName()}s value '${val}'") + ext.setProperty(key, val) + if (output) { + println("Setting ext property '${key}' with ${file(propsFile).getName()}s value '${val}'") + } } } } catch (Exception e) { - System.out.println("Exception reading local.properties") + println("Exception reading local.properties") + e.printStackTrace() } } +} + +ext { + jalviewDirAbsolutePath = file(jalviewDir).getAbsolutePath() + jalviewDirRelativePath = jalviewDir + date = new Date() + + getdownChannelName = CHANNEL.toLowerCase() + // default to "default". Currently only has different cosmetics for "develop", "release", "default" + propertiesChannelName = ["develop", "release", "test-release", "jalviewjs", "jalviewjs-release" ].contains(getdownChannelName) ? getdownChannelName : "default" + channelDirName = propertiesChannelName + // Import channel_properties + if (getdownChannelName.startsWith("develop-")) { + channelDirName = "develop-SUFFIX" + } + channelDir = string("${jalviewDir}/${channel_properties_dir}/${channelDirName}") + channelGradleProperties = string("${channelDir}/channel_gradle.properties") + channelPropsFile = string("${channelDir}/${resource_dir}/${channel_props}") + overrideProperties(channelGradleProperties, false) + // local build environment properties + // can be "projectDir/local.properties" + overrideProperties("${projectDir}/local.properties", true) + // or "../projectDir_local.properties" + overrideProperties(projectDir.getParent() + "/" + projectDir.getName() + "_local.properties", true) //// // Import releaseProps from the RELEASE file // or a file specified via JALVIEW_RELEASE_FILE if defined // Expect jalview.version and target release branch in jalview.release - def releaseProps = new Properties(); + releaseProps = new Properties(); def releasePropFile = findProperty("JALVIEW_RELEASE_FILE"); def defaultReleasePropFile = "${jalviewDirAbsolutePath}/RELEASE"; try { @@ -98,9 +147,10 @@ ext { } //// // Set JALVIEW_VERSION if it is not already set - if (findProperty(JALVIEW_VERSION)==null || "".equals(JALVIEW_VERSION)) { + if (findProperty("JALVIEW_VERSION")==null || "".equals(JALVIEW_VERSION)) { JALVIEW_VERSION = releaseProps.get("jalview.version") } + println("JALVIEW_VERSION is set to '${JALVIEW_VERSION}'") // this property set when running Eclipse headlessly j2sHeadlessBuildProperty = string("net.sf.j2s.core.headlessbuild") @@ -134,6 +184,9 @@ ext { } */ + // datestamp + buildDate = new Date().format("yyyyMMdd") + // essentials bareSourceDir = string(source_dir) sourceDir = string("${jalviewDir}/${bareSourceDir}") @@ -148,35 +201,58 @@ ext { cloverBuildDir = "${buildDir}/clover" cloverInstrDir = file("${cloverBuildDir}/clover-instr") cloverClassesDir = file("${cloverBuildDir}/clover-classes") - cloverReportDir = file("${buildDir}/reports") + cloverReportDir = file("${buildDir}/reports/clover") cloverTestInstrDir = file("${cloverBuildDir}/clover-test-instr") cloverTestClassesDir = file("${cloverBuildDir}/clover-test-classes") //cloverTestClassesDir = cloverClassesDir cloverDb = string("${cloverBuildDir}/clover.db") - resourceClassesDir = useClover ? cloverClassesDir : classesDir - testSourceDir = useClover ? cloverTestInstrDir : testDir testClassesDir = useClover ? cloverTestClassesDir : "${jalviewDir}/${test_output_dir}" - getdownWebsiteDir = string("${jalviewDir}/${getdown_website_dir}/${JAVA_VERSION}") + channelSuffix = "" + backgroundImageText = BACKGROUNDIMAGETEXT + getdownChannelDir = string("${getdown_website_dir}/${propertiesChannelName}") + getdownAppBaseDir = string("${jalviewDir}/${getdownChannelDir}/${JAVA_VERSION}") + getdownArchiveDir = string("${jalviewDir}/${getdown_archive_dir}") + getdownFullArchiveDir = null + getdownTextLines = [] + getdownLaunchJvl = null + getdownVersionLaunchJvl = null buildDist = true + buildProperties = null // the following values might be overridden by the CHANNEL switch - getdownChannelName = CHANNEL.toLowerCase() getdownDir = string("${getdownChannelName}/${JAVA_VERSION}") getdownAppBase = string("${getdown_channel_base}/${getdownDir}") + getdownArchiveAppBase = getdown_archive_base getdownLauncher = string("${jalviewDir}/${getdown_lib_dir}/${getdown_launcher}") getdownAppDistDir = getdown_app_dir_alt - buildProperties = string("${resourceDir}/${build_properties_file}") + getdownImagesDir = string("${jalviewDir}/${getdown_images_dir}") + getdownImagesBuildDir = string("${buildDir}/imagemagick/getdown") + getdownSetAppBaseProperty = false // whether to pass the appbase and appdistdir to the application reportRsyncCommand = false jvlChannelName = CHANNEL.toLowerCase() install4jSuffix = CHANNEL.substring(0, 1).toUpperCase() + CHANNEL.substring(1).toLowerCase(); // BUILD -> Build - install4jDSStore = "DS_Store-NON-RELEASE" - install4jDMGBackgroundImage = "jalview_dmg_background-NON-RELEASE.png" - install4jInstallerName = "${jalview_name} Non-Release Installer" - install4jExecutableName = jalview_name.replaceAll("[^\\w]+", "_").toLowerCase() - install4jExtraScheme = "jalviewx" + install4jDMGDSStore = "${install4j_images_dir}/${install4j_dmg_ds_store}" + install4jDMGDSStoreJSON = "${install4j_images_dir}/${install4j_dmg_ds_store_json}" + install4jDMGBackgroundImageDir = "${install4j_images_dir}" + install4jDMGBackgroundImageBuildDir = "build/imagemagick/install4j" + install4jDMGBackgroundImageFile = "${install4j_dmg_background}" + install4jmacOSArchiveName = "${jalview_name} Non-Release ${JALVIEW_VERSION} Installer" + install4jExecutableName = install4j_executable_name + install4jExtraScheme = "jalviewextra" + install4jMacIconsFile = string("${install4j_images_dir}/${install4j_mac_icons_file}") + install4jWindowsIconsFile = string("${install4j_images_dir}/${install4j_windows_icons_file}") + install4jPngIconFile = string("${install4j_images_dir}/${install4j_png_icon_file}") + install4jBackground = string("${install4j_images_dir}/${install4j_background}") + install4jBuildDir = "${install4j_build_dir}/${JAVA_VERSION}" + install4jDMGFixedDSStoreX64 = "build/macos_dmg/${install4j_dmg_ds_store}-x64" + install4jDMGFixedDSStoreAarch64 = "build/macos_dmg/${install4j_dmg_ds_store}-aarch64" + install4jDMGVolumeIcon = string("${install4j_images_dir}/${install4j_dmg_volume_icon}") + install4jCheckSums = true + + applicationName = "${jalview_name}" switch (CHANNEL) { case "BUILD": @@ -190,15 +266,16 @@ ext { testng_excluded_groups = "Not-bamboo" } install4jExtraScheme = "jalviewb" + backgroundImageText = true break - case "RELEASE": + case [ "RELEASE", "JALVIEWJS-RELEASE" ]: getdownAppDistDir = getdown_app_dir_release + getdownSetAppBaseProperty = true reportRsyncCommand = true install4jSuffix = "" - install4jDSStore = "DS_Store" - install4jDMGBackgroundImage = "jalview_dmg_background.png" - install4jInstallerName = "${jalview_name} Installer" + install4jmacOSArchiveName = "Install ${jalview_name} ${JALVIEW_VERSION}" + install4jExtraScheme = (CHANNEL=="RELEASE")?"jalviewx":"jalviewjs" break case "ARCHIVE": @@ -219,9 +296,9 @@ ext { case "ARCHIVELOCAL": getdownChannelName = string("archive/${JALVIEW_VERSION}") getdownDir = string("${getdownChannelName}/${JAVA_VERSION}") - getdownAppBase = file(getdownWebsiteDir).toURI().toString() + getdownAppBase = file(getdownAppBaseDir).toURI().toString() if (!file("${ARCHIVEDIR}/${package_dir}").exists()) { - throw new GradleException("Must provide an ARCHIVEDIR value to produce an archive distribution") + throw new GradleException("Must provide an ARCHIVEDIR value to produce an archive distribution [did not find '${ARCHIVEDIR}/${package_dir}']") } else { package_dir = string("${ARCHIVEDIR}/${package_dir}") buildProperties = string("${ARCHIVEDIR}/${classes_dir}/${build_properties_file}") @@ -233,21 +310,38 @@ ext { install4jExtraScheme = "jalviewa" break + case ~/^DEVELOP-([\.\-\w]*)$/: + def suffix = Matcher.lastMatcher[0][1] + reportRsyncCommand = true + getdownSetAppBaseProperty = true + JALVIEW_VERSION=JALVIEW_VERSION+"-d${suffix}-${buildDate}" + install4jSuffix = "Develop ${suffix}" + install4jExtraScheme = "jalviewd" + install4jmacOSArchiveName = "Install ${jalview_name} Develop ${suffix} ${JALVIEW_VERSION}" + getdownChannelName = string("develop-${suffix}") + getdownChannelDir = string("${getdown_website_dir}/${getdownChannelName}") + getdownAppBaseDir = string("${jalviewDir}/${getdownChannelDir}/${JAVA_VERSION}") + getdownDir = string("${getdownChannelName}/${JAVA_VERSION}") + getdownAppBase = string("${getdown_channel_base}/${getdownDir}") + channelSuffix = string(suffix) + backgroundImageText = true + break + case "DEVELOP": reportRsyncCommand = true - + getdownSetAppBaseProperty = true // DEVELOP-RELEASE is usually associated with a Jalview release series so set the version - JALVIEW_VERSION=JALVIEW_VERSION+"-develop" + JALVIEW_VERSION=JALVIEW_VERSION+"-d${buildDate}" install4jSuffix = "Develop" - install4jDSStore = "DS_Store-DEVELOP" - install4jDMGBackgroundImage = "jalview_dmg_background-DEVELOP.png" install4jExtraScheme = "jalviewd" - install4jInstallerName = "${jalview_name} Develop Installer" + install4jmacOSArchiveName = "Install ${jalview_name} Develop ${JALVIEW_VERSION}" + backgroundImageText = true break case "TEST-RELEASE": reportRsyncCommand = true + getdownSetAppBaseProperty = true // Don't ignore transpile errors for release build if (jalviewjs_ignore_transpile_errors.equals("true")) { jalviewjs_ignore_transpile_errors = "false" @@ -255,10 +349,9 @@ ext { } JALVIEW_VERSION = JALVIEW_VERSION+"-test" install4jSuffix = "Test" - install4jDSStore = "DS_Store-TEST-RELEASE" - install4jDMGBackgroundImage = "jalview_dmg_background-TEST.png" install4jExtraScheme = "jalviewt" - install4jInstallerName = "${jalview_name} Test Installer" + install4jmacOSArchiveName = "Install ${jalview_name} Test ${JALVIEW_VERSION}" + backgroundImageText = true break case ~/^SCRATCH(|-[-\w]*)$/: @@ -280,17 +373,18 @@ ext { } JALVIEW_VERSION = "TEST" install4jSuffix = "Test-Local" - install4jDSStore = "DS_Store-TEST-RELEASE" - install4jDMGBackgroundImage = "jalview_dmg_background-TEST.png" install4jExtraScheme = "jalviewt" - install4jInstallerName = "${jalview_name} Test Installer" + install4jmacOSArchiveName = "Install ${jalview_name} Test ${JALVIEW_VERSION}" + backgroundImageText = true break - case "LOCAL": + case [ "LOCAL", "JALVIEWJS" ]: JALVIEW_VERSION = "TEST" - getdownAppBase = file(getdownWebsiteDir).toURI().toString() + getdownAppBase = file(getdownAppBaseDir).toURI().toString() + getdownArchiveAppBase = file("${jalviewDir}/${getdown_archive_dir}").toURI().toString() getdownLauncher = string("${jalviewDir}/${getdown_lib_dir}/${getdown_launcher_local}") install4jExtraScheme = "jalviewl" + install4jCheckSums = false break default: // something wrong specified @@ -298,20 +392,31 @@ ext { break } + JALVIEW_VERSION_UNDERSCORES = JALVIEW_VERSION.replaceAll("\\.", "_") + hugoDataJsonFile = file("${jalviewDir}/${hugo_build_dir}/${hugo_data_installers_dir}/installers-${JALVIEW_VERSION_UNDERSCORES}.json") + hugoArchiveMdFile = file("${jalviewDir}/${hugo_build_dir}/${hugo_version_archive_dir}/Version-${JALVIEW_VERSION_UNDERSCORES}/_index.md") // override getdownAppBase if requested if (findProperty("getdown_appbase_override") != null) { - getdownAppBase = string(getProperty("getdown_appbase_override")) + // revert to LOCAL if empty string + if (string(getdown_appbase_override) == "") { + getdownAppBase = file(getdownAppBaseDir).toURI().toString() + getdownLauncher = string("${jalviewDir}/${getdown_lib_dir}/${getdown_launcher_local}") + } else if (string(getdown_appbase_override).startsWith("file://")) { + getdownAppBase = string(getdown_appbase_override) + getdownLauncher = string("${jalviewDir}/${getdown_lib_dir}/${getdown_launcher_local}") + } else { + getdownAppBase = string(getdown_appbase_override) + } println("Overriding getdown appbase with '${getdownAppBase}'") } // sanitise file name for jalview launcher file for this channel jvlChannelName = jvlChannelName.replaceAll("[^\\w\\-]+", "_") // install4j application and folder names if (install4jSuffix == "") { - install4jApplicationName = "${jalview_name}" install4jBundleId = "${install4j_bundle_id}" install4jWinApplicationId = install4j_release_win_application_id } else { - install4jApplicationName = "${jalview_name} ${install4jSuffix}" + applicationName = "${jalview_name} ${install4jSuffix}" install4jBundleId = "${install4j_bundle_id}-" + install4jSuffix.toLowerCase() // add int hash of install4jSuffix to the last part of the application_id def id = install4j_release_win_application_id @@ -321,33 +426,49 @@ ext { } // sanitise folder and id names // install4jApplicationFolder = e.g. "Jalview Build" - install4jApplicationFolder = install4jApplicationName + install4jApplicationFolder = applicationName .replaceAll("[\"'~:/\\\\\\s]", "_") // replace all awkward filename chars " ' ~ : / \ .replaceAll("_+", "_") // collapse __ - install4jInternalId = install4jApplicationName + install4jInternalId = applicationName .replaceAll(" ","_") .replaceAll("[^\\w\\-\\.]", "_") // replace other non [alphanumeric,_,-,.] .replaceAll("_+", "") // collapse __ //.replaceAll("_*-_*", "-") // collapse _-_ - install4jUnixApplicationFolder = install4jApplicationName + install4jUnixApplicationFolder = applicationName .replaceAll(" ","_") .replaceAll("[^\\w\\-\\.]", "_") // replace other non [alphanumeric,_,-,.] .replaceAll("_+", "_") // collapse __ .replaceAll("_*-_*", "-") // collapse _-_ .toLowerCase() - - getdownAppDir = string("${getdownWebsiteDir}/${getdownAppDistDir}") - //getdownJ11libDir = "${getdownWebsiteDir}/${getdown_j11lib_dir}" - getdownResourceDir = string("${getdownWebsiteDir}/${getdown_resource_dir}") - getdownInstallDir = string("${getdownWebsiteDir}/${getdown_install_dir}") + install4jmacOSArchiveX64Name = "${install4jmacOSArchiveName} (Intel)" + install4jmacOSArchiveAarch64Name = "${install4jmacOSArchiveName} (Apple Silicon)" + + getdownWrapperLink = install4jUnixApplicationFolder // e.g. "jalview_local" + getdownAppDir = string("${getdownAppBaseDir}/${getdownAppDistDir}") + //getdownJ11libDir = "${getdownAppBaseDir}/${getdown_j11lib_dir}" + getdownResourceDir = string("${getdownAppBaseDir}/${getdown_resource_dir}") + getdownInstallDir = string("${getdownAppBaseDir}/${getdown_install_dir}") getdownFilesDir = string("${jalviewDir}/${getdown_files_dir}/${JAVA_VERSION}/") getdownFilesInstallDir = string("${getdownFilesDir}/${getdown_install_dir}") /* compile without modules -- using classpath libraries modules_compileClasspath = fileTree(dir: "${jalviewDir}/${j11modDir}", include: ["*.jar"]) modules_runtimeClasspath = modules_compileClasspath */ - gitHash = string("") - gitBranch = string("") + + gitHash = "SOURCE" + gitBranch = "Source" + try { + apply plugin: "com.palantir.git-version" + def details = versionDetails() + gitHash = details.gitHash + gitBranch = details.branchName + } catch(org.gradle.api.internal.plugins.PluginApplicationException e) { + println("Not in a git repository. Using git values from RELEASE properties file.") + gitHash = releaseProps.getProperty("git.hash") + gitBranch = releaseProps.getProperty("git.branch") + } catch(java.lang.RuntimeException e1) { + throw new GradleException("Error with git-version plugin. Directory '.git' exists but versionDetails() cannot be found.") + } println("Using a ${CHANNEL} profile.") @@ -383,16 +504,16 @@ ext { '--add-modules', j11modules ] */ - } else if (JAVA_VERSION.equals("12") || JAVA_VERSION.equals("13")) { - JAVA_INTEGER_VERSION = JAVA_VERSION - libDir = j11libDir - libDistDir = j11libDir - compile_source_compatibility = JAVA_VERSION - compile_target_compatibility = JAVA_VERSION + } else if (JAVA_VERSION.equals("17")) { + JAVA_INTEGER_VERSION = string("17") + libDir = j17libDir + libDistDir = j17libDir + compile_source_compatibility = 17 + compile_target_compatibility = 17 getdownAltJavaMinVersion = string(findProperty("getdown_alt_java11_min_version")) getdownAltJavaMaxVersion = string(findProperty("getdown_alt_java11_max_version")) getdownAltMultiJavaLocation = string(findProperty("getdown_alt_java11_txt_multi_java_location")) - eclipseJavaRuntimeName = string("JavaSE-11") + eclipseJavaRuntimeName = string("JavaSE-17") /* compile without modules -- using classpath libraries additional_compiler_args += [ '--module-path', modules_compileClasspath.asPath, @@ -407,16 +528,10 @@ ext { // for install4j JAVA_MIN_VERSION = JAVA_VERSION JAVA_MAX_VERSION = JAVA_VERSION - def jreInstallsDir = string(jre_installs_dir) + jreInstallsDir = string(jre_installs_dir) if (jreInstallsDir.startsWith("~/")) { jreInstallsDir = System.getProperty("user.home") + jreInstallsDir.substring(1) } - macosJavaVMDir = string("${jreInstallsDir}/jre-${JAVA_INTEGER_VERSION}-mac-x64/jre") - macosJavaVMTgz = string("${jreInstallsDir}/tgz/jre-${JAVA_INTEGER_VERSION}-mac-x64.tar.gz") - windowsJavaVMDir = string("${jreInstallsDir}/jre-${JAVA_INTEGER_VERSION}-windows-x64/jre") - windowsJavaVMTgz = string("${jreInstallsDir}/tgz/jre-${JAVA_INTEGER_VERSION}-windows-x64.tar.gz") - linuxJavaVMDir = string("${jreInstallsDir}/jre-${JAVA_INTEGER_VERSION}-linux-x64/jre") - linuxJavaVMTgz = string("${jreInstallsDir}/tgz/jre-${JAVA_INTEGER_VERSION}-linux-x64.tar.gz") install4jDir = string("${jalviewDir}/${install4j_utils_dir}") install4jConfFileName = string("jalview-install4j-conf.install4j") install4jConfFile = file("${install4jDir}/${install4jConfFileName}") @@ -424,14 +539,31 @@ ext { if (install4jHomeDir.startsWith("~/")) { install4jHomeDir = System.getProperty("user.home") + install4jHomeDir.substring(1) } + install4jmacOSArchiveX64DMGFilename = "${install4jApplicationFolder}-${JALVIEW_VERSION}-macos-x64-java_${JAVA_INTEGER_VERSION}" + install4jmacOSArchiveAarch64DMGFilename = "${install4jApplicationFolder}-${JALVIEW_VERSION}-macos-aarch64-java_${JAVA_INTEGER_VERSION}" + resourceBuildDir = string("${buildDir}/resources") + resourcesBuildDir = string("${resourceBuildDir}/resources_build") + helpBuildDir = string("${resourceBuildDir}/help_build") + docBuildDir = string("${resourceBuildDir}/doc_build") + if (buildProperties == null) { + buildProperties = string("${resourcesBuildDir}/${build_properties_file}") + } buildingHTML = string("${jalviewDir}/${doc_dir}/building.html") - helpFile = string("${resourceClassesDir}/${help_dir}/help.jhm") helpParentDir = string("${jalviewDir}/${help_parent_dir}") helpSourceDir = string("${helpParentDir}/${help_dir}") + helpFile = string("${helpBuildDir}/${help_dir}/help.jhm") + convertBinary = null + convertBinaryExpectedLocation = imagemagick_convert + if (convertBinaryExpectedLocation.startsWith("~/")) { + convertBinaryExpectedLocation = System.getProperty("user.home") + convertBinaryExpectedLocation.substring(1) + } + if (file(convertBinaryExpectedLocation).exists()) { + convertBinary = convertBinaryExpectedLocation + } relativeBuildDir = file(jalviewDirAbsolutePath).toPath().relativize(buildDir.toPath()) jalviewjsBuildDir = string("${relativeBuildDir}/jalviewjs") @@ -449,13 +581,19 @@ ext { jalviewjsCoreClasslists = [] jalviewjsJalviewTemplateName = string(jalviewjs_name) jalviewjsJ2sSettingsFileName = string("${jalviewDir}/${jalviewjs_j2s_settings}") + jalviewjsJ2sAltSettingsFileName = string("${jalviewDir}/${jalviewjs_j2s_alt_settings}") jalviewjsJ2sProps = null jalviewjsJ2sPlugin = jalviewjs_j2s_plugin + jalviewjsStderrLaunchFilename = "${jalviewjsSiteDir}/"+(file(jalviewjs_stderr_launch).getName()) eclipseWorkspace = null eclipseBinary = string("") eclipseVersion = string("") eclipseDebug = false + + jalviewjsChromiumUserDir = "${jalviewjsBuildDir}/${jalviewjs_chromium_user_dir}" + jalviewjsChromiumProfileDir = "${ext.jalviewjsChromiumUserDir}/${jalviewjs_chromium_profile_name}" + // ENDEXT } @@ -468,16 +606,14 @@ sourceSets { } resources { - srcDirs resourceDir - srcDirs += helpParentDir + srcDirs = [ resourcesBuildDir, docBuildDir, helpBuildDir ] } - jar.destinationDir = file("${jalviewDir}/${package_dir}") - compileClasspath = files(sourceSets.main.java.outputDir) compileClasspath += fileTree(dir: "${jalviewDir}/${libDir}", include: ["*.jar"]) runtimeClasspath = compileClasspath + runtimeClasspath += files(sourceSets.main.resources.srcDirs) } clover { @@ -514,6 +650,7 @@ sourceSets { compileClasspath += fileTree(dir: "${jalviewDir}/${utils_dir}/testnglibs", include: ["**/*.jar"]) runtimeClasspath = compileClasspath + runtimeClasspath += files(sourceSets.test.resources.srcDirs) } } @@ -534,14 +671,12 @@ eclipse { classpath { //defaultOutputDir = sourceSets.main.java.outputDir - def removeThese = [] - configurations.each{ - if (it.isCanBeResolved()) { - removeThese += it + configurations.each{ c-> + if (c.isCanBeResolved()) { + minusConfigurations += [c] } } - minusConfigurations += removeThese plusConfigurations = [ ] file { @@ -733,6 +868,7 @@ eclipseGroovyCorePreferences.mustRunAfter eclipseJdt task cleanClover { doFirst { delete cloverBuildDir + delete cloverReportDir } } @@ -855,7 +991,7 @@ task cloverHtmlReport(type: JavaExec) { file(cloverDb).exists() } - def cloverHtmlDir = "${cloverReportDir}/clover" + def cloverHtmlDir = cloverReportDir inputs.dir cloverClassesDir outputs.dir cloverHtmlDir @@ -893,7 +1029,7 @@ task cloverXmlReport(type: JavaExec) { file(cloverDb).exists() } - def cloverXmlFile = "${cloverReportDir}/clover/clover.xml" + def cloverXmlFile = "${cloverReportDir}/clover.xml" inputs.dir cloverClassesDir outputs.file cloverXmlFile @@ -949,7 +1085,7 @@ compileJava { // JBP->BS should the print statement in doFirst refer to compile_target_compatibility ? sourceCompatibility = compile_source_compatibility targetCompatibility = compile_target_compatibility - options.compilerArgs = additional_compiler_args + options.compilerArgs += additional_compiler_args options.encoding = "UTF-8" doFirst { print ("Setting target compatibility to "+compile_target_compatibility+"\n") @@ -961,7 +1097,7 @@ compileJava { compileTestJava { sourceCompatibility = compile_source_compatibility targetCompatibility = compile_target_compatibility - options.compilerArgs = additional_compiler_args + options.compilerArgs += additional_compiler_args doFirst { print ("Setting target compatibility to "+targetCompatibility+"\n") } @@ -985,117 +1121,358 @@ cleanTest { // format is a string like date.format("dd MMMM yyyy") def getDate(format) { - def date = new Date() return date.format(format) } -task setGitVals { - def hashStdOut = new ByteArrayOutputStream() - exec { - commandLine "git", "rev-parse", "--short", "HEAD" - standardOutput = hashStdOut - ignoreExitValue true - } - - def branchStdOut = new ByteArrayOutputStream() - exec { - commandLine "git", "rev-parse", "--abbrev-ref", "HEAD" - standardOutput = branchStdOut - ignoreExitValue true - } +def convertMdToHtml (FileTree mdFiles, File cssFile) { + MutableDataSet options = new MutableDataSet() - gitHash = hashStdOut.toString().trim() - gitBranch = branchStdOut.toString().trim() + def extensions = new ArrayList<>() + extensions.add(AnchorLinkExtension.create()) + extensions.add(AutolinkExtension.create()) + extensions.add(StrikethroughExtension.create()) + extensions.add(TaskListExtension.create()) + extensions.add(TablesExtension.create()) + extensions.add(TocExtension.create()) + + options.set(Parser.EXTENSIONS, extensions) + + // set GFM table parsing options + options.set(TablesExtension.WITH_CAPTION, false) + options.set(TablesExtension.COLUMN_SPANS, false) + options.set(TablesExtension.MIN_HEADER_ROWS, 1) + options.set(TablesExtension.MAX_HEADER_ROWS, 1) + options.set(TablesExtension.APPEND_MISSING_COLUMNS, true) + options.set(TablesExtension.DISCARD_EXTRA_COLUMNS, true) + options.set(TablesExtension.HEADER_SEPARATOR_COLUMN_MATCH, true) + // GFM anchor links + options.set(AnchorLinkExtension.ANCHORLINKS_SET_ID, false) + options.set(AnchorLinkExtension.ANCHORLINKS_ANCHOR_CLASS, "anchor") + options.set(AnchorLinkExtension.ANCHORLINKS_SET_NAME, true) + options.set(AnchorLinkExtension.ANCHORLINKS_TEXT_PREFIX, "") + + Parser parser = Parser.builder(options).build() + HtmlRenderer renderer = HtmlRenderer.builder(options).build() + + mdFiles.each { mdFile -> + // add table of contents + def mdText = "[TOC]\n"+mdFile.text + + // grab the first top-level title + def title = null + def titleRegex = /(?m)^#(\s+|([^#]))(.*)/ + def matcher = mdText =~ titleRegex + if (matcher.size() > 0) { + // matcher[0][2] is the first character of the title if there wasn't any whitespace after the # + title = (matcher[0][2] != null ? matcher[0][2] : "")+matcher[0][3] + } + // or use the filename if none found + if (title == null) { + title = mdFile.getName() + } - outputs.upToDateWhen { false } + Node document = parser.parse(mdText) + String htmlBody = renderer.render(document) + def htmlText = ''' + + + + + + +''' + htmlText += ((title != null) ? " ${title}" : '' ) + htmlText += ''' + +''' + htmlText += ((cssFile != null) ? cssFile.text : '') + htmlText += ''' + +''' + htmlText += htmlBody + htmlText += ''' + + +''' + + def htmlFilePath = mdFile.getPath().replaceAll(/\..*?$/, ".html") + def htmlFile = file(htmlFilePath) + println("Creating ${htmlFilePath}") + htmlFile.text = htmlText + } } -task createBuildProperties(type: WriteProperties) { - group = "build" - description = "Create the ${buildProperties} file" - - dependsOn setGitVals - inputs.dir(sourceDir) - inputs.dir(resourceDir) - file(buildProperties).getParentFile().mkdirs() - outputFile (buildProperties) - // taking time specific comment out to allow better incremental builds - comment "--Jalview Build Details--\n"+getDate("yyyy-MM-dd HH:mm:ss") - //comment "--Jalview Build Details--\n"+getDate("yyyy-MM-dd") - property "BUILD_DATE", getDate("HH:mm:ss dd MMMM yyyy") - property "VERSION", JALVIEW_VERSION - property "INSTALLATION", INSTALLATION+" git-commit:"+gitHash+" ["+gitBranch+"]" - outputs.file(outputFile) +task copyDocs(type: Copy) { + def inputDir = "${jalviewDir}/${doc_dir}" + def outputDir = "${docBuildDir}/${doc_dir}" + from(inputDir) { + include('**/*.txt') + include('**/*.md') + include('**/*.html') + include('**/*.xml') + filter(ReplaceTokens, + beginToken: '$$', + endToken: '$$', + tokens: [ + 'Version-Rel': JALVIEW_VERSION, + 'Year-Rel': getDate("yyyy") + ] + ) + } + from(inputDir) { + exclude('**/*.txt') + exclude('**/*.md') + exclude('**/*.html') + exclude('**/*.xml') + } + into outputDir + + inputs.dir(inputDir) + outputs.dir(outputDir) } -clean { - doFirst { - delete buildProperties +task convertMdFiles { + dependsOn copyDocs + def mdFiles = fileTree(dir: docBuildDir, include: "**/*.md") + def cssFile = file("${jalviewDir}/${flexmark_css}") + + doLast { + convertMdToHtml(mdFiles, cssFile) } -} + inputs.files(mdFiles) + inputs.file(cssFile) -task cleanBuildingHTML(type: Delete) { - doFirst { - delete buildingHTML + def htmlFiles = [] + mdFiles.each { mdFile -> + def htmlFilePath = mdFile.getPath().replaceAll(/\..*?$/, ".html") + htmlFiles.add(file(htmlFilePath)) } + outputs.files(htmlFiles) } -task convertBuildingMD(type: Exec) { - dependsOn cleanBuildingHTML - def buildingMD = "${jalviewDir}/${doc_dir}/building.md" - def css = "${jalviewDir}/${doc_dir}/github.css" - - def pandoc = null - pandoc_exec.split(",").each { - if (file(it.trim()).exists()) { - pandoc = it.trim() - return true +def hugoTemplateSubstitutions(String input, Map extras=null) { + def replacements = [ + DATE: getDate("yyyy-MM-dd"), + CHANNEL: propertiesChannelName, + APPLICATION_NAME: applicationName, + GIT_HASH: gitHash, + GIT_BRANCH: gitBranch, + VERSION: JALVIEW_VERSION, + JAVA_VERSION: JAVA_VERSION, + VERSION_UNDERSCORES: JALVIEW_VERSION_UNDERSCORES, + DRAFT: "false", + JVL_HEADER: "" + ] + def output = input + if (extras != null) { + extras.each{ k, v -> + output = output.replaceAll("__${k}__", ((v == null)?"":v)) } } + replacements.each{ k, v -> + output = output.replaceAll("__${k}__", ((v == null)?"":v)) + } + return output +} - def buildtoolsPandoc = System.getProperty("user.home")+"/buildtools/pandoc/bin/pandoc" - if ((pandoc == null || ! file(pandoc).exists()) && file(buildtoolsPandoc).exists()) { - pandoc = System.getProperty("user.home")+"/buildtools/pandoc/bin/pandoc" +def mdFileComponents(File mdFile, def dateOnly=false) { + def map = [:] + def content = "" + if (mdFile.exists()) { + def inFrontMatter = false + def firstLine = true + mdFile.eachLine { line -> + if (line.matches("---")) { + def prev = inFrontMatter + inFrontMatter = firstLine + if (inFrontMatter != prev) + return false + } + if (inFrontMatter) { + def m = null + if (m = line =~ /^date:\s*(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2})/) { + map["date"] = new Date().parse("yyyy-MM-dd HH:mm:ss", m[0][1]) + } else if (m = line =~ /^date:\s*(\d{4}-\d{2}-\d{2})/) { + map["date"] = new Date().parse("yyyy-MM-dd", m[0][1]) + } else if (m = line =~ /^channel:\s*(\S+)/) { + map["channel"] = m[0][1] + } else if (m = line =~ /^version:\s*(\S+)/) { + map["version"] = m[0][1] + } else if (m = line =~ /^\s*([^:]+)\s*:\s*(\S.*)/) { + map[ m[0][1] ] = m[0][2] + } + if (dateOnly && map["date"] != null) { + return false + } + } else { + if (dateOnly) + return false + content += line+"\n" + } + firstLine = false + } } + return dateOnly ? map["date"] : [map, content] +} + +task hugoTemplates { + group "website" + description "Create partially populated md pages for hugo website build" + + def hugoTemplatesDir = file("${jalviewDir}/${hugo_templates_dir}") + def hugoBuildDir = "${jalviewDir}/${hugo_build_dir}" + def templateFiles = fileTree(dir: hugoTemplatesDir) + def releaseMdFile = file("${jalviewDir}/${releases_dir}/release-${JALVIEW_VERSION_UNDERSCORES}.md") + def whatsnewMdFile = file("${jalviewDir}/${whatsnew_dir}/whatsnew-${JALVIEW_VERSION_UNDERSCORES}.md") + def oldJvlFile = file("${jalviewDir}/${hugo_old_jvl}") + def jalviewjsFile = file("${jalviewDir}/${hugo_jalviewjs}") doFirst { - if (pandoc != null && file(pandoc).exists()) { - commandLine pandoc, '-s', '-o', buildingHTML, '--metadata', 'pagetitle="Building Jalview from Source"', '--toc', '-H', css, buildingMD - } else { - println("Cannot find pandoc. Skipping convert building.md to HTML") - throw new StopExecutionException("Cannot find pandoc. Skipping convert building.md to HTML") + // specific release template for version archive + def changes = "" + def whatsnew = null + def givenDate = null + def givenChannel = null + def givenVersion = null + if (CHANNEL == "RELEASE") { + def (map, content) = mdFileComponents(releaseMdFile) + givenDate = map.date + givenChannel = map.channel + givenVersion = map.version + changes = content + if (givenVersion != null && givenVersion != JALVIEW_VERSION) { + throw new GradleException("'version' header (${givenVersion}) found in ${releaseMdFile} does not match JALVIEW_VERSION (${JALVIEW_VERSION})") + } + + if (whatsnewMdFile.exists()) + whatsnew = whatsnewMdFile.text + } + + def oldJvl = oldJvlFile.exists() ? oldJvlFile.collect{it} : [] + def jalviewjsLink = jalviewjsFile.exists() ? jalviewjsFile.collect{it} : [] + + def changesHugo = null + if (changes != null) { + changesHugo = '
\n\n' + def inSection = false + changes.eachLine { line -> + def m = null + if (m = line =~ /^##([^#].*)$/) { + if (inSection) { + changesHugo += "
\n\n" + } + def section = m[0][1].trim() + section = section.toLowerCase() + section = section.replaceAll(/ +/, "_") + section = section.replaceAll(/[^a-z0-9_\-]/, "") + changesHugo += "
\n\n" + inSection = true + } else if (m = line =~ /^(\s*-\s*)(.*?)()?\s*$/) { + def comment = m[0][2].trim() + if (comment != "") { + comment = comment.replaceAll('"', """) + def issuekeys = [] + comment.eachMatch(/JAL-\d+/) { jal -> issuekeys += jal } + def newline = m[0][1] + if (comment.trim() != "") + newline += "{{}}${comment}{{}} " + newline += m[0][3].trim() + if (issuekeys.size() > 0) + newline += " {{< jal issue=\"${issuekeys.join(",")}\" alt=\"${comment}\" >}}" + if (m[0][4] != null) + newline += m[0][4] + line = newline + } + } + changesHugo += line+"\n" + } + if (inSection) { + changesHugo += "\n
\n\n" + } + changesHugo += '' } - } - ignoreExitValue true + templateFiles.each{ templateFile -> + def newFileName = string(hugoTemplateSubstitutions(templateFile.getName())) + def relPath = hugoTemplatesDir.toPath().relativize(templateFile.toPath()).getParent() + def newRelPathName = hugoTemplateSubstitutions( relPath.toString() ) - inputs.file(buildingMD) - inputs.file(css) - outputs.file(buildingHTML) -} + def outPathName = string("${hugoBuildDir}/$newRelPathName") + copy { + from templateFile + rename(templateFile.getName(), newFileName) + into outPathName + } + + def newFile = file("${outPathName}/${newFileName}".toString()) + def content = newFile.text + newFile.text = hugoTemplateSubstitutions(content, + [ + WHATSNEW: whatsnew, + CHANGES: changesHugo, + DATE: givenDate == null ? "" : givenDate.format("yyyy-MM-dd"), + DRAFT: givenDate == null ? "true" : "false", + JALVIEWJSLINK: jalviewjsLink.contains(JALVIEW_VERSION) ? "true" : "false", + JVL_HEADER: oldJvl.contains(JALVIEW_VERSION) ? "jvl: true" : "" + ] + ) + } + + } -task syncDocs(type: Sync) { - dependsOn convertBuildingMD - def syncDir = "${classesDir}/${doc_dir}" - from fileTree("${jalviewDir}/${doc_dir}") - into syncDir + inputs.file(oldJvlFile) + inputs.dir(hugoTemplatesDir) + inputs.property("JALVIEW_VERSION", { JALVIEW_VERSION }) + inputs.property("CHANNEL", { CHANNEL }) +} + +def getMdDate(File mdFile) { + return mdFileComponents(mdFile, true) +} +def getMdSections(String content) { + def sections = [:] + def sectionContent = "" + def sectionName = null + content.eachLine { line -> + def m = null + if (m = line =~ /^##([^#].*)$/) { + if (sectionName != null) { + sections[sectionName] = sectionContent + sectionName = null + sectionContent = "" + } + sectionName = m[0][1].trim() + sectionName = sectionName.toLowerCase() + sectionName = sectionName.replaceAll(/ +/, "_") + sectionName = sectionName.replaceAll(/[^a-z0-9_\-]/, "") + } else if (sectionName != null) { + sectionContent += line+"\n" + } + } + if (sectionContent != null) { + sections[sectionName] = sectionContent + } + return sections } task copyHelp(type: Copy) { def inputDir = helpSourceDir - def outputDir = "${resourceClassesDir}/${help_dir}" + def outputDir = "${helpBuildDir}/${help_dir}" from(inputDir) { - exclude '**/*.gif' - exclude '**/*.jpg' - exclude '**/*.png' + include('**/*.txt') + include('**/*.md') + include('**/*.html') + include('**/*.hs') + include('**/*.xml') + include('**/*.jhm') filter(ReplaceTokens, beginToken: '$$', endToken: '$$', @@ -1106,9 +1483,12 @@ task copyHelp(type: Copy) { ) } from(inputDir) { - include '**/*.gif' - include '**/*.jpg' - include '**/*.png' + exclude('**/*.txt') + exclude('**/*.md') + exclude('**/*.html') + exclude('**/*.hs') + exclude('**/*.xml') + exclude('**/*.jhm') } into outputDir @@ -1118,84 +1498,556 @@ task copyHelp(type: Copy) { } -task syncLib(type: Sync) { - def syncDir = "${resourceClassesDir}/${libDistDir}" - from fileTree("${jalviewDir}/${libDistDir}") - into syncDir +task releasesTemplates { + group "help" + description "Recreate whatsNew.html and releases.html from markdown files and templates in help" + + dependsOn copyHelp + + def releasesTemplateFile = file("${jalviewDir}/${releases_template}") + def whatsnewTemplateFile = file("${jalviewDir}/${whatsnew_template}") + def releasesHtmlFile = file("${helpBuildDir}/${help_dir}/${releases_html}") + def whatsnewHtmlFile = file("${helpBuildDir}/${help_dir}/${whatsnew_html}") + def releasesMdDir = "${jalviewDir}/${releases_dir}" + def whatsnewMdDir = "${jalviewDir}/${whatsnew_dir}" + + doFirst { + def releaseMdFile = file("${releasesMdDir}/release-${JALVIEW_VERSION_UNDERSCORES}.md") + def whatsnewMdFile = file("${whatsnewMdDir}/whatsnew-${JALVIEW_VERSION_UNDERSCORES}.md") + + if (CHANNEL == "RELEASE") { + if (!releaseMdFile.exists()) { + throw new GradleException("File ${releaseMdFile} must be created for RELEASE") + } + if (!whatsnewMdFile.exists()) { + throw new GradleException("File ${whatsnewMdFile} must be created for RELEASE") + } + } + + def releaseFiles = fileTree(dir: releasesMdDir, include: "release-*.md") + def releaseFilesDates = releaseFiles.collectEntries { + [(it): getMdDate(it)] + } + releaseFiles = releaseFiles.sort { a,b -> releaseFilesDates[a].compareTo(releaseFilesDates[b]) } + + def releasesTemplate = releasesTemplateFile.text + def m = releasesTemplate =~ /(?s)__VERSION_LOOP_START__(.*)__VERSION_LOOP_END__/ + def versionTemplate = m[0][1] + + MutableDataSet options = new MutableDataSet() + + def extensions = new ArrayList<>() + options.set(Parser.EXTENSIONS, extensions) + options.set(Parser.HTML_BLOCK_COMMENT_ONLY_FULL_LINE, true) + + Parser parser = Parser.builder(options).build() + HtmlRenderer renderer = HtmlRenderer.builder(options).build() + + def actualVersions = releaseFiles.collect { rf -> + def (rfMap, rfContent) = mdFileComponents(rf) + return rfMap.version + } + def versionsHtml = "" + def linkedVersions = [] + releaseFiles.reverse().each { rFile -> + def (rMap, rContent) = mdFileComponents(rFile) + + def versionLink = "" + def partialVersion = "" + def firstPart = true + rMap.version.split("\\.").each { part -> + def displayPart = ( firstPart ? "" : "." ) + part + partialVersion += displayPart + if ( + linkedVersions.contains(partialVersion) + || ( actualVersions.contains(partialVersion) && partialVersion != rMap.version ) + ) { + versionLink += displayPart + } else { + versionLink += "${displayPart}" + linkedVersions += partialVersion + } + firstPart = false + } + def displayDate = releaseFilesDates[rFile].format("dd/MM/yyyy") + + def lm = null + def rContentProcessed = "" + rContent.eachLine { line -> + if (lm = line =~ /^(\s*-)(\s*)(.*)$/) { + line = "${lm[0][1]}${lm[0][3]}${lm[0][2]}" + } else if (lm = line =~ /^###([^#]+.*)$/) { + line = "_${lm[0][1].trim()}_" + } + rContentProcessed += line + "\n" + } + + def rContentSections = getMdSections(rContentProcessed) + def rVersion = versionTemplate + if (rVersion != "") { + def rNewFeatures = rContentSections["new_features"] + def rIssuesResolved = rContentSections["issues_resolved"] + Node newFeaturesNode = parser.parse(rNewFeatures) + String newFeaturesHtml = renderer.render(newFeaturesNode) + Node issuesResolvedNode = parser.parse(rIssuesResolved) + String issuesResolvedHtml = renderer.render(issuesResolvedNode) + rVersion = hugoTemplateSubstitutions(rVersion, + [ + VERSION: rMap.version, + VERSION_LINK: versionLink, + DISPLAY_DATE: displayDate, + NEW_FEATURES: newFeaturesHtml, + ISSUES_RESOLVED: issuesResolvedHtml + ] + ) + versionsHtml += rVersion + } + } + + releasesTemplate = releasesTemplate.replaceAll("(?s)__VERSION_LOOP_START__.*__VERSION_LOOP_END__", versionsHtml) + releasesTemplate = hugoTemplateSubstitutions(releasesTemplate) + releasesHtmlFile.text = releasesTemplate + + if (whatsnewMdFile.exists()) { + def wnDisplayDate = releaseFilesDates[releaseMdFile] != null ? releaseFilesDates[releaseMdFile].format("dd MMMM yyyy") : "" + def whatsnewMd = hugoTemplateSubstitutions(whatsnewMdFile.text) + Node whatsnewNode = parser.parse(whatsnewMd) + String whatsnewHtml = renderer.render(whatsnewNode) + whatsnewHtml = whatsnewTemplateFile.text.replaceAll("__WHATS_NEW__", whatsnewHtml) + whatsnewHtmlFile.text = hugoTemplateSubstitutions(whatsnewHtml, + [ + VERSION: JALVIEW_VERSION, + DISPLAY_DATE: wnDisplayDate + ] + ) + } else if (gradle.taskGraph.hasTask(":linkCheck")) { + whatsnewHtmlFile.text = "Development build " + getDate("yyyy-MM-dd HH:mm:ss") + } + + } + + inputs.file(releasesTemplateFile) + inputs.file(whatsnewTemplateFile) + inputs.dir(releasesMdDir) + inputs.dir(whatsnewMdDir) + outputs.file(releasesHtmlFile) + outputs.file(whatsnewHtmlFile) } -task syncResources(type: Sync) { - dependsOn createBuildProperties - from resourceDir - include "**/*.*" - into "${resourceClassesDir}" - preserve { - include "**" +task copyResources(type: Copy) { + group = "build" + description = "Copy (and make text substitutions in) the resources dir to the build area" + + def inputDir = resourceDir + def outputDir = resourcesBuildDir + from(inputDir) { + include('**/*.txt') + include('**/*.md') + include('**/*.html') + include('**/*.xml') + filter(ReplaceTokens, + beginToken: '$$', + endToken: '$$', + tokens: [ + 'Version-Rel': JALVIEW_VERSION, + 'Year-Rel': getDate("yyyy") + ] + ) + } + from(inputDir) { + exclude('**/*.txt') + exclude('**/*.md') + exclude('**/*.html') + exclude('**/*.xml') + } + into outputDir + + inputs.dir(inputDir) + outputs.dir(outputDir) +} + +task copyChannelResources(type: Copy) { + dependsOn copyResources + group = "build" + description = "Copy the channel resources dir to the build resources area" + + def inputDir = "${channelDir}/${resource_dir}" + def outputDir = resourcesBuildDir + from(inputDir) { + include(channel_props) + filter(ReplaceTokens, + beginToken: '__', + endToken: '__', + tokens: [ + 'SUFFIX': channelSuffix + ] + ) + } + from(inputDir) { + exclude(channel_props) + } + into outputDir + + inputs.dir(inputDir) + outputs.dir(outputDir) +} + +task createBuildProperties(type: WriteProperties) { + dependsOn copyResources + group = "build" + description = "Create the ${buildProperties} file" + + inputs.dir(sourceDir) + inputs.dir(resourcesBuildDir) + outputFile (buildProperties) + // taking time specific comment out to allow better incremental builds + comment "--Jalview Build Details--\n"+getDate("yyyy-MM-dd HH:mm:ss") + //comment "--Jalview Build Details--\n"+getDate("yyyy-MM-dd") + property "BUILD_DATE", getDate("HH:mm:ss dd MMMM yyyy") + property "VERSION", JALVIEW_VERSION + property "INSTALLATION", INSTALLATION+" git-commit:"+gitHash+" ["+gitBranch+"]" + property "JAVA_COMPILE_VERSION", JAVA_INTEGER_VERSION + if (getdownSetAppBaseProperty) { + property "GETDOWNAPPBASE", getdownAppBase + property "GETDOWNAPPDISTDIR", getdownAppDistDir } + outputs.file(outputFile) +} + + +task buildIndices(type: JavaExec) { + dependsOn copyHelp + classpath = sourceSets.main.compileClasspath + main = "com.sun.java.help.search.Indexer" + workingDir = "${helpBuildDir}/${help_dir}" + def argDir = "html" + args = [ argDir ] + inputs.dir("${workingDir}/${argDir}") + + outputs.dir("${classesDir}/doc") + outputs.dir("${classesDir}/help") + outputs.file("${workingDir}/JavaHelpSearch/DOCS") + outputs.file("${workingDir}/JavaHelpSearch/DOCS.TAB") + outputs.file("${workingDir}/JavaHelpSearch/OFFSETS") + outputs.file("${workingDir}/JavaHelpSearch/POSITIONS") + outputs.file("${workingDir}/JavaHelpSearch/SCHEMA") + outputs.file("${workingDir}/JavaHelpSearch/TMAP") } +task buildResources { + dependsOn copyResources + dependsOn copyChannelResources + dependsOn createBuildProperties +} task prepare { - dependsOn syncResources - dependsOn syncDocs + dependsOn buildResources + dependsOn copyDocs dependsOn copyHelp + dependsOn releasesTemplates + dependsOn convertMdFiles + dependsOn buildIndices } -//testReportDirName = "test-reports" // note that test workingDir will be $jalviewDir +compileJava.dependsOn prepare +run.dependsOn compileJava +compileTestJava.dependsOn compileJava + + + test { - dependsOn prepare - //dependsOn compileJava ////? DELETE + group = "Verification" + description = "Runs all testTaskN tasks)" if (useClover) { dependsOn cloverClasses - } else { //? - dependsOn compileJava //? + } else { //? + dependsOn testClasses + } + + // not running tests in this task + exclude "**/*" +} +/* testTask0 is the main test task */ +task testTask0(type: Test) { + group = "Verification" + description = "The main test task. Runs all non-testTaskN-labelled tests (unless excluded)" + useTestNG() { + includeGroups testng_groups.split(",") + excludeGroups testng_excluded_groups.split(",") + tasks.withType(Test).matching {it.name.startsWith("testTask") && it.name != name}.all {t -> excludeGroups t.name} + preserveOrder true + useDefaultListeners=true + } + timeout = Duration.ofMinutes(15) +} + +/* separated tests */ +task testTask1(type: Test) { + group = "Verification" + description = "Tests that need to be isolated from the main test run" + useTestNG() { + includeGroups name + excludeGroups testng_excluded_groups.split(",") + preserveOrder true + useDefaultListeners=true + } + timeout = Duration.ofMinutes(5) +} + +task testTask2(type: Test) { + group = "Verification" + description = "Tests that need to be isolated from the main test run" + useTestNG() { + includeGroups name + excludeGroups testng_excluded_groups.split(",") + preserveOrder true + useDefaultListeners=true + } + timeout = Duration.ofMinutes(5) +} +task testTask3(type: Test) { + group = "Verification" + description = "Tests that need to be isolated from the main test run" + useTestNG() { + includeGroups name + excludeGroups testng_excluded_groups.split(",") + preserveOrder true + useDefaultListeners=true } + timeout = Duration.ofMinutes(5) +} +/* insert more testTaskNs here -- change N to next digit or other string */ +/* +task testTaskN(type: Test) { + group = "Verification" + description = "Tests that need to be isolated from the main test run" useTestNG() { - includeGroups testng_groups - excludeGroups testng_excluded_groups + includeGroups name + excludeGroups testng_excluded_groups.split(",") preserveOrder true useDefaultListeners=true } +} +*/ + + +/* + * adapted from https://medium.com/@wasyl/pretty-tests-summary-in-gradle-744804dd676c + * to summarise test results from all Test tasks + */ +/* START of test tasks results summary */ +import groovy.time.TimeCategory +import org.gradle.api.tasks.testing.logging.TestExceptionFormat +import org.gradle.api.tasks.testing.logging.TestLogEvent +rootProject.ext.testsResults = [] // Container for tests summaries + +tasks.withType(Test).matching {t -> t.getName().startsWith("testTask")}.all { testTask -> + + // from original test task + if (useClover) { + dependsOn cloverClasses + } else { //? + dependsOn testClasses //? + } + + // run main tests first + if (!testTask.name.equals("testTask0")) + testTask.mustRunAfter "testTask0" + + testTask.testLogging { logging -> + events TestLogEvent.FAILED +// TestLogEvent.SKIPPED, +// TestLogEvent.STANDARD_OUT, +// TestLogEvent.STANDARD_ERROR + + exceptionFormat TestExceptionFormat.FULL + showExceptions true + showCauses true + showStackTraces true + if (test_output) { + showStandardStreams true + } + info.events = [ TestLogEvent.FAILED ] + } + + if (OperatingSystem.current().isMacOsX()) { + testTask.systemProperty "apple.awt.UIElement", "true" + testTask.environment "JAVA_TOOL_OPTIONS", "-Dapple.awt.UIElement=true" + } + + ignoreFailures = true // Always try to run all tests for all modules + + afterSuite { desc, result -> + if (desc.parent) + return // Only summarize results for whole modules + + def resultsInfo = [testTask.project.name, testTask.name, result, TimeCategory.minus(new Date(result.endTime), new Date(result.startTime)), testTask.reports.html.entryPoint] + + rootProject.ext.testsResults.add(resultsInfo) + } + + // from original test task maxHeapSize = "1024m" workingDir = jalviewDir - //systemProperties 'clover.jar' System.properties.clover.jar + def testLaf = project.findProperty("test_laf") + if (testLaf != null) { + println("Setting Test LaF to '${testLaf}'") + systemProperty "laf", testLaf + } + def testHiDPIScale = project.findProperty("test_HiDPIScale") + if (testHiDPIScale != null) { + println("Setting Test HiDPI Scale to '${testHiDPIScale}'") + systemProperty "sun.java2d.uiScale", testHiDPIScale + } sourceCompatibility = compile_source_compatibility targetCompatibility = compile_target_compatibility jvmArgs += additional_compiler_args doFirst { + // this is not perfect yet -- we should only add the commandLineIncludePatterns to the + // testTasks that include the tests, and exclude all from the others. + // get --test argument + filter.commandLineIncludePatterns = test.filter.commandLineIncludePatterns + // do something with testTask.getCandidateClassFiles() to see if the test should silently finish because of the + // commandLineIncludePatterns not matching anything. Instead we are doing setFailOnNoMatchingTests(false) below + + if (useClover) { println("Running tests " + (useClover?"WITH":"WITHOUT") + " clover") } } + + + /* don't fail on no matching tests (so --tests will run across all testTasks) */ + testTask.filter.setFailOnNoMatchingTests(false) + + /* ensure the "test" task dependsOn all the testTasks */ + test.dependsOn testTask } +gradle.buildFinished { + def allResults = rootProject.ext.testsResults -task buildIndices(type: JavaExec) { - dependsOn copyHelp - classpath = sourceSets.main.compileClasspath - main = "com.sun.java.help.search.Indexer" - workingDir = "${classesDir}/${help_dir}" - def argDir = "html" - args = [ argDir ] - inputs.dir("${workingDir}/${argDir}") + if (!allResults.isEmpty()) { + printResults allResults + allResults.each {r -> + if (r[2].resultType == TestResult.ResultType.FAILURE) + throw new GradleException("Failed tests!") + } + } +} - outputs.dir("${classesDir}/doc") - outputs.dir("${classesDir}/help") - outputs.file("${workingDir}/JavaHelpSearch/DOCS") - outputs.file("${workingDir}/JavaHelpSearch/DOCS.TAB") - outputs.file("${workingDir}/JavaHelpSearch/OFFSETS") - outputs.file("${workingDir}/JavaHelpSearch/POSITIONS") - outputs.file("${workingDir}/JavaHelpSearch/SCHEMA") - outputs.file("${workingDir}/JavaHelpSearch/TMAP") +private static String colString(styler, col, colour, text) { + return col?"${styler[colour](text)}":text +} + +private static String getSummaryLine(s, pn, tn, rt, rc, rs, rf, rsk, t, col) { + def colour = 'black' + def text = rt + def nocol = false + if (rc == 0) { + text = "-----" + nocol = true + } else { + switch(rt) { + case TestResult.ResultType.SUCCESS: + colour = 'green' + break; + case TestResult.ResultType.FAILURE: + colour = 'red' + break; + default: + nocol = true + break; + } + } + StringBuilder sb = new StringBuilder() + sb.append("${pn}") + if (tn != null) + sb.append(":${tn}") + sb.append(" results: ") + sb.append(colString(s, col && !nocol, colour, text)) + sb.append(" (") + sb.append("${rc} tests, ") + sb.append(colString(s, col && rs > 0, 'green', rs)) + sb.append(" successes, ") + sb.append(colString(s, col && rf > 0, 'red', rf)) + sb.append(" failures, ") + sb.append("${rsk} skipped) in ${t}") + return sb.toString() } +private static void printResults(allResults) { + + // styler from https://stackoverflow.com/a/56139852 + def styler = 'black red green yellow blue magenta cyan white'.split().toList().withIndex(30).collectEntries { key, val -> [(key) : { "\033[${val}m${it}\033[0m" }] } + + def maxLength = 0 + def failedTests = false + def summaryLines = [] + def totalcount = 0 + def totalsuccess = 0 + def totalfail = 0 + def totalskip = 0 + def totaltime = TimeCategory.getSeconds(0) + // sort on project name then task name + allResults.sort {a, b -> a[0] == b[0]? a[1]<=>b[1]:a[0] <=> b[0]}.each { + def projectName = it[0] + def taskName = it[1] + def result = it[2] + def time = it[3] + def report = it[4] + def summaryCol = getSummaryLine(styler, projectName, taskName, result.resultType, result.testCount, result.successfulTestCount, result.failedTestCount, result.skippedTestCount, time, true) + def summaryPlain = getSummaryLine(styler, projectName, taskName, result.resultType, result.testCount, result.successfulTestCount, result.failedTestCount, result.skippedTestCount, time, false) + def reportLine = "Report file: ${report}" + def ls = summaryPlain.length() + def lr = reportLine.length() + def m = [ls, lr].max() + if (m > maxLength) + maxLength = m + def info = [ls, summaryCol, reportLine] + summaryLines.add(info) + failedTests |= result.resultType == TestResult.ResultType.FAILURE + totalcount += result.testCount + totalsuccess += result.successfulTestCount + totalfail += result.failedTestCount + totalskip += result.skippedTestCount + totaltime += time + } + def totalSummaryCol = getSummaryLine(styler, "OVERALL", "", failedTests?TestResult.ResultType.FAILURE:TestResult.ResultType.SUCCESS, totalcount, totalsuccess, totalfail, totalskip, totaltime, true) + def totalSummaryPlain = getSummaryLine(styler, "OVERALL", "", failedTests?TestResult.ResultType.FAILURE:TestResult.ResultType.SUCCESS, totalcount, totalsuccess, totalfail, totalskip, totaltime, false) + def tls = totalSummaryPlain.length() + if (tls > maxLength) + maxLength = tls + def info = [tls, totalSummaryCol, null] + summaryLines.add(info) + + def allSummaries = [] + for(sInfo : summaryLines) { + def ls = sInfo[0] + def summary = sInfo[1] + def report = sInfo[2] + + StringBuilder sb = new StringBuilder() + sb.append("│" + summary + " " * (maxLength - ls) + "│") + if (report != null) { + sb.append("\n│" + report + " " * (maxLength - report.length()) + "│") + } + allSummaries += sb.toString() + } + + println "┌${"${"─" * maxLength}"}┐" + println allSummaries.join("\n├${"${"─" * maxLength}"}┤\n") + println "└${"${"─" * maxLength}"}┘" +} +/* END of test tasks results summary */ + task compileLinkCheck(type: JavaCompile) { options.fork = true @@ -1211,30 +2063,31 @@ task compileLinkCheck(type: JavaCompile) { task linkCheck(type: JavaExec) { - dependsOn prepare, compileLinkCheck + dependsOn prepare + dependsOn compileLinkCheck def helpLinksCheckerOutFile = file("${jalviewDir}/${utils_dir}/HelpLinksChecker.out") classpath = files("${jalviewDir}/${utils_dir}") main = "HelpLinksChecker" - workingDir = jalviewDir - args = [ "${classesDir}/${help_dir}", "-nointernet" ] + workingDir = "${helpBuildDir}" + args = [ "${helpBuildDir}/${help_dir}", "-nointernet" ] def outFOS = new FileOutputStream(helpLinksCheckerOutFile, false) // false == don't append - def errFOS = outFOS standardOutput = new org.apache.tools.ant.util.TeeOutputStream( outFOS, - standardOutput) + System.out) errorOutput = new org.apache.tools.ant.util.TeeOutputStream( outFOS, - errorOutput) + System.err) - inputs.dir("${classesDir}/${help_dir}") + inputs.dir(helpBuildDir) outputs.file(helpLinksCheckerOutFile) } + // import the pubhtmlhelp target ant.properties.basedir = "${jalviewDir}" -ant.properties.helpBuildDir = "${jalviewDirAbsolutePath}/${classes_dir}/${help_dir}" +ant.properties.helpBuildDir = "${helpBuildDir}/${help_dir}" ant.importBuild "${utils_dir}/publishHelp.xml" @@ -1246,19 +2099,22 @@ task cleanPackageDir(type: Delete) { jar { + dependsOn prepare dependsOn linkCheck - dependsOn buildIndices - dependsOn createBuildProperties manifest { attributes "Main-Class": main_class, "Permissions": "all-permissions", - "Application-Name": "Jalview Desktop", - "Codebase": application_codebase + "Application-Name": applicationName, + "Codebase": application_codebase, + "Implementation-Version": JALVIEW_VERSION } - destinationDir = file("${jalviewDir}/${package_dir}") - archiveName = rootProject.name+".jar" + def outputDir = "${jalviewDir}/${package_dir}" + destinationDirectory = file(outputDir) + archiveFileName = rootProject.name+".jar" + duplicatesStrategy "EXCLUDE" + exclude "cache*/**" exclude "*.jar" @@ -1266,8 +2122,11 @@ jar { exclude "**/*.jar" exclude "**/*.jar.*" - inputs.dir(classesDir) - outputs.file("${jalviewDir}/${package_dir}/${archiveName}") + inputs.dir(sourceSets.main.java.outputDir) + sourceSets.main.resources.srcDirs.each{ dir -> + inputs.dir(dir) + } + outputs.file("${outputDir}/${archiveFileName}") } @@ -1279,10 +2138,11 @@ task copyJars(type: Copy) { // doing a Sync instead of Copy as Copy doesn't deal with "outputs" very well task syncJars(type: Sync) { + dependsOn jar from fileTree(dir: "${jalviewDir}/${libDistDir}", include: "**/*.jar").files into "${jalviewDir}/${package_dir}" preserve { - include jar.archiveName + include jar.archiveFileName.getOrNull() } } @@ -1306,49 +2166,149 @@ task cleanDist { dependsOn clean } + +task launcherJar(type: Jar) { + manifest { + attributes ( + "Main-Class": shadow_jar_main_class, + "Implementation-Version": JALVIEW_VERSION, + "Application-Name": applicationName + ) + } +} + shadowJar { group = "distribution" + description = "Create a single jar file with all dependency libraries merged. Can be run with java -jar" if (buildDist) { dependsOn makeDist } - from ("${jalviewDir}/${libDistDir}") { - include("*.jar") - } + + def jarFiles = fileTree(dir: "${jalviewDir}/${libDistDir}", include: "*.jar", exclude: "regex.jar").getFiles() + def groovyJars = jarFiles.findAll {it1 -> file(it1).getName().startsWith("groovy-swing")} + def otherJars = jarFiles.findAll {it2 -> !file(it2).getName().startsWith("groovy-swing")} + from groovyJars + from otherJars + manifest { - attributes 'Implementation-Version': JALVIEW_VERSION + // shadowJar manifest must inheritFrom another Jar task. Can't set attributes here. + inheritFrom(project.tasks.launcherJar.manifest) + } + // we need to include the groovy-swing Include-Package for it to run in the shadowJar + doFirst { + def jarFileManifests = [] + groovyJars.each { jarFile -> + def mf = zipTree(jarFile).getFiles().find { it.getName().equals("MANIFEST.MF") } + if (mf != null) { + jarFileManifests += mf + } + } + manifest { + from (jarFileManifests) { + eachEntry { details -> + if (!details.key.equals("Import-Package")) { + details.exclude() + } + } + } + } } + + duplicatesStrategy "INCLUDE" + + // this mainClassName is mandatory but gets ignored due to manifest created in doFirst{}. Set the Main-Class as an attribute in launcherJar instead mainClassName = shadow_jar_main_class mergeServiceFiles() classifier = "all-"+JALVIEW_VERSION+"-j"+JAVA_VERSION minimize() } +task getdownImagesCopy() { + inputs.dir getdownImagesDir + outputs.dir getdownImagesBuildDir -task getdownWebsite() { + doFirst { + copy { + from(getdownImagesDir) { + include("*getdown*.png") + } + into getdownImagesBuildDir + } + } +} + +task getdownImagesProcess() { + dependsOn getdownImagesCopy + + doFirst { + if (backgroundImageText) { + if (convertBinary == null) { + throw new StopExecutionException("No ImageMagick convert binary installed at '${convertBinaryExpectedLocation}'") + } + if (!project.hasProperty("getdown_background_image_text_suffix_cmd")) { + throw new StopExecutionException("No property 'getdown_background_image_text_suffix_cmd' defined. See channel_gradle.properties for channel ${CHANNEL}") + } + fileTree(dir: getdownImagesBuildDir, include: "*background*.png").getFiles().each { file -> + exec { + executable convertBinary + args = [ + file.getPath(), + '-font', getdown_background_image_text_font, + '-fill', getdown_background_image_text_colour, + '-draw', sprintf(getdown_background_image_text_suffix_cmd, channelSuffix), + '-draw', sprintf(getdown_background_image_text_commit_cmd, "git-commit: ${gitHash}"), + '-draw', sprintf(getdown_background_image_text_date_cmd, getDate("yyyy-MM-dd HH:mm:ss")), + file.getPath() + ] + } + } + } + } +} + +task getdownImages() { + dependsOn getdownImagesProcess +} + +task getdownWebsiteBuild() { group = "distribution" - description = "Create the getdown minimal app folder, and website folder for this version of jalview. Website folder also used for offline app installer" + description = "Create the getdown minimal app folder, and website folder for this version of jalview. Website folder also used for offline app installer. No digest is created." + + dependsOn getdownImages if (buildDist) { dependsOn makeDist } def getdownWebsiteResourceFilenames = [] - def getdownTextString = "" def getdownResourceDir = getdownResourceDir def getdownResourceFilenames = [] doFirst { // clean the getdown website and files dir before creating getdown folders - delete getdownWebsiteDir + delete getdownAppBaseDir delete getdownFilesDir copy { from buildProperties - rename(build_properties_file, getdown_build_properties) + rename(file(buildProperties).getName(), getdown_build_properties) into getdownAppDir } getdownWebsiteResourceFilenames += "${getdownAppDistDir}/${getdown_build_properties}" - // set some getdown_txt_ properties then go through all properties looking for getdown_txt_... + copy { + from channelPropsFile + filter(ReplaceTokens, + beginToken: '__', + endToken: '__', + tokens: [ + 'SUFFIX': channelSuffix + ] + ) + into getdownAppBaseDir + } + getdownWebsiteResourceFilenames += file(channelPropsFile).getName() + + // set some getdownTxt_ properties then go through all properties looking for getdownTxt_... def props = project.properties.sort { it.key } if (getdownAltJavaMinVersion != null && getdownAltJavaMinVersion.length() > 0) { props.put("getdown_txt_java_min_version", getdownAltJavaMinVersion) @@ -1359,19 +2319,27 @@ task getdownWebsite() { if (getdownAltMultiJavaLocation != null && getdownAltMultiJavaLocation.length() > 0) { props.put("getdown_txt_multi_java_location", getdownAltMultiJavaLocation) } + if (getdownImagesBuildDir != null && file(getdownImagesBuildDir).exists()) { + props.put("getdown_txt_ui.background_image", "${getdownImagesBuildDir}/${getdown_background_image}") + props.put("getdown_txt_ui.instant_background_image", "${getdownImagesBuildDir}/${getdown_instant_background_image}") + props.put("getdown_txt_ui.error_background", "${getdownImagesBuildDir}/${getdown_error_background}") + props.put("getdown_txt_ui.progress_image", "${getdownImagesBuildDir}/${getdown_progress_image}") + props.put("getdown_txt_ui.icon", "${getdownImagesDir}/${getdown_icon}") + props.put("getdown_txt_ui.mac_dock_icon", "${getdownImagesDir}/${getdown_mac_dock_icon}") + } props.put("getdown_txt_title", jalview_name) - props.put("getdown_txt_ui.name", install4jApplicationName) + props.put("getdown_txt_ui.name", applicationName) // start with appbase - getdownTextString += "appbase = ${getdownAppBase}\n" + getdownTextLines += "appbase = ${getdownAppBase}" props.each{ prop, val -> if (prop.startsWith("getdown_txt_") && val != null) { if (prop.startsWith("getdown_txt_multi_")) { def key = prop.substring(18) val.split(",").each{ v -> - def line = "${key} = ${v}\n" - getdownTextString += line + def line = "${key} = ${v}" + getdownTextLines += line } } else { // file values rationalised @@ -1391,15 +2359,15 @@ task getdownWebsite() { } } if (! prop.startsWith("getdown_txt_resource")) { - def line = prop.substring(12) + " = ${val}\n" - getdownTextString += line + def line = prop.substring(12) + " = ${val}" + getdownTextLines += line } } } } getdownWebsiteResourceFilenames.each{ filename -> - getdownTextString += "resource = ${filename}\n" + getdownTextLines += "resource = ${filename}" } getdownResourceFilenames.each{ filename -> copy { @@ -1407,6 +2375,18 @@ task getdownWebsite() { into getdownResourceDir } } + + def getdownWrapperScripts = [ getdown_bash_wrapper_script, getdown_powershell_wrapper_script, getdown_batch_wrapper_script ] + getdownWrapperScripts.each{ script -> + def s = file( "${jalviewDir}/utils/getdown/${getdown_wrapper_script_dir}/${script}" ) + if (s.exists()) { + copy { + from s + into "${getdownAppBaseDir}/${getdown_wrapper_script_dir}" + } + getdownTextLines += "xresource = ${getdown_wrapper_script_dir}/${script}" + } + } def codeFiles = [] fileTree(file(package_dir)).each{ f -> @@ -1417,10 +2397,12 @@ task getdownWebsite() { codeFiles += f } } - codeFiles.sort().each{f -> + def jalviewJar = jar.archiveFileName.getOrNull() + // put jalview.jar first for CLASSPATH and .properties files reasons + codeFiles.sort{a, b -> ( a.getName() == jalviewJar ? -1 : ( b.getName() == jalviewJar ? 1 : a <=> b ) ) }.each{f -> def name = f.getName() - def line = "code = ${getdownAppDistDir}/${name}\n" - getdownTextString += line + def line = "code = ${getdownAppDistDir}/${name}" + getdownTextLines += line copy { from f.getPath() into getdownAppDir @@ -1433,8 +2415,8 @@ task getdownWebsite() { def j11libFiles = fileTree(dir: "${jalviewDir}/${j11libDir}", include: ["*.jar"]).getFiles() j11libFiles.sort().each{f -> def name = f.getName() - def line = "code = ${getdown_j11lib_dir}/${name}\n" - getdownTextString += line + def line = "code = ${getdown_j11lib_dir}/${name}" + getdownTextLines += line copy { from f.getPath() into getdownJ11libDir @@ -1444,59 +2426,71 @@ task getdownWebsite() { */ // getdown-launcher.jar should not be in main application class path so the main application can move it when updated. Listed as a resource so it gets updated. - //getdownTextString += "class = " + file(getdownLauncher).getName() + "\n" - getdownTextString += "resource = ${getdown_launcher_new}\n" - getdownTextString += "class = ${main_class}\n" + //getdownTextLines += "class = " + file(getdownLauncher).getName() + getdownTextLines += "resource = ${getdown_launcher_new}" + getdownTextLines += "class = ${main_class}" + // Not setting these properties in general so that getdownappbase and getdowndistdir will default to release version in jalview.bin.Cache + if (getdownSetAppBaseProperty) { + getdownTextLines += "jvmarg = -Dgetdowndistdir=${getdownAppDistDir}" + getdownTextLines += "jvmarg = -Dgetdownappbase=${getdownAppBase}" + } - def getdown_txt = file("${getdownWebsiteDir}/getdown.txt") - getdown_txt.write(getdownTextString) + def getdownTxt = file("${getdownAppBaseDir}/getdown.txt") + getdownTxt.write(getdownTextLines.join("\n")) - def getdownLaunchJvl = getdown_launch_jvl_name + ( (jvlChannelName != null && jvlChannelName.length() > 0)?"-${jvlChannelName}":"" ) + ".jvl" - def launchJvl = file("${getdownWebsiteDir}/${getdownLaunchJvl}") + getdownLaunchJvl = getdown_launch_jvl_name + ( (jvlChannelName != null && jvlChannelName.length() > 0)?"-${jvlChannelName}":"" ) + ".jvl" + def launchJvl = file("${getdownAppBaseDir}/${getdownLaunchJvl}") launchJvl.write("appbase=${getdownAppBase}") + // files going into the getdown website dir: getdown-launcher.jar copy { from getdownLauncher rename(file(getdownLauncher).getName(), getdown_launcher_new) - into getdownWebsiteDir + into getdownAppBaseDir } + // files going into the getdown website dir: getdown-launcher(-local).jar copy { from getdownLauncher if (file(getdownLauncher).getName() != getdown_launcher) { rename(file(getdownLauncher).getName(), getdown_launcher) } - into getdownWebsiteDir + into getdownAppBaseDir } + // files going into the getdown website dir: ./install dir and files if (! (CHANNEL.startsWith("ARCHIVE") || CHANNEL.startsWith("DEVELOP"))) { copy { - from getdown_txt + from getdownTxt from getdownLauncher - from "${getdownWebsiteDir}/${getdown_build_properties}" + from "${getdownAppDir}/${getdown_build_properties}" if (file(getdownLauncher).getName() != getdown_launcher) { rename(file(getdownLauncher).getName(), getdown_launcher) } into getdownInstallDir } + // and make a copy in the getdown files dir (these are not downloaded by getdown) copy { from getdownInstallDir into getdownFilesInstallDir } } + // files going into the getdown files dir: getdown.txt, getdown-launcher.jar, channel-launch.jvl, build_properties copy { - from getdown_txt + from getdownTxt from launchJvl from getdownLauncher - from "${getdownWebsiteDir}/${getdown_build_properties}" + from "${getdownAppBaseDir}/${getdown_build_properties}" + from "${getdownAppBaseDir}/${channel_props}" if (file(getdownLauncher).getName() != getdown_launcher) { rename(file(getdownLauncher).getName(), getdown_launcher) } into getdownFilesDir } + // and ./resource (not all downloaded by getdown) copy { from getdownResourceDir into "${getdownFilesDir}/${getdown_resource_dir}" @@ -1506,7 +2500,7 @@ task getdownWebsite() { if (buildDist) { inputs.dir("${jalviewDir}/${package_dir}") } - outputs.dir(getdownWebsiteDir) + outputs.dir(getdownAppBaseDir) outputs.dir(getdownFilesDir) } @@ -1532,14 +2526,16 @@ task getdownDigestDir(type: JavaExec) { task getdownDigest(type: JavaExec) { group = "distribution" description = "Digest the getdown website folder" - dependsOn getdownWebsite + + dependsOn getdownWebsiteBuild + doFirst { classpath = files(getdownLauncher) } main = "com.threerings.getdown.tools.Digester" - args getdownWebsiteDir - inputs.dir(getdownWebsiteDir) - outputs.file("${getdownWebsiteDir}/digest2.txt") + args getdownAppBaseDir + inputs.dir(getdownAppBaseDir) + outputs.file("${getdownAppBaseDir}/digest2.txt") } @@ -1549,7 +2545,7 @@ task getdown() { dependsOn getdownDigest doLast { if (reportRsyncCommand) { - def fromDir = getdownWebsiteDir + (getdownWebsiteDir.endsWith('/')?'':'/') + def fromDir = getdownAppBaseDir + (getdownAppBaseDir.endsWith('/')?'':'/') def toDir = "${getdown_rsync_dest}/${getdownDir}" + (getdownDir.endsWith('/')?'':'/') println "LIKELY RSYNC COMMAND:" println "mkdir -p '$toDir'\nrsync -avh --delete '$fromDir' '$toDir'" @@ -1565,6 +2561,121 @@ task getdown() { } } +task getdownWebsite { + group = "distribution" + description = "A task to create the whole getdown channel website dir including digest file" + + dependsOn getdownWebsiteBuild + dependsOn getdownDigest +} + +task getdownArchiveBuild() { + group = "distribution" + description = "Put files in the archive dir to go on the website" + + dependsOn getdownWebsiteBuild + + def v = "v${JALVIEW_VERSION_UNDERSCORES}" + def vDir = "${getdownArchiveDir}/${v}" + getdownFullArchiveDir = "${vDir}/getdown" + getdownVersionLaunchJvl = "${vDir}/jalview-${v}.jvl" + + def vAltDir = "alt_${v}" + def archiveImagesDir = "${jalviewDir}/${channel_properties_dir}/old/images" + + doFirst { + // cleanup old "old" dir + delete getdownArchiveDir + + def getdownArchiveTxt = file("${getdownFullArchiveDir}/getdown.txt") + getdownArchiveTxt.getParentFile().mkdirs() + def getdownArchiveTextLines = [] + def getdownFullArchiveAppBase = "${getdownArchiveAppBase}${getdownArchiveAppBase.endsWith("/")?"":"/"}${v}/getdown/" + + // the libdir + copy { + from "${getdownAppBaseDir}/${getdownAppDistDir}" + into "${getdownFullArchiveDir}/${vAltDir}" + } + + getdownTextLines.each { line -> + line = line.replaceAll("^(?appbase\\s*=\\s*).*", '${s}'+getdownFullArchiveAppBase) + line = line.replaceAll("^(?(resource|code)\\s*=\\s*)${getdownAppDistDir}/", '${s}'+vAltDir+"/") + line = line.replaceAll("^(?ui.background_image\\s*=\\s*).*\\.png", '${s}'+"${getdown_resource_dir}/jalview_archive_getdown_background.png") + line = line.replaceAll("^(?ui.instant_background_image\\s*=\\s*).*\\.png", '${s}'+"${getdown_resource_dir}/jalview_archive_getdown_background_initialising.png") + line = line.replaceAll("^(?ui.error_background\\s*=\\s*).*\\.png", '${s}'+"${getdown_resource_dir}/jalview_archive_getdown_background_error.png") + line = line.replaceAll("^(?ui.progress_image\\s*=\\s*).*\\.png", '${s}'+"${getdown_resource_dir}/jalview_archive_getdown_progress_bar.png") + // remove the existing resource = resource/ or bin/ lines + if (! line.matches("resource\\s*=\\s*(resource|bin)/.*")) { + getdownArchiveTextLines += line + } + } + + // the resource dir -- add these files as resource lines in getdown.txt + copy { + from "${archiveImagesDir}" + into "${getdownFullArchiveDir}/${getdown_resource_dir}" + eachFile { file -> + getdownArchiveTextLines += "resource = ${getdown_resource_dir}/${file.getName()}" + } + } + + // the wrapper scripts dir + if ( file("${getdownAppBaseDir}/${getdown_wrapper_script_dir}").exists() ) { + copy { + from "${getdownAppBaseDir}/${getdown_wrapper_script_dir}" + into "${getdownFullArchiveDir}/${getdown_wrapper_script_dir}" + } + } + + getdownArchiveTxt.write(getdownArchiveTextLines.join("\n")) + + def vLaunchJvl = file(getdownVersionLaunchJvl) + vLaunchJvl.getParentFile().mkdirs() + vLaunchJvl.write("appbase=${getdownFullArchiveAppBase}\n") + def vLaunchJvlPath = vLaunchJvl.toPath().toAbsolutePath() + def jvlLinkPath = file("${vDir}/jalview.jvl").toPath().toAbsolutePath() + // for some reason filepath.relativize(fileInSameDirPath) gives a path to "../" which is wrong + //java.nio.file.Files.createSymbolicLink(jvlLinkPath, jvlLinkPath.relativize(vLaunchJvlPath)); + java.nio.file.Files.createSymbolicLink(jvlLinkPath, java.nio.file.Paths.get(".",vLaunchJvl.getName())); + + // files going into the getdown files dir: getdown.txt, getdown-launcher.jar, channel-launch.jvl, build_properties + copy { + from getdownLauncher + from "${getdownAppBaseDir}/${getdownLaunchJvl}" + from "${getdownAppBaseDir}/${getdown_launcher_new}" + from "${getdownAppBaseDir}/${channel_props}" + if (file(getdownLauncher).getName() != getdown_launcher) { + rename(file(getdownLauncher).getName(), getdown_launcher) + } + into getdownFullArchiveDir + } + + } +} + +task getdownArchiveDigest(type: JavaExec) { + group = "distribution" + description = "Digest the getdown archive folder" + + dependsOn getdownArchiveBuild + + doFirst { + classpath = files(getdownLauncher) + args getdownFullArchiveDir + } + main = "com.threerings.getdown.tools.Digester" + inputs.dir(getdownFullArchiveDir) + outputs.file("${getdownFullArchiveDir}/digest2.txt") +} + +task getdownArchive() { + group = "distribution" + description = "Build the website archive dir with getdown digest" + + dependsOn getdownArchiveBuild + dependsOn getdownArchiveDigest +} tasks.withType(JavaCompile) { options.encoding = 'UTF-8' @@ -1573,8 +2684,9 @@ tasks.withType(JavaCompile) { clean { doFirst { - delete getdownWebsiteDir + delete getdownAppBaseDir delete getdownFilesDir + delete getdownArchiveDir } } @@ -1614,14 +2726,31 @@ task copyInstall4jTemplate { } } - // turn off checksum creation for LOCAL channel - def e = install4jConfigXml.application[0] - if (CHANNEL == "LOCAL") { - e.'@createChecksums' = "false" + // delete .VolumeIcon.icns in macos DMG if there isn't one + if (!file(install4jDMGVolumeIcon).exists()) { + println("No '.VolumeIcon.icns' file found. Removing from install4j file.") + install4jConfigXml.'**'.macosArchive.topLevelFiles.each { topLevelFiles -> + topLevelFiles.file.each() { file -> + if (file.attribute("name") && file.attribute("name").equals(".VolumeIcon.icns")) { + println("Removing "+file.toString()) + topLevelFiles.remove(file) + } + } + } } else { - e.'@createChecksums' = "true" + println("Using '.VolumeIcon.icns' file '${install4jDMGVolumeIcon}'") + } + + // disable install screen for OSX dmg (for 2.11.2.0) + install4jConfigXml.'**'.macosArchive.each { macosArchive -> + macosArchive.attributes().remove('executeSetupApp') + macosArchive.attributes().remove('setupAppId') } + // turn off checksum creation for LOCAL channel + def e = install4jConfigXml.application[0] + e.'@createChecksums' = string(install4jCheckSums) + // put file association actions where placeholder action is def install4jFileAssociationsText = install4jFileAssociationsFile.text def fileAssociationActions = new XmlParser().parseText("${install4jFileAssociationsText}") @@ -1661,12 +2790,6 @@ task copyInstall4jTemplate { } } - // remove the "Uninstall Old Jalview (optional)" symlink from DMG for non-release DS_Stores - if (! (CHANNEL == "RELEASE" || CHANNEL == "TEST-RELEASE" ) ) { - def symlink = install4jConfigXml.'**'.topLevelFiles.symlink.find { sl -> sl.'@name' == "Uninstall Old Jalview (optional).app" } - symlink.parent().remove(symlink) - } - // write install4j file install4jConfFile.text = XmlUtil.serialize(install4jConfigXml) } @@ -1679,16 +2802,134 @@ clean { } } +task cleanInstallersDataFiles { + def installersOutputTxt = file("${jalviewDir}/${install4jBuildDir}/output.txt") + def installersSha256 = file("${jalviewDir}/${install4jBuildDir}/sha256sums") + def hugoDataJsonFile = file("${jalviewDir}/${install4jBuildDir}/installers-${JALVIEW_VERSION_UNDERSCORES}.json") + doFirst { + delete installersOutputTxt + delete installersSha256 + delete hugoDataJsonFile + } +} + +task install4jDMGBackgroundImageCopy { + inputs.file "${install4jDMGBackgroundImageDir}/${install4jDMGBackgroundImageFile}" + outputs.dir "${install4jDMGBackgroundImageBuildDir}" + doFirst { + copy { + from(install4jDMGBackgroundImageDir) { + include(install4jDMGBackgroundImageFile) + } + into install4jDMGBackgroundImageBuildDir + } + } +} + +task install4jDMGBackgroundImageProcess { + dependsOn install4jDMGBackgroundImageCopy + + doFirst { + if (backgroundImageText) { + if (convertBinary == null) { + throw new StopExecutionException("No ImageMagick convert binary installed at '${convertBinaryExpectedLocation}'") + } + if (!project.hasProperty("install4j_background_image_text_suffix_cmd")) { + throw new StopExecutionException("No property 'install4j_background_image_text_suffix_cmd' defined. See channel_gradle.properties for channel ${CHANNEL}") + } + fileTree(dir: install4jDMGBackgroundImageBuildDir, include: "*.png").getFiles().each { file -> + exec { + executable convertBinary + args = [ + file.getPath(), + '-font', install4j_background_image_text_font, + '-fill', install4j_background_image_text_colour, + '-draw', sprintf(install4j_background_image_text_suffix_cmd, channelSuffix), + '-draw', sprintf(install4j_background_image_text_commit_cmd, "git-commit: ${gitHash}"), + '-draw', sprintf(install4j_background_image_text_date_cmd, getDate("yyyy-MM-dd HH:mm:ss")), + file.getPath() + ] + } + } + } + } +} + + +python { + pip 'ds_store:1.3.0' +} + +task install4jCustomiseDS_StoreX64(type: PythonTask) { + inputs.file(install4jDMGDSStore) + outputs.file(install4jDMGFixedDSStoreX64) + def command_args = [ jalview_customise_ds_store, + '--input', install4jDMGDSStore, + '--output', install4jDMGFixedDSStoreX64, + '--volumename', install4jmacOSArchiveX64Name, + '--backgroundfile', install4j_dmg_background_filename, + '--dmg', install4jmacOSArchiveX64DMGFilename + ".dmg", + '--appname', "${applicationName}.app", + ] + if (file(install4jDMGDSStoreJSON).exists()) { + command_args += [ '--config', install4jDMGDSStoreJSON ] + inputs.file(install4jDMGDSStoreJSON) + } + command = command_args + doFirst { + println("Running command '${command_args.join(' ')}'") + } +} + +task install4jCustomiseDS_StoreAarch64(type: PythonTask) { + inputs.file(install4jDMGDSStore) + outputs.file(install4jDMGFixedDSStoreAarch64) + def command_args = [ jalview_customise_ds_store, + '--input', install4jDMGDSStore, + '--output', install4jDMGFixedDSStoreAarch64, + '--volumename', install4jmacOSArchiveAarch64Name, + '--backgroundfile', install4j_dmg_background_filename, + '--dmg', install4jmacOSArchiveAarch64DMGFilename + ".dmg", + '--appname', "${applicationName}.app", + ] + if (file(install4jDMGDSStoreJSON).exists()) { + command_args += [ '--config', install4jDMGDSStoreJSON ] + inputs.file(install4jDMGDSStoreJSON) + } + command = command_args + doFirst { + def print_args = [] + for (int i = 0; i < command_args.size(); i++) { + def arg = command_args[i] + print_args += (i > 0 && !arg.startsWith("-")) ? "\"${arg}\"" : arg + } + println("Running command '${print_args.join(' ')}'") + } +} + +task install4jCustomiseDS_Store { + dependsOn install4jCustomiseDS_StoreX64 + dependsOn install4jCustomiseDS_StoreAarch64 +} + +task install4jDMGProcesses { + dependsOn install4jDMGBackgroundImageProcess + dependsOn install4jCustomiseDS_Store +} -task installers(type: com.install4j.gradle.Install4jTask) { +task installerFiles(type: com.install4j.gradle.Install4jTask) { group = "distribution" description = "Create the install4j installers" - dependsOn setGitVals dependsOn getdown dependsOn copyInstall4jTemplate + dependsOn cleanInstallersDataFiles + dependsOn install4jDMGProcesses projectFile = install4jConfFile + // run install4j with 4g + vmParameters = ["-Xmx4294967296"] + // create an md5 for the input files to use as version for install4j conf file def digest = MessageDigest.getInstance("MD5") digest.update( @@ -1700,14 +2941,14 @@ task installers(type: com.install4j.gradle.Install4jTask) { filesMd5 = filesMd5.substring(0,8) } def install4jTemplateVersion = "${JALVIEW_VERSION}_F${filesMd5}_C${gitHash}" - // make install4jBuildDir relative to jalviewDir - def install4jBuildDir = "${install4j_build_dir}/${JAVA_VERSION}" variables = [ 'JALVIEW_NAME': jalview_name, - 'JALVIEW_APPLICATION_NAME': install4jApplicationName, + 'JALVIEW_APPLICATION_NAME': applicationName, 'JALVIEW_DIR': "../..", 'OSX_KEYSTORE': OSX_KEYSTORE, + 'OSX_APPLEID': OSX_APPLEID, + 'OSX_ALTOOLPASS': OSX_ALTOOLPASS, 'JSIGN_SH': JSIGN_SH, 'JRE_DIR': getdown_app_dir_java, 'INSTALLER_TEMPLATE_VERSION': install4jTemplateVersion, @@ -1717,21 +2958,23 @@ task installers(type: com.install4j.gradle.Install4jTask) { 'JAVA_VERSION': JAVA_VERSION, 'JAVA_INTEGER_VERSION': JAVA_INTEGER_VERSION, 'VERSION': JALVIEW_VERSION, - 'MACOS_JAVA_VM_DIR': macosJavaVMDir, - 'WINDOWS_JAVA_VM_DIR': windowsJavaVMDir, - 'LINUX_JAVA_VM_DIR': linuxJavaVMDir, - 'MACOS_JAVA_VM_TGZ': macosJavaVMTgz, - 'WINDOWS_JAVA_VM_TGZ': windowsJavaVMTgz, - 'LINUX_JAVA_VM_TGZ': linuxJavaVMTgz, 'COPYRIGHT_MESSAGE': install4j_copyright_message, 'BUNDLE_ID': install4jBundleId, 'INTERNAL_ID': install4jInternalId, 'WINDOWS_APPLICATION_ID': install4jWinApplicationId, - 'MACOS_DS_STORE': install4jDSStore, - 'MACOS_DMG_BG_IMAGE': install4jDMGBackgroundImage, - 'INSTALLER_NAME': install4jInstallerName, + 'MACOS_X64_DMG_DS_STORE': install4jDMGFixedDSStoreX64, + 'MACOS_AARCH64_DMG_DS_STORE': install4jDMGFixedDSStoreAarch64, + 'MACOS_DMG_BG_IMAGE': "${install4jDMGBackgroundImageBuildDir}/${install4jDMGBackgroundImageFile}", + 'MACOS_DMG_BG_FILENAME': install4j_dmg_background_filename, + 'WRAPPER_LINK': getdownWrapperLink, + 'BASH_WRAPPER_SCRIPT': getdown_bash_wrapper_script, + 'POWERSHELL_WRAPPER_SCRIPT': getdown_powershell_wrapper_script, + 'BATCH_WRAPPER_SCRIPT': getdown_batch_wrapper_script, + 'WRAPPER_SCRIPT_BIN_DIR': getdown_wrapper_script_dir, + 'MACOSARCHIVE_X64_NAME': install4jmacOSArchiveX64Name, + 'MACOSARCHIVE_AARCH64_NAME': install4jmacOSArchiveAarch64Name, 'INSTALL4J_UTILS_DIR': install4j_utils_dir, - 'GETDOWN_WEBSITE_DIR': getdown_website_dir, + 'GETDOWN_CHANNEL_DIR': getdownChannelDir, 'GETDOWN_FILES_DIR': getdown_files_dir, 'GETDOWN_RESOURCE_DIR': getdown_resource_dir, 'GETDOWN_DIST_DIR': getdownAppDistDir, @@ -1744,7 +2987,36 @@ task installers(type: com.install4j.gradle.Install4jTask) { 'UNIX_APPLICATION_FOLDER': install4jUnixApplicationFolder, 'EXECUTABLE_NAME': install4jExecutableName, 'EXTRA_SCHEME': install4jExtraScheme, + 'MAC_ICONS_FILE': install4jMacIconsFile, + 'WINDOWS_ICONS_FILE': install4jWindowsIconsFile, + 'PNG_ICON_FILE': install4jPngIconFile, + 'BACKGROUND': install4jBackground, + 'MACOSARCHIVE_X64_DMG_FILENAME': install4jmacOSArchiveX64DMGFilename, + 'MACOSARCHIVE_AARCH64_DMG_FILENAME': install4jmacOSArchiveAarch64DMGFilename, + 'MACOSARCHIVE_VOLUMEICON': install4jDMGVolumeIcon, + ] + + def varNameMap = [ + 'mac': 'MACOS', + 'windows': 'WINDOWS', + 'linux': 'LINUX' + ] + + // these are the bundled OS/architecture VMs needed by install4j + def osArch = [ + [ "mac", "x64" ], + [ "mac", "aarch64" ], + [ "windows", "x64" ], + [ "linux", "x64" ], + [ "linux", "aarch64" ] ] + osArch.forEach { os, arch -> + variables[ sprintf("%s_%s_JAVA_VM_DIR", varNameMap[os], arch.toUpperCase(Locale.ROOT)) ] = sprintf("%s/jre-%s-%s-%s/jre", jreInstallsDir, JAVA_INTEGER_VERSION, os, arch) + // N.B. For some reason install4j requires the below filename to have underscores and not hyphens + // otherwise running `gradle installers` generates a non-useful error: + // `install4j: compilation failed. Reason: java.lang.NumberFormatException: For input string: "windows"` + variables[ sprintf("%s_%s_JAVA_VM_TGZ", varNameMap[os], arch.toUpperCase(Locale.ROOT)) ] = sprintf("%s/tgz/jre_%s_%s_%s.tar.gz", jreInstallsDir, JAVA_INTEGER_VERSION, os, arch) + } //println("INSTALL4J VARIABLES:") //variables.each{k,v->println("${k}=${v}")} @@ -1755,24 +3027,110 @@ task installers(type: com.install4j.gradle.Install4jTask) { if (install4j_faster.equals("true") || CHANNEL.startsWith("LOCAL")) { faster = true disableSigning = true + disableNotarization = true } if (OSX_KEYPASS) { macKeystorePassword = OSX_KEYPASS + } + + if (OSX_ALTOOLPASS) { + appleIdPassword = OSX_ALTOOLPASS + disableNotarization = false + } else { + disableNotarization = true } doFirst { println("Using projectFile "+projectFile) + if (!disableNotarization) { println("Will notarize OSX App DMG") } } + //verbose=true - inputs.dir(getdownWebsiteDir) + inputs.dir(getdownAppBaseDir) inputs.file(install4jConfFile) inputs.file("${install4jDir}/${install4j_info_plist_file_associations}") - inputs.dir(macosJavaVMDir) - inputs.dir(windowsJavaVMDir) outputs.dir("${jalviewDir}/${install4j_build_dir}/${JAVA_VERSION}") } +def getDataHash(File myFile) { + HashCode hash = Files.asByteSource(myFile).hash(Hashing.sha256()) + return myFile.exists() + ? [ + "file" : myFile.getName(), + "filesize" : myFile.length(), + "sha256" : hash.toString() + ] + : null +} + +def writeDataJsonFile(File installersOutputTxt, File installersSha256, File dataJsonFile) { + def hash = [ + "channel" : getdownChannelName, + "date" : getDate("yyyy-MM-dd HH:mm:ss"), + "git-commit" : "${gitHash} [${gitBranch}]", + "version" : JALVIEW_VERSION + ] + // install4j installer files + if (installersOutputTxt.exists()) { + def idHash = [:] + installersOutputTxt.readLines().each { def line -> + if (line.startsWith("#")) { + return; + } + line.replaceAll("\n","") + def vals = line.split("\t") + def filename = vals[3] + def filesize = file(filename).length() + filename = filename.replaceAll(/^.*\//, "") + hash[vals[0]] = [ "id" : vals[0], "os" : vals[1], "name" : vals[2], "file" : filename, "filesize" : filesize ] + idHash."${filename}" = vals[0] + } + if (install4jCheckSums && installersSha256.exists()) { + installersSha256.readLines().each { def line -> + if (line.startsWith("#")) { + return; + } + line.replaceAll("\n","") + def vals = line.split(/\s+\*?/) + def filename = vals[1] + def innerHash = (hash.(idHash."${filename}"))."sha256" = vals[0] + } + } + } + + [ + "JAR": shadowJar.archiveFile, // executable JAR + "JVL": getdownVersionLaunchJvl, // version JVL + "SOURCE": sourceDist.archiveFile // source TGZ + ].each { key, value -> + def file = file(value) + if (file.exists()) { + def fileHash = getDataHash(file) + if (fileHash != null) { + hash."${key}" = fileHash; + } + } + } + return dataJsonFile.write(new JsonBuilder(hash).toPrettyString()) +} + +task staticMakeInstallersJsonFile { + doFirst { + def output = findProperty("i4j_output") + def sha256 = findProperty("i4j_sha256") + def json = findProperty("i4j_json") + if (output == null || sha256 == null || json == null) { + throw new GradleException("Must provide paths to all of output.txt, sha256sums, and output.json with '-Pi4j_output=... -Pi4j_sha256=... -Pi4j_json=...") + } + writeDataJsonFile(file(output), file(sha256), file(json)) + } +} + +task installers { + dependsOn installerFiles +} + spotless { java { @@ -1780,23 +3138,42 @@ spotless { } } - -task sourceDist(type: Tar) { +task createSourceReleaseProperties(type: WriteProperties) { + group = "distribution" + description = "Create the source RELEASE properties file" - def VERSION_UNDERSCORES = JALVIEW_VERSION.replaceAll("\\.", "_") - def outputFileName = "${project.name}_${VERSION_UNDERSCORES}.tar.gz" - // cater for buildship < 3.1 [3.0.1 is max version in eclipse 2018-09] - try { - archiveFileName = outputFileName - } catch (Exception e) { - archiveName = outputFileName + def sourceTarBuildDir = "${buildDir}/sourceTar" + def sourceReleasePropertiesFile = "${sourceTarBuildDir}/RELEASE" + outputFile (sourceReleasePropertiesFile) + + doFirst { + releaseProps.each{ key, val -> property key, val } + property "git.branch", gitBranch + property "git.hash", gitHash } + + outputs.file(outputFile) +} + +task sourceDist(type: Tar) { + group "distribution" + description "Create a source .tar.gz file for distribution" + + dependsOn createBuildProperties + dependsOn convertMdFiles + dependsOn eclipseAllPreferences + dependsOn createSourceReleaseProperties + + + def outputFileName = "${project.name}_${JALVIEW_VERSION_UNDERSCORES}.tar.gz" + archiveFileName = outputFileName compression Compression.GZIP into project.name def EXCLUDE_FILES=[ + "dist/*", "build/*", "bin/*", "test-output/", @@ -1811,6 +3188,7 @@ task sourceDist(type: Tar) { "*locales/**", "utils/InstallAnywhere", "**/*.log", + "RELEASE", ] def PROCESS_FILES=[ "AUTHORS", @@ -1820,7 +3198,6 @@ task sourceDist(type: Tar) { "FEATURETODO", "LICENSE", "**/README", - "RELEASE", "THIRDPARTYLIBS", "TESTNG", "build.gradle", @@ -1835,7 +3212,9 @@ task sourceDist(type: Tar) { "**/*.sh", ] def INCLUDE_FILES=[ - ".settings/org.eclipse.jdt.core.jalview.prefs", + ".classpath", + ".settings/org.eclipse.buildship.core.prefs", + ".settings/org.eclipse.jdt.core.prefs" ] from(jalviewDir) { @@ -1859,7 +3238,9 @@ task sourceDist(type: Tar) { exclude ("utils/InstallAnywhere") exclude (getdown_files_dir) - exclude (getdown_website_dir) + // getdown_website_dir and getdown_archive_dir moved to build/website/docroot/getdown + //exclude (getdown_website_dir) + //exclude (getdown_archive_dir) // exluding these as not using jars as modules yet exclude ("${j11modDir}/**/*.jar") @@ -1873,14 +3254,64 @@ task sourceDist(type: Tar) { // exclude(EXCLUDE_FILES) // exclude(PROCESS_FILES) // } + + from(file(buildProperties).getParent()) { + include(file(buildProperties).getName()) + rename(file(buildProperties).getName(), "build_properties") + filter({ line -> + line.replaceAll("^INSTALLATION=.*\$","INSTALLATION=Source Release"+" git-commit\\\\:"+gitHash+" ["+gitBranch+"]") + }) + } + + def sourceTarBuildDir = "${buildDir}/sourceTar" + from(sourceTarBuildDir) { + // this includes the appended RELEASE properties file + } } +task dataInstallersJson { + group "website" + description "Create the installers-VERSION.json data file for installer files created" + + mustRunAfter installers + mustRunAfter shadowJar + mustRunAfter sourceDist + mustRunAfter getdownArchive + + def installersOutputTxt = file("${jalviewDir}/${install4jBuildDir}/output.txt") + def installersSha256 = file("${jalviewDir}/${install4jBuildDir}/sha256sums") + + if (installersOutputTxt.exists()) { + inputs.file(installersOutputTxt) + } + if (install4jCheckSums && installersSha256.exists()) { + inputs.file(installersSha256) + } + [ + shadowJar.archiveFile, // executable JAR + getdownVersionLaunchJvl, // version JVL + sourceDist.archiveFile // source TGZ + ].each { fileName -> + if (file(fileName).exists()) { + inputs.file(fileName) + } + } + + outputs.file(hugoDataJsonFile) + + doFirst { + writeDataJsonFile(installersOutputTxt, installersSha256, hugoDataJsonFile) + } +} task helppages { + group "help" + description "Copies all help pages to build dir. Runs ant task 'pubhtmlhelp'." + dependsOn copyHelp dependsOn pubhtmlhelp - inputs.dir("${classesDir}/${help_dir}") + inputs.dir("${helpBuildDir}/${help_dir}") outputs.dir("${buildDir}/distributions/${help_dir}") } @@ -1892,6 +3323,33 @@ task j2sSetHeadlessBuild { } +task jalviewjsEnableAltFileProperty(type: WriteProperties) { + group "jalviewjs" + description "Enable the alternative J2S Config file for headless build" + + outputFile = jalviewjsJ2sSettingsFileName + def j2sPropsFile = file(jalviewjsJ2sSettingsFileName) + def j2sProps = new Properties() + if (j2sPropsFile.exists()) { + try { + def j2sPropsFileFIS = new FileInputStream(j2sPropsFile) + j2sProps.load(j2sPropsFileFIS) + j2sPropsFileFIS.close() + + j2sProps.each { prop, val -> + property(prop, val) + } + } catch (Exception e) { + println("Exception reading ${jalviewjsJ2sSettingsFileName}") + e.printStackTrace() + } + } + if (! j2sProps.stringPropertyNames().contains(jalviewjs_j2s_alt_file_property_config)) { + property(jalviewjs_j2s_alt_file_property_config, jalviewjs_j2s_alt_file_property) + } +} + + task jalviewjsSetEclipseWorkspace { def propKey = "jalviewjs_eclipse_workspace" def propVal = null @@ -2104,7 +3562,7 @@ task jalviewjsTransferUnzipAllLibs { task jalviewjsCreateJ2sSettings(type: WriteProperties) { group "JalviewJS" - description "Create the .j2s file from the j2s.* properties" + description "Create the alternative j2s file from the j2s.* properties" jalviewjsJ2sProps = project.properties.findAll { it.key.startsWith("j2s.") }.sort { it.key } def siteDirProperty = "j2s.site.directory" @@ -2123,11 +3581,11 @@ task jalviewjsCreateJ2sSettings(type: WriteProperties) { property(siteDirProperty,"${jalviewDirRelativePath}/${jalviewjsTransferSiteJsDir}") } } - outputFile = jalviewjsJ2sSettingsFileName + outputFile = jalviewjsJ2sAltSettingsFileName if (! IN_ECLIPSE) { inputs.properties(jalviewjsJ2sProps) - outputs.file(jalviewjsJ2sSettingsFileName) + outputs.file(jalviewjsJ2sAltSettingsFileName) } } @@ -2155,13 +3613,19 @@ task jalviewjsSyncAllLibs (type: Sync) { preserve { include "**" } + + // should this be exclude really ? + duplicatesStrategy "INCLUDE" + outputs.files outputFiles inputs.files inputFiles } task jalviewjsSyncResources (type: Sync) { - def inputFiles = fileTree(dir: resourceDir) + dependsOn buildResources + + def inputFiles = fileTree(dir: resourcesBuildDir) def outputDir = "${jalviewDir}/${jalviewjsSiteDir}/${jalviewjs_j2s_subdir}" from inputFiles @@ -2243,6 +3707,7 @@ task jalviewjsProjectImport(type: Exec) { args += [ "--launcher.appendVmargs", "-vmargs", "-Dorg.eclipse.equinox.p2.reconciler.dropins.directory=${jalviewDirAbsolutePath}/${jalviewjsBuildDir}/${jalviewjs_eclipse_tmp_dropins_dir}" ] if (!IN_ECLIPSE) { args += [ "-D${j2sHeadlessBuildProperty}=true" ] + args += [ "-D${jalviewjs_j2s_alt_file_property}=${jalviewjsJ2sAltSettingsFileName}" ] } inputs.file("${jalviewDir}/.project") @@ -2256,6 +3721,9 @@ task jalviewjsTranspile(type: Exec) { dependsOn jalviewjsEclipseSetup dependsOn jalviewjsProjectImport dependsOn jalviewjsEclipsePaths + if (!IN_ECLIPSE) { + dependsOn jalviewjsEnableAltFileProperty + } doFirst { // do not run a headless transpile when we claim to be in Eclipse @@ -2275,6 +3743,7 @@ task jalviewjsTranspile(type: Exec) { args += [ "--launcher.appendVmargs", "-vmargs", "-Dorg.eclipse.equinox.p2.reconciler.dropins.directory=${jalviewDirAbsolutePath}/${jalviewjsBuildDir}/${jalviewjs_eclipse_tmp_dropins_dir}" ] if (!IN_ECLIPSE) { args += [ "-D${j2sHeadlessBuildProperty}=true" ] + args += [ "-D${jalviewjs_j2s_alt_file_property}=${jalviewjsJ2sAltSettingsFileName}" ] } def stdout @@ -2302,12 +3771,12 @@ DEBUG: ${eclipseDebug} new org.apache.tools.ant.util.TeeOutputStream( logOutFOS, stdout), - standardOutput) + System.out) errorOutput = new org.apache.tools.ant.util.TeeOutputStream( new org.apache.tools.ant.util.TeeOutputStream( logErrFOS, stderr), - errorOutput) + System.err) } else { standardOutput = new org.apache.tools.ant.util.TeeOutputStream( logOutFOS, @@ -2395,7 +3864,7 @@ def jalviewjsCallCore(String name, FileCollection list, String prefixFile, Strin new org.apache.tools.ant.util.TeeOutputStream( logErrFOS, stderr), - errorOutput) + System.err) } else { standardOutput = new org.apache.tools.ant.util.TeeOutputStream( logOutFOS, @@ -2684,11 +4153,7 @@ task jalviewjsSiteTar(type: Tar) { description "Creates a tar.gz file for the website" dependsOn jalviewjsBuildSite def outputFilename = "jalviewjs-site-${JALVIEW_VERSION}.tar.gz" - try { - archiveFileName = outputFilename - } catch (Exception e) { - archiveName = outputFilename - } + archiveFileName = outputFilename compression Compression.GZIP @@ -2706,7 +4171,13 @@ task jalviewjsServer { def htmlFile = "${jalviewDirAbsolutePath}/${filename}" doLast { - SimpleHttpFileServerFactory factory = new SimpleHttpFileServerFactory() + def factory + try { + def f = Class.forName("org.gradle.plugins.javascript.envjs.http.simple.SimpleHttpFileServerFactory") + factory = f.newInstance() + } catch (ClassNotFoundException e) { + throw new GradleException("Unable to create SimpleHttpFileServerFactory") + } def port = Integer.valueOf(jalviewjs_server_port) def start = port def running = false @@ -2759,7 +4230,7 @@ task cleanJalviewjsAll { if (eclipseWorkspace != null && file(eclipseWorkspace.getAbsolutePath()+"/.metadata").exists()) { delete file(eclipseWorkspace.getAbsolutePath()+"/.metadata") } - delete "${jalviewDir}/${jalviewjs_j2s_settings}" + delete jalviewjsJ2sAltSettingsFileName } outputs.upToDateWhen( { false } ) @@ -2777,10 +4248,25 @@ task jalviewjsIDE_checkJ2sPlugin { if (eclipseHome == null || ! IN_ECLIPSE) { throw new StopExecutionException("Cannot find running Eclipse home from System.properties['eclipse.home.location']. Skipping J2S Plugin Check.") } - def eclipseJ2sPlugin = "${eclipseHome}/dropins/${j2sPluginFile.getName()}" - def eclipseJ2sPluginFile = file(eclipseJ2sPlugin) - if (!eclipseJ2sPluginFile.exists()) { - def msg = "Eclipse J2S Plugin is not installed (could not find '${eclipseJ2sPlugin}')\nTry running task jalviewjsIDE_copyJ2sPlugin" + def eclipseJ2sPluginDirs = [ "${eclipseHome}/dropins" ] + def altPluginsDir = System.properties["org.eclipse.equinox.p2.reconciler.dropins.directory"] + if (altPluginsDir != null && file(altPluginsDir).exists()) { + eclipseJ2sPluginDirs += altPluginsDir + } + def foundPlugin = false + def j2sPluginFileName = j2sPluginFile.getName() + def eclipseJ2sPlugin + def eclipseJ2sPluginFile + eclipseJ2sPluginDirs.any { dir -> + eclipseJ2sPlugin = "${dir}/${j2sPluginFileName}" + eclipseJ2sPluginFile = file(eclipseJ2sPlugin) + if (eclipseJ2sPluginFile.exists()) { + foundPlugin = true + return true + } + } + if (!foundPlugin) { + def msg = "Eclipse J2S Plugin is not installed (could not find '${j2sPluginFileName}' in\n"+eclipseJ2sPluginDirs.join("\n")+"\n)\nTry running task jalviewjsIDE_copyJ2sPlugin" System.err.println(msg) throw new StopExecutionException(msg) } @@ -2798,7 +4284,7 @@ task jalviewjsIDE_checkJ2sPlugin { System.err.println(msg) throw new StopExecutionException(msg) } else { - def msg = "Eclipse J2S Plugin is the same as '${j2sPlugin}' (this is good)" + def msg = "Eclipse J2S Plugin '${eclipseJ2sPlugin}' is the same as '${j2sPlugin}' (this is good)" println(msg) } } @@ -2858,7 +4344,7 @@ task jalviewjsIDE_PrepareSite { description "Sync libs and resources to site dir, but not closure cores" dependsOn jalviewjsIDE_SyncSiteAll - dependsOn cleanJalviewjsTransferSite + //dependsOn cleanJalviewjsTransferSite // not sure why this clean is here -- will slow down a re-run of this task } @@ -2902,8 +4388,157 @@ task eclipseAutoBuildTask { } +task jalviewjsCopyStderrLaunchFile(type: Copy) { + from file(jalviewjs_stderr_launch) + into jalviewjsSiteDir + + inputs.file jalviewjs_stderr_launch + outputs.file jalviewjsStderrLaunchFilename +} + +task cleanJalviewjsChromiumUserDir { + doFirst { + delete jalviewjsChromiumUserDir + } + outputs.dir jalviewjsChromiumUserDir + // always run when depended on + outputs.upToDateWhen { !file(jalviewjsChromiumUserDir).exists() } +} + +task jalviewjsChromiumProfile { + dependsOn cleanJalviewjsChromiumUserDir + mustRunAfter cleanJalviewjsChromiumUserDir + + def firstRun = file("${jalviewjsChromiumUserDir}/First Run") + + doFirst { + mkdir jalviewjsChromiumProfileDir + firstRun.text = "" + } + outputs.file firstRun +} + +task jalviewjsLaunchTest { + group "Test" + description "Check JalviewJS opens in a browser" + dependsOn jalviewjsBuildSite + dependsOn jalviewjsCopyStderrLaunchFile + dependsOn jalviewjsChromiumProfile + + def macOS = OperatingSystem.current().isMacOsX() + def chromiumBinary = macOS ? jalviewjs_macos_chromium_binary : jalviewjs_chromium_binary + if (chromiumBinary.startsWith("~/")) { + chromiumBinary = System.getProperty("user.home") + chromiumBinary.substring(1) + } + + def stdout + def stderr + doFirst { + def timeoutms = Integer.valueOf(jalviewjs_chromium_overall_timeout) * 1000 + + def binary = file(chromiumBinary) + if (!binary.exists()) { + throw new StopExecutionException("Could not find chromium binary '${chromiumBinary}'. Cannot run task ${name}.") + } + stdout = new ByteArrayOutputStream() + stderr = new ByteArrayOutputStream() + def execStdout + def execStderr + if (jalviewjs_j2s_to_console.equals("true")) { + execStdout = new org.apache.tools.ant.util.TeeOutputStream( + stdout, + System.out) + execStderr = new org.apache.tools.ant.util.TeeOutputStream( + stderr, + System.err) + } else { + execStdout = stdout + execStderr = stderr + } + def execArgs = [ + "--no-sandbox", // --no-sandbox IS USED BY THE THORIUM APPIMAGE ON THE BUILDSERVER + "--headless=new", + "--disable-gpu", + "--timeout=${timeoutms}", + "--virtual-time-budget=${timeoutms}", + "--user-data-dir=${jalviewDirAbsolutePath}/${jalviewjsBuildDir}/${jalviewjs_chromium_user_dir}", + "--profile-directory=${jalviewjs_chromium_profile_name}", + "--allow-file-access-from-files", + "--enable-logging=stderr", + "file://${jalviewDirAbsolutePath}/${jalviewjsStderrLaunchFilename}" + ] + + if (true || macOS) { + ScheduledExecutorService executor = Executors.newScheduledThreadPool(3); + Future f1 = executor.submit( + () -> { + exec { + standardOutput = execStdout + errorOutput = execStderr + executable(chromiumBinary) + args(execArgs) + println "COMMAND: '"+commandLine.join(" ")+"'" + } + executor.shutdownNow() + } + ) + + def noChangeBytes = 0 + def noChangeIterations = 0 + executor.scheduleAtFixedRate( + () -> { + String stderrString = stderr.toString() + // shutdown the task if we have a success string + if (stderrString.contains(jalviewjs_desktop_init_string)) { + f1.cancel() + Thread.sleep(1000) + executor.shutdownNow() + } + // if no change in stderr for 10s then also end + if (noChangeIterations >= jalviewjs_chromium_idle_timeout) { + executor.shutdownNow() + } + if (stderrString.length() == noChangeBytes) { + noChangeIterations++ + } else { + noChangeBytes = stderrString.length() + noChangeIterations = 0 + } + }, + 1, 1, TimeUnit.SECONDS) + + executor.schedule(new Runnable(){ + public void run(){ + f1.cancel() + executor.shutdownNow() + } + }, timeoutms, TimeUnit.MILLISECONDS) + + executor.awaitTermination(timeoutms+10000, TimeUnit.MILLISECONDS) + executor.shutdownNow() + } + + } + + doLast { + def found = false + stderr.toString().eachLine { line -> + if (line.contains(jalviewjs_desktop_init_string)) { + println("Found line '"+line+"'") + found = true + return + } + } + if (!found) { + throw new GradleException("Could not find evidence of Desktop launch in JalviewJS.") + } + } +} + + task jalviewjs { group "JalviewJS" - description "Build the site" + description "Build the JalviewJS site and run the launch test" dependsOn jalviewjsBuildSite + dependsOn jalviewjsLaunchTest }