From: Ben Soares Date: Mon, 18 Nov 2019 16:55:51 +0000 (+0000) Subject: JAL-3477 New jvmmemmax config option in getdown and getdown config. Set to 32G cap. X-Git-Tag: Develop-2_11_2_0-d20201215~88^2~12 X-Git-Url: http://source.jalview.org/gitweb/?a=commitdiff_plain;h=5911188c77dae6665babf4041dc11484f4a09a0b;p=jalview.git JAL-3477 New jvmmemmax config option in getdown and getdown config. Set to 32G cap. --- diff --git a/getdown/lib/getdown-core.jar b/getdown/lib/getdown-core.jar index bbe1476..1d6b83c 100644 Binary files a/getdown/lib/getdown-core.jar and b/getdown/lib/getdown-core.jar differ diff --git a/getdown/lib/getdown-launcher-local.jar b/getdown/lib/getdown-launcher-local.jar index 4a36222..50faf1b 100644 Binary files a/getdown/lib/getdown-launcher-local.jar and b/getdown/lib/getdown-launcher-local.jar differ diff --git a/getdown/lib/getdown-launcher.jar b/getdown/lib/getdown-launcher.jar index 85ede0c..077c2c4 100644 Binary files a/getdown/lib/getdown-launcher.jar and b/getdown/lib/getdown-launcher.jar differ diff --git a/getdown/src/getdown/ant/pom.xml b/getdown/src/getdown/ant/pom.xml index 6d6f8e9..5a2352b 100644 --- a/getdown/src/getdown/ant/pom.xml +++ b/getdown/src/getdown/ant/pom.xml @@ -4,7 +4,7 @@ com.threerings.getdown getdown - 1.8.3-1.1.8_FJVL + 1.8.3-1.2.0_FJVL getdown-ant diff --git a/getdown/src/getdown/core/pom.xml b/getdown/src/getdown/core/pom.xml index 51d3d5f..c5d3b2d 100644 --- a/getdown/src/getdown/core/pom.xml +++ b/getdown/src/getdown/core/pom.xml @@ -4,7 +4,7 @@ com.threerings.getdown getdown - 1.8.3-1.1.8_FJVL + 1.8.3-1.2.0_FJVL getdown-core diff --git a/getdown/src/getdown/core/src/main/java/com/threerings/getdown/data/Application.java b/getdown/src/getdown/core/src/main/java/com/threerings/getdown/data/Application.java index 7c2a427..f41bf6a 100644 --- a/getdown/src/getdown/core/src/main/java/com/threerings/getdown/data/Application.java +++ b/getdown/src/getdown/core/src/main/java/com/threerings/getdown/data/Application.java @@ -790,45 +790,27 @@ public class Application addAll(jvmargs, _jvmargs); } - // see if a percentage of physical memory option exists - int jvmmempc = config.getInt("jvmmempc", -1); + // see if a percentage of physical memory, or max heap size options exist + String jvmmempc = config.getString("jvmmempc", null); + String jvmmemmax = config.getString("jvmmemmax", null); // app_id prefixed setting overrides if (appPrefix.length() > 0) { - jvmmempc = config.getInt(appPrefix + "jvmmempc", jvmmempc); - } - if (0 <= jvmmempc && jvmmempc <= 100) { - - long maxMemLong = -1; - - try - { - maxMemLong = MemorySetting.memPercent(jvmmempc); - } catch (Exception e) - { - e.printStackTrace(); - } catch (Throwable t) - { - t.printStackTrace(); - } - - if (maxMemLong > 0) - { - - String[] maxMemHeapArg = new String[]{"-Xmx"+Long.toString(maxMemLong)}; - // remove other max heap size arg - ARG: for (int i = 0; i < _jvmargs.size(); i++) { - if (_jvmargs.get(i) instanceof java.lang.String && _jvmargs.get(i).startsWith("-Xmx")) { - _jvmargs.remove(i); - break ARG; - } + jvmmempc = config.getString(appPrefix + "jvmmempc", jvmmempc); + jvmmemmax = config.getString(appPrefix + "jvmmemmax", jvmmemmax); + } + long maxMemLong = -1; + maxMemLong = MemorySetting.getMemorySetting(jvmmemmax, jvmmempc); + if (maxMemLong > 0) + { + String[] maxMemHeapArg = new String[]{"-Xmx"+Long.toString(maxMemLong)}; + // remove other max heap size arg + ARG: for (int i = 0; i < _jvmargs.size(); i++) { + if (_jvmargs.get(i) instanceof java.lang.String && _jvmargs.get(i).startsWith("-Xmx")) { + _jvmargs.remove(i); + break ARG; } - addAll(maxMemHeapArg, _jvmargs); - log.info("Max memory set", "maxMemHeapArg", maxMemHeapArg[0]); - } - - } else if (jvmmempc != -1) { - log.warning("'jvmmempc' value must be in range 0 to 100 (read as '"+Integer.toString(jvmmempc)+"')"); + addAll(maxMemHeapArg, _jvmargs); } // get the set of optimum JVM arguments diff --git a/getdown/src/getdown/core/src/main/java/com/threerings/getdown/util/Config.java b/getdown/src/getdown/core/src/main/java/com/threerings/getdown/util/Config.java index 8767ae0..c7e2933 100644 --- a/getdown/src/getdown/core/src/main/java/com/threerings/getdown/util/Config.java +++ b/getdown/src/getdown/core/src/main/java/com/threerings/getdown/util/Config.java @@ -471,7 +471,7 @@ public class Config private final Map _data; - public static final List allowedReplaceKeys = Arrays.asList("appbase","apparg","jvmarg","jvmmempc"); // these are the ones we might use + public static final List allowedReplaceKeys = Arrays.asList("appbase","apparg","jvmarg","jvmmempc","jvmmemmax"); // these are the ones we might use public static final List allowedMergeKeys = Arrays.asList("apparg","jvmarg"); // these are the ones we might use //private final List allowedMergeKeys = Arrays.asList("apparg","jvmarg","resource","code","java_location"); // (not exhaustive list here) } diff --git a/getdown/src/getdown/core/src/main/java/jalview/bin/MemoryPercent.java b/getdown/src/getdown/core/src/main/java/jalview/bin/MemoryPercent.java new file mode 100644 index 0000000..2de2157 --- /dev/null +++ b/getdown/src/getdown/core/src/main/java/jalview/bin/MemoryPercent.java @@ -0,0 +1,53 @@ +package jalview.bin; + +import java.lang.management.ManagementFactory; +import java.lang.management.OperatingSystemMXBean; + +public class MemoryPercent +{ + + protected static long getPhysicalMemory() + { + final OperatingSystemMXBean o = ManagementFactory + .getOperatingSystemMXBean(); + + try + { + if (o instanceof com.sun.management.OperatingSystemMXBean) + { + final com.sun.management.OperatingSystemMXBean osb = (com.sun.management.OperatingSystemMXBean) o; + return osb.getTotalPhysicalMemorySize(); + } + } catch (NoClassDefFoundError e) + { + // com.sun.management.OperatingSystemMXBean doesn't exist in this JVM + System.out.println("No com.sun.management.OperatingSystemMXBean"); + } + + // We didn't get a com.sun.management.OperatingSystemMXBean. + return -1; + } + + public static long memPercent(int percent) + { + return memPercent(percent); + } + public static long memPercent(float percent) + { + long memPercent = -1; + + long physicalMem = getPhysicalMemory(); + if (physicalMem > MemorySetting.applicationMinMemory) + { + // try and set at least applicationMinMemory and thereafter ensure + // leaveFreeMinMemory is left for the OS + memPercent = Math.max(MemorySetting.applicationMinMemory, + (long) (physicalMem + - Math.max(physicalMem * (100 - percent) / 100, + MemorySetting.leaveFreeMinMemory))); + } + + return memPercent; + } + +} diff --git a/getdown/src/getdown/core/src/main/java/jalview/bin/MemorySetting.java b/getdown/src/getdown/core/src/main/java/jalview/bin/MemorySetting.java index 8af09da..ac1ac28 100644 --- a/getdown/src/getdown/core/src/main/java/jalview/bin/MemorySetting.java +++ b/getdown/src/getdown/core/src/main/java/jalview/bin/MemorySetting.java @@ -1,51 +1,177 @@ package jalview.bin; -import java.lang.management.ManagementFactory; -import java.lang.management.OperatingSystemMXBean; - public class MemorySetting { public static final long leaveFreeMinMemory = 536870912; // 0.5 GB public static final long applicationMinMemory = 536870912; // 0.5 GB - protected static long getPhysicalMemory() + private final static int maxHeapSizePerCentDefault = 90; + + public final static String maxHeapSizePerCentProperty = "jvmmempc"; + + private final static long maxHeapSizeDefault = 34359738368L; // 32GB + + private final static long noMemMaxHeapSizeDefault = 8589934592L; // 8GB + + public final static String maxHeapSizeProperty = "jvmmemmax"; + + public static long getMemorySetting() { - final OperatingSystemMXBean o = ManagementFactory - .getOperatingSystemMXBean(); + return getMemorySetting(null, null); + } - try + public static long getMemorySetting(String jvmmemmaxString, + String jvmmempcString) + { + // actual Xmx value-to-be + long maxMemLong = -1; + + // get (absolute) jvmmaxmem setting + long memmax = maxHeapSizeDefault; + String jvmmemmaxorig = jvmmemmaxString; + if (jvmmemmaxorig == null) + { + jvmmemmaxorig = System.getProperty(maxHeapSizeProperty); + } + String jvmmemmax = jvmmemmaxorig; + if (jvmmemmax != null && jvmmemmax.length() > 0) { - if (o instanceof com.sun.management.OperatingSystemMXBean) + long multiplier = 1; + switch (jvmmemmax.toLowerCase().substring(jvmmemmax.length() - 1)) + { + case "t": + multiplier = 1099511627776L; // 2^40 + jvmmemmax = jvmmemmax.substring(0, jvmmemmax.length() - 1); + break; + case "g": + multiplier = 1073741824; // 2^30 + jvmmemmax = jvmmemmax.substring(0, jvmmemmax.length() - 1); + break; + case "m": + multiplier = 1048576; // 2^20 + jvmmemmax = jvmmemmax.substring(0, jvmmemmax.length() - 1); + break; + case "k": + multiplier = 1024; // 2^10 + jvmmemmax = jvmmemmax.substring(0, jvmmemmax.length() - 1); + break; + case "b": + multiplier = 1; // 2^0 + jvmmemmax = jvmmemmax.substring(0, jvmmemmax.length() - 1); + break; + default: + break; + } + + // parse the arg + try + { + memmax = Long.parseLong(jvmmemmax); + } catch (NumberFormatException e) + { + memmax = maxHeapSizeDefault; + System.out.println("MemorySetting Property '" + maxHeapSizeProperty + + "' (" + + jvmmemmaxorig + "') badly formatted, using default (" + + memmax + ")."); + } + + // apply multiplier if not too big (i.e. bigger than a long) + if (Long.MAX_VALUE / memmax < multiplier) + { + memmax = maxHeapSizeDefault; + System.out.println( + "MemorySetting Property '" + maxHeapSizeProperty + "' (" + + jvmmemmaxorig + + ") too big, using default (" + memmax + ")."); + } + else + { + memmax = multiplier * memmax; + } + + // check at least minimum value (this accounts for negatives too) + if (memmax < MemorySetting.applicationMinMemory) { - final com.sun.management.OperatingSystemMXBean osb = (com.sun.management.OperatingSystemMXBean) o; - return osb.getTotalPhysicalMemorySize(); + memmax = MemorySetting.applicationMinMemory; + System.out.println( + "MemorySetting Property '" + maxHeapSizeProperty + "' (" + + jvmmemmaxorig + + ") too small, using minimum (" + memmax + ")."); } - } catch (NoClassDefFoundError e) + + } + else { - // com.sun.management.OperatingSystemMXBean doesn't exist in this JVM - System.err.println("No com.sun.management.OperatingSystemMXBean"); + // no need to warn if no setting + // System.out.println("MemorySetting Property '" + maxHeapSizeProperty + "' not + // set."); } - // We didn't get a com.sun.management.OperatingSystemMXBean. - return -1; - } + // get max percent of physical memory + float percent = maxHeapSizePerCentDefault; + String jvmmempc = jvmmempcString; + if (jvmmempc == null) + { + jvmmempc = System.getProperty(maxHeapSizePerCentProperty); + } + long pcmem = -1; + try + { + if (jvmmempc != null) + { + float trypercent = Float.parseFloat(jvmmempc); + if (0 < trypercent && trypercent <= 100f) + { + percent = trypercent; + } + else + { + System.out.println( + "MemorySetting Property '" + maxHeapSizePerCentProperty + + "' should be in range 1..100"); + } + } + } catch (NumberFormatException e) + { + System.out.println( + "MemorySetting property '" + maxHeapSizePerCentProperty + + "' (" + jvmmempc + ") badly formatted"); + } - public static long memPercent(int percent) - { - long memPercent = -1; + // catch everything in case of no com.sun.management.OperatingSystemMXBean + boolean memoryPercentError = false; + try + { + pcmem = MemoryPercent.memPercent(percent); + } catch (Throwable t) + { + memoryPercentError = true; + System.out.println("Problem calling MemoryPercent.memPercent(" + + percent + + "). Likely to be problem with com.sun.management.OperatingSystemMXBean"); + t.printStackTrace(); + } + // In the case of an error reading the percentage if physical memory, let's cap maxMemLong to 8GB + if (memoryPercentError && jvmmempc != null && pcmem == -1 + && memmax > noMemMaxHeapSizeDefault) + { + System.out.println( + "Capping maximum memory to 8GB due to failure to read physical memory size."); + memmax = noMemMaxHeapSizeDefault; + } - long physicalMem = getPhysicalMemory(); - if (physicalMem > applicationMinMemory) + if (pcmem == -1) // not set + { + maxMemLong = memmax; + } + else { - // try and set at least applicationMinMemory and thereafter ensure - // leaveFreeMinMemory is left for the OS - memPercent = Math.max(applicationMinMemory, - physicalMem - Math.max(physicalMem * (100 - percent) / 100, - leaveFreeMinMemory)); + maxMemLong = Math.min(pcmem, memmax); } - return memPercent; + return maxMemLong; } } diff --git a/getdown/src/getdown/launcher/pom.xml b/getdown/src/getdown/launcher/pom.xml index bf0b736..7c86a9f 100644 --- a/getdown/src/getdown/launcher/pom.xml +++ b/getdown/src/getdown/launcher/pom.xml @@ -4,7 +4,7 @@ com.threerings.getdown getdown - 1.8.3-1.1.8_FJVL + 1.8.3-1.2.0_FJVL getdown-launcher diff --git a/getdown/src/getdown/mvn_cmd b/getdown/src/getdown/mvn_cmd index 0cd54eb..a70deeb 100755 --- a/getdown/src/getdown/mvn_cmd +++ b/getdown/src/getdown/mvn_cmd @@ -3,7 +3,7 @@ if [ x$JVLVERSION != x ]; then export VERSION=$JVLVERSION else - export VERSION=1.8.3-1.1.8_JVL + export VERSION=1.8.3-1.2.0_JVL fi if [ x${VERSION%_JVL} = x$VERSION ]; then diff --git a/getdown/src/getdown/pom.xml b/getdown/src/getdown/pom.xml index a73b59e..238eac9 100644 --- a/getdown/src/getdown/pom.xml +++ b/getdown/src/getdown/pom.xml @@ -10,7 +10,7 @@ com.threerings.getdown getdown pom - 1.8.3-1.1.8_FJVL + 1.8.3-1.2.0_FJVL getdown An application installer and updater. diff --git a/gradle.properties b/gradle.properties index 2dbdad3..3d74a81 100644 --- a/gradle.properties +++ b/gradle.properties @@ -63,6 +63,7 @@ getdown_txt_title = Jalview getdown_txt_allow_offline = true getdown_txt_max_concurrent_downloads = 10 getdown_txt_jalview.jvmmempc = 90 +getdown_txt_jalview.jvmmemmax = 32G getdown_txt_multi_jvmarg = -Dgetdownappdir=%APPDIR% getdown_txt_strict_comments = true getdown_txt_title = Jalview @@ -139,4 +140,4 @@ RUNRSYNC=false bamboo_channelbase = https://builds.jalview.org/browse bamboo_planKey = bamboo_getdown_channel_suffix = /latest/artifact/shared/getdown-channel - \ No newline at end of file + diff --git a/j11lib/getdown-core.jar b/j11lib/getdown-core.jar index bbe1476..1d6b83c 100644 Binary files a/j11lib/getdown-core.jar and b/j11lib/getdown-core.jar differ diff --git a/j8lib/getdown-core.jar b/j8lib/getdown-core.jar index bbe1476..1d6b83c 100644 Binary files a/j8lib/getdown-core.jar and b/j8lib/getdown-core.jar differ diff --git a/src/jalview/bin/MemoryPercent.java b/src/jalview/bin/MemoryPercent.java index 66c9859..2de2157 100644 --- a/src/jalview/bin/MemoryPercent.java +++ b/src/jalview/bin/MemoryPercent.java @@ -21,7 +21,7 @@ public class MemoryPercent } catch (NoClassDefFoundError e) { // com.sun.management.OperatingSystemMXBean doesn't exist in this JVM - Cache.log.error("No com.sun.management.OperatingSystemMXBean"); + System.out.println("No com.sun.management.OperatingSystemMXBean"); } // We didn't get a com.sun.management.OperatingSystemMXBean. diff --git a/src/jalview/bin/MemorySetting.java b/src/jalview/bin/MemorySetting.java index 7984f3e..ac1ac28 100644 --- a/src/jalview/bin/MemorySetting.java +++ b/src/jalview/bin/MemorySetting.java @@ -12,6 +12,8 @@ public class MemorySetting private final static long maxHeapSizeDefault = 34359738368L; // 32GB + private final static long noMemMaxHeapSizeDefault = 8589934592L; // 8GB + public final static String maxHeapSizeProperty = "jvmmemmax"; public static long getMemorySetting() @@ -153,11 +155,11 @@ public class MemorySetting } // In the case of an error reading the percentage if physical memory, let's cap maxMemLong to 8GB if (memoryPercentError && jvmmempc != null && pcmem == -1 - && memmax > 8589934592L) + && memmax > noMemMaxHeapSizeDefault) { System.out.println( "Capping maximum memory to 8GB due to failure to read physical memory size."); - memmax = 8589934592L; + memmax = noMemMaxHeapSizeDefault; } if (pcmem == -1) // not set