From fb97a6a7e2f0c39910f283d858b185514521f1b5 Mon Sep 17 00:00:00 2001 From: Ben Soares Date: Mon, 18 Nov 2019 15:21:14 +0000 Subject: [PATCH] JAL-3477 New sensible default decisions on memory, and new jvmmemmax setting for setting absolute memory level --- src/jalview/bin/Launcher.java | 47 ++----- src/jalview/bin/MemoryPercent.java | 53 ++++++++ src/jalview/bin/MemorySetting.java | 178 +++++++++++++++++++++++---- test/jalview/bin/MemorySettingTest.java | 108 ++++++++++++++++ utils/{ => testnglibs}/classgraph-4.1.6.jar | Bin 346926 -> 346926 bytes 5 files changed, 319 insertions(+), 67 deletions(-) create mode 100644 src/jalview/bin/MemoryPercent.java create mode 100644 test/jalview/bin/MemorySettingTest.java rename utils/{ => testnglibs}/classgraph-4.1.6.jar (100%) diff --git a/src/jalview/bin/Launcher.java b/src/jalview/bin/Launcher.java index 412f119..041a7b1 100644 --- a/src/jalview/bin/Launcher.java +++ b/src/jalview/bin/Launcher.java @@ -7,13 +7,8 @@ import java.util.ArrayList; public class Launcher { - private final static String startClass = "jalview.bin.Jalview"; - private final static int maxHeapSizePerCent = 90; - - private final static String maxHeapSizePerCentProperty = "jvmmempc"; - private final static String dockIconPath = "JalviewLogo_Huge.png"; public static void main(String[] args) @@ -65,41 +60,8 @@ public class Launcher if (!memSet) { - long maxMemLong = -1; - int percent = maxHeapSizePerCent; - String jvmmempc = System.getProperty(maxHeapSizePerCentProperty); - try - { - if (jvmmempc != null) - { - int trypercent = Integer.parseInt(jvmmempc); - if (0 < trypercent && trypercent <= 100) - { - percent = trypercent; - } - else - { - System.out.println("Property '" + maxHeapSizePerCentProperty - + "' should be in range 1..100"); - } - } - } catch (Exception e) - { - System.out.println("Error parsing " + maxHeapSizePerCentProperty - + " '" + jvmmempc + "'"); - } - - try - { - maxMemLong = MemorySetting.memPercent(percent); - } catch (Exception e) - { - e.printStackTrace(); - } catch (Throwable t) - { - t.printStackTrace(); - } - + long maxMemLong = MemorySetting.getMemorySetting(); + if (maxMemLong > 0) { memSetting = "-Xmx" + Long.toString(maxMemLong); @@ -130,6 +92,11 @@ public class Launcher System.out.println("Running " + startClass + " with " + (memSetting == null ? "no memSetting" : memSetting)); + if (System.getProperty("launcherstop") != null + && System.getProperty("launcherstop").equals("true")) + { + System.exit(0); + } try { builder.inheritIO(); diff --git a/src/jalview/bin/MemoryPercent.java b/src/jalview/bin/MemoryPercent.java new file mode 100644 index 0000000..66c9859 --- /dev/null +++ b/src/jalview/bin/MemoryPercent.java @@ -0,0 +1,53 @@ +package jalview.bin; + +import java.lang.management.ManagementFactory; +import java.lang.management.OperatingSystemMXBean; + +public class MemoryPercent +{ + + protected static long getPhysicalMemory() + { + final OperatingSystemMXBean o = ManagementFactory + .getOperatingSystemMXBean(); + + try + { + if (o instanceof com.sun.management.OperatingSystemMXBean) + { + final com.sun.management.OperatingSystemMXBean osb = (com.sun.management.OperatingSystemMXBean) o; + return osb.getTotalPhysicalMemorySize(); + } + } catch (NoClassDefFoundError e) + { + // com.sun.management.OperatingSystemMXBean doesn't exist in this JVM + Cache.log.error("No com.sun.management.OperatingSystemMXBean"); + } + + // We didn't get a com.sun.management.OperatingSystemMXBean. + return -1; + } + + public static long memPercent(int percent) + { + return memPercent(percent); + } + public static long memPercent(float percent) + { + long memPercent = -1; + + long physicalMem = getPhysicalMemory(); + if (physicalMem > MemorySetting.applicationMinMemory) + { + // try and set at least applicationMinMemory and thereafter ensure + // leaveFreeMinMemory is left for the OS + memPercent = Math.max(MemorySetting.applicationMinMemory, + (long) (physicalMem + - Math.max(physicalMem * (100 - percent) / 100, + MemorySetting.leaveFreeMinMemory))); + } + + return memPercent; + } + +} diff --git a/src/jalview/bin/MemorySetting.java b/src/jalview/bin/MemorySetting.java index c8bc222..7984f3e 100644 --- a/src/jalview/bin/MemorySetting.java +++ b/src/jalview/bin/MemorySetting.java @@ -1,51 +1,175 @@ package jalview.bin; -import java.lang.management.ManagementFactory; -import java.lang.management.OperatingSystemMXBean; - public class MemorySetting { public static final long leaveFreeMinMemory = 536870912; // 0.5 GB public static final long applicationMinMemory = 536870912; // 0.5 GB - protected static long getPhysicalMemory() + private final static int maxHeapSizePerCentDefault = 90; + + public final static String maxHeapSizePerCentProperty = "jvmmempc"; + + private final static long maxHeapSizeDefault = 34359738368L; // 32GB + + public final static String maxHeapSizeProperty = "jvmmemmax"; + + public static long getMemorySetting() { - final OperatingSystemMXBean o = ManagementFactory - .getOperatingSystemMXBean(); + return getMemorySetting(null, null); + } - try + public static long getMemorySetting(String jvmmemmaxString, + String jvmmempcString) + { + // actual Xmx value-to-be + long maxMemLong = -1; + + // get (absolute) jvmmaxmem setting + long memmax = maxHeapSizeDefault; + String jvmmemmaxorig = jvmmemmaxString; + if (jvmmemmaxorig == null) + { + jvmmemmaxorig = System.getProperty(maxHeapSizeProperty); + } + String jvmmemmax = jvmmemmaxorig; + if (jvmmemmax != null && jvmmemmax.length() > 0) { - if (o instanceof com.sun.management.OperatingSystemMXBean) + long multiplier = 1; + switch (jvmmemmax.toLowerCase().substring(jvmmemmax.length() - 1)) { - final com.sun.management.OperatingSystemMXBean osb = (com.sun.management.OperatingSystemMXBean) o; - return osb.getTotalPhysicalMemorySize(); + case "t": + multiplier = 1099511627776L; // 2^40 + jvmmemmax = jvmmemmax.substring(0, jvmmemmax.length() - 1); + break; + case "g": + multiplier = 1073741824; // 2^30 + jvmmemmax = jvmmemmax.substring(0, jvmmemmax.length() - 1); + break; + case "m": + multiplier = 1048576; // 2^20 + jvmmemmax = jvmmemmax.substring(0, jvmmemmax.length() - 1); + break; + case "k": + multiplier = 1024; // 2^10 + jvmmemmax = jvmmemmax.substring(0, jvmmemmax.length() - 1); + break; + case "b": + multiplier = 1; // 2^0 + jvmmemmax = jvmmemmax.substring(0, jvmmemmax.length() - 1); + break; + default: + break; } - } catch (NoClassDefFoundError e) + + // parse the arg + try + { + memmax = Long.parseLong(jvmmemmax); + } catch (NumberFormatException e) + { + memmax = maxHeapSizeDefault; + System.out.println("MemorySetting Property '" + maxHeapSizeProperty + + "' (" + + jvmmemmaxorig + "') badly formatted, using default (" + + memmax + ")."); + } + + // apply multiplier if not too big (i.e. bigger than a long) + if (Long.MAX_VALUE / memmax < multiplier) + { + memmax = maxHeapSizeDefault; + System.out.println( + "MemorySetting Property '" + maxHeapSizeProperty + "' (" + + jvmmemmaxorig + + ") too big, using default (" + memmax + ")."); + } + else + { + memmax = multiplier * memmax; + } + + // check at least minimum value (this accounts for negatives too) + if (memmax < MemorySetting.applicationMinMemory) + { + memmax = MemorySetting.applicationMinMemory; + System.out.println( + "MemorySetting Property '" + maxHeapSizeProperty + "' (" + + jvmmemmaxorig + + ") too small, using minimum (" + memmax + ")."); + } + + } + else { - // com.sun.management.OperatingSystemMXBean doesn't exist in this JVM - Cache.log.error("No com.sun.management.OperatingSystemMXBean"); + // no need to warn if no setting + // System.out.println("MemorySetting Property '" + maxHeapSizeProperty + "' not + // set."); } - // We didn't get a com.sun.management.OperatingSystemMXBean. - return -1; - } + // get max percent of physical memory + float percent = maxHeapSizePerCentDefault; + String jvmmempc = jvmmempcString; + if (jvmmempc == null) + { + jvmmempc = System.getProperty(maxHeapSizePerCentProperty); + } + long pcmem = -1; + try + { + if (jvmmempc != null) + { + float trypercent = Float.parseFloat(jvmmempc); + if (0 < trypercent && trypercent <= 100f) + { + percent = trypercent; + } + else + { + System.out.println( + "MemorySetting Property '" + maxHeapSizePerCentProperty + + "' should be in range 1..100"); + } + } + } catch (NumberFormatException e) + { + System.out.println( + "MemorySetting property '" + maxHeapSizePerCentProperty + + "' (" + jvmmempc + ") badly formatted"); + } - public static long memPercent(int percent) - { - long memPercent = -1; + // catch everything in case of no com.sun.management.OperatingSystemMXBean + boolean memoryPercentError = false; + try + { + pcmem = MemoryPercent.memPercent(percent); + } catch (Throwable t) + { + memoryPercentError = true; + System.out.println("Problem calling MemoryPercent.memPercent(" + + percent + + "). Likely to be problem with com.sun.management.OperatingSystemMXBean"); + t.printStackTrace(); + } + // In the case of an error reading the percentage if physical memory, let's cap maxMemLong to 8GB + if (memoryPercentError && jvmmempc != null && pcmem == -1 + && memmax > 8589934592L) + { + System.out.println( + "Capping maximum memory to 8GB due to failure to read physical memory size."); + memmax = 8589934592L; + } - long physicalMem = getPhysicalMemory(); - if (physicalMem > applicationMinMemory) + if (pcmem == -1) // not set + { + maxMemLong = memmax; + } + else { - // try and set at least applicationMinMemory and thereafter ensure - // leaveFreeMinMemory is left for the OS - memPercent = Math.max(applicationMinMemory, - physicalMem - Math.max(physicalMem * (100 - percent) / 100, - leaveFreeMinMemory)); + maxMemLong = Math.min(pcmem, memmax); } - return memPercent; + return maxMemLong; } } diff --git a/test/jalview/bin/MemorySettingTest.java b/test/jalview/bin/MemorySettingTest.java new file mode 100644 index 0000000..5577f6c --- /dev/null +++ b/test/jalview/bin/MemorySettingTest.java @@ -0,0 +1,108 @@ +/* + * Jalview - A Sequence Alignment Editor and Viewer ($$Version-Rel$$) + * Copyright (C) $$Year-Rel$$ The Jalview Authors + * + * This file is part of Jalview. + * + * Jalview is free software: you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation, either version 3 + * of the License, or (at your option) any later version. + * + * Jalview is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty + * of MERCHANTABILITY or FITNESS FOR A PARTICULAR + * PURPOSE. See the GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Jalview. If not, see . + * The Jalview Authors are detailed in the 'AUTHORS' file. + */ +package jalview.bin; + +import static org.testng.AssertJUnit.assertEquals; +import static org.testng.AssertJUnit.assertTrue; + +import org.testng.annotations.Test; + +public class MemorySettingTest +{ + + @Test(groups = "Functional") + public void testGetMemorySetting() + { + long KB = 1024; + long MB = KB * KB; + long GB = MB * KB; + // long TB = GB * KB; + + /* some of these tests assume a host machine with RAM somewhere between 1GB and 1TB */ + + // should return 100% of physical memory available (or 1TB whichever is smaller) + long mem1 = MemorySetting.getMemorySetting("1T", "100"); + long fullmem = mem1 + 512 * MB; // mem1 gets 512MB removed for the OS + long mem1b = MemorySetting.getMemorySetting("1t", "100"); + assertTrue(mem1 > 1 * GB); + assertEquals(mem1, mem1b); + + // test 10% memory. Note 512MB is set as minimum, so adjust to 50% if less than + // 5GB RAM. + String pc; + Float pcf; + if (mem1 > 5 * GB) + { + pc = "10"; + pcf = 0.1f; + } + else + { + pc = "50"; + pcf = 0.5f; + } + long mem1c = MemorySetting.getMemorySetting("1T", pc); + assertTrue(mem1c > (pcf - 0.01) * fullmem && mem1c < (pcf + 0.01) * fullmem); // allowing for floating point errors + + // should return 1GB (assuming host machine has more than 1GB RAM) + long mem2 = MemorySetting.getMemorySetting("1G", "100"); + long mem2b = MemorySetting.getMemorySetting("1g", "100"); + assertEquals(mem2, 1 * GB); + assertEquals(mem2, mem2b); + + long mem3 = MemorySetting.getMemorySetting("1024M", "100"); + long mem3b = MemorySetting.getMemorySetting("1024m", "100"); + assertEquals(mem3, 1024 * MB); + assertEquals(mem3, mem3b); + + long mem4 = MemorySetting.getMemorySetting("1048576K", "100"); + long mem4b = MemorySetting.getMemorySetting("1048576k", "100"); + assertEquals(mem4, 1048576 * KB); + assertEquals(mem4, mem4b); + + long mem5 = MemorySetting.getMemorySetting("1073741824B", "100"); + long mem5b = MemorySetting.getMemorySetting("1073741824b", "100"); + long mem5c = MemorySetting.getMemorySetting("1073741824", "100"); + assertEquals(mem5, 1073741824L); + assertEquals(mem5, mem5b); + assertEquals(mem5, mem5c); + + // check g, m, k, b, "" acting as they should + assertEquals(mem2, mem3); + assertEquals(mem2, mem4); + assertEquals(mem2, mem5); + + // default should not be more than 90% memory or 32GB + long mem6 = MemorySetting.getMemorySetting(); + assertTrue(mem6 <= (long) (0.905 * fullmem)); + assertTrue(mem6 <= 32 * GB); + + // ensure enough memory for application + long mem7 = MemorySetting.getMemorySetting("1B", "0.000000001"); + assertEquals(mem7, 512 * MB); + + // ensure enough memory for OS + long mem8 = MemorySetting.getMemorySetting("2TB", "100"); // this should be short of 512MB + long mem8b = MemorySetting.getMemorySetting("2TB", "50"); + assertEquals(mem8b * 2 - mem8, 512 * MB); + } + +} diff --git a/utils/classgraph-4.1.6.jar b/utils/testnglibs/classgraph-4.1.6.jar similarity index 100% rename from utils/classgraph-4.1.6.jar rename to utils/testnglibs/classgraph-4.1.6.jar -- 1.7.10.2