From: Fábio Madeira Date: Fri, 2 Jun 2017 14:07:58 +0000 (+0100) Subject: JWS-121 Improved the AnnualStat and ServiceStatus classes so they start the scheduler... X-Git-Url: http://source.jalview.org/gitweb/?a=commitdiff_plain;h=1d231ce2b4de665ef21d4d5bca681e08ad2d210b;p=jabaws.git JWS-121 Improved the AnnualStat and ServiceStatus classes so they start the scheduler in the init() method and refresh the in memory cache every X minutes. The number of minutes is defined in the Engine.local.properties config. In memory values are cached in the ServletContext. --- diff --git a/conf/Engine.local.properties b/conf/Engine.local.properties index 1f54acb..39d7e63 100644 --- a/conf/Engine.local.properties +++ b/conf/Engine.local.properties @@ -36,3 +36,9 @@ local.jobdir.maxlifespan=168 # Frequency of cleaning job directory (in minutes) # normal rate: once a date = 24 * 60 = 1440 local.jobdir.cleaning.frequency=1440 + +################################################################################# +# Frequency of Service Status Memory Cache refresh (in minutes) +local.service.status.refresh.frequency=10 +# Frequency of Usage Statistics Memory Cache refresh (in minutes) +local.usage.stats.refresh.frequency=60 diff --git a/webservices/compbio/stat/servlet/AnnualStat.java b/webservices/compbio/stat/servlet/AnnualStat.java index eead7a5..bfa439c 100644 --- a/webservices/compbio/stat/servlet/AnnualStat.java +++ b/webservices/compbio/stat/servlet/AnnualStat.java @@ -19,6 +19,8 @@ package compbio.stat.servlet; import java.io.IOException; import java.io.PrintWriter; +import java.net.HttpURLConnection; +import java.net.URL; import java.sql.SQLException; import java.util.Date; import java.util.Map; @@ -31,51 +33,175 @@ import javax.servlet.http.HttpServlet; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; +import compbio.engine.conf.PropertyHelperManager; import compbio.stat.collector.StatDB; -import compbio.stat.servlet.util.StatCollection; -import compbio.stat.servlet.util.Totals; +import compbio.stat.servlet.util.*; +import compbio.util.PropertyHelper; +import compbio.util.Util; + +import org.apache.log4j.Logger; public class AnnualStat extends HttpServlet { - @Override - protected void doGet(HttpServletRequest req, HttpServletResponse resp) - throws ServletException, IOException { - - try { - long startTime = System.nanoTime(); - StatDB db = new StatDB(); - Date earliestRec = db.getEarliestRecord(); - if (earliestRec == null) { - PrintWriter writer = resp.getWriter(); - writer.println("No statistics found in the database. Please allow " - + "at least one hour after a server start for the statistics " - + "collector to collect the data. "); - writer.close(); - return; - } - Map monthlyTotals = StatCollection.getStats(earliestRec); - long endTime = System.nanoTime(); - req.setAttribute("stat", monthlyTotals); - req.setAttribute("total", Totals.sumOfTotals(monthlyTotals)); -// req.setAttribute("timeexec", (endTime - startTime) / 1000000); - req.setAttribute("timeexec", (endTime - startTime) / 100000000); - String timeStamp = new SimpleDateFormat("dd/MM/yyyy HH:mm:ss").format( - Calendar.getInstance().getTime()); - req.setAttribute("timestamp", timeStamp); - - RequestDispatcher dispatcher = req.getRequestDispatcher("statpages/MonthlySummary.jsp"); - - req.setAttribute("isAdmin", isAdmin(req)); - dispatcher.forward(req, resp); - - } catch (SQLException e) { - e.printStackTrace(); - throw new ServletException(e); - } - } - - static boolean isAdmin(final HttpServletRequest request) { - return request.isUserInRole("admin"); - } + private static final Logger log = Logger.getLogger(AnnualStat.class); + + private static PropertyHelper ph = PropertyHelperManager.getPropertyHelper(); + + private final Scheduler scheduler = new Scheduler(); + + private static int getIntProperty(String propValue) { + int value = 0; + if (!Util.isEmpty(propValue)) { + propValue = propValue.trim(); + value = Integer.parseInt(propValue); + } + return value; + } + + static int refreshUsageStatsFrequency() { + return getIntProperty(ph.getProperty("local.usage.stats.refresh.frequency")); + } + + @Override + public void init() { + // startup the scheduler with current time + int hour = Calendar.getInstance().get(Calendar.HOUR_OF_DAY); + int min = Calendar.getInstance().get(Calendar.MINUTE); + int sec = Calendar.getInstance().get(Calendar.SECOND); + // refresh sleep time (in between requests) - minutes + final int refresh_freq = refreshUsageStatsFrequency(); + + scheduler.schedule(new SchedulerTask() { + public void run() { + refreshCache(); + } + + private void refreshCache() { + hitEndpointForRefresh(); + log.info("Refreshing the In Memory Cache..."); + log.info(String.format("Refreshing again in %d minutes", refresh_freq)); + } + }, new RefreshIterator(hour, min, sec, refresh_freq)); + } + + public void clearContextCache() { + // try remove the current values from the context + try { + this.getServletConfig().getServletContext().removeAttribute("usageStatsResults"); + this.getServletConfig().getServletContext().removeAttribute("usageStatsTimestamp"); + this.getServletConfig().getServletContext().removeAttribute("usageStatsStart"); + this.getServletConfig().getServletContext().removeAttribute("usageStatsEnd"); + log.info("In Memory Cache Cleared!"); + } catch (Exception e) { + log.warn("In Memory Cache Not Cleared. Perhaps not available yet!"); + } + } + + private void hitEndpointForRefresh() { + String endpointURL; + endpointURL = (String) this.getServletConfig().getServletContext().getAttribute("usageStatsURL"); + if (endpointURL != null){ + try { + URL url = new URL(endpointURL); + System.out.println(endpointURL); + HttpURLConnection connection = (HttpURLConnection) url.openConnection(); + connection.setRequestProperty("User-Agent", "JABAWS Usage Stats Refreshing the Context Cache"); + int status = connection.getResponseCode(); + log.info(String.format("Hitting %s with an internal user-agent! status code: %s", + endpointURL, status)); + } catch (IOException e) { + log.warn("Something wrong when hitting " + endpointURL); + log.warn(e); + } + } + } + + public Map checkMonthlyTotals(HttpServletResponse resp) throws ServletException, IOException { + + try { + System.out.println("Updating the Usage Stats In Memory Cache..."); + StatDB db = new StatDB(); + Date earliestRec = db.getEarliestRecord(); + if (earliestRec == null) { + PrintWriter writer = resp.getWriter(); + writer.println("No statistics found in the database. Please allow " + + "at least one hour after a server start for the statistics " + + "collector to collect the data. "); + writer.close(); + return null; + } + return StatCollection.getStats(earliestRec); + } catch (SQLException e) { + e.printStackTrace(); + throw new ServletException(e); + } + } + + @Override + protected void doGet(HttpServletRequest req, HttpServletResponse resp) + throws ServletException, IOException { + + Map monthlyTotals; + String timeStamp = new String(); + long startTime; + long endTime; + String refresh_freq; + + // save the main caller URL in the context + String url = new String(); + url = req.getRequestURL().toString(); + System.out.println("UR: " + url); + this.getServletConfig().getServletContext().setAttribute("usageStatsURL", url); + + // check user-agent: this is a trick to get the cache refreshed periodically + String useragent = new String(); + useragent = req.getHeader("user-agent"); + // check if there are values available in the context + if (this.getServletConfig().getServletContext().getAttribute("usageStatsResults") == null || + useragent.equals("JABAWS Usage Stats Refreshing the Context Cache")) { + + // get stats from db + startTime = System.nanoTime(); + monthlyTotals = checkMonthlyTotals(resp); + endTime = System.nanoTime(); + if (monthlyTotals != null) { + // try clear the previous values if any + clearContextCache(); + // add the current values to the context + this.getServletConfig().getServletContext().setAttribute("usageStatsResults", monthlyTotals); + timeStamp = new SimpleDateFormat("dd/MM/yyyy HH:mm:ss").format( + Calendar.getInstance().getTime()); + this.getServletConfig().getServletContext().setAttribute("usageStatsTimestamp", timeStamp); + this.getServletConfig().getServletContext().setAttribute("usageStatsStart", startTime); + this.getServletConfig().getServletContext().setAttribute("usageStatsEnd", endTime); + refresh_freq = String.valueOf(refreshUsageStatsFrequency()); + this.getServletConfig().getServletContext().setAttribute("usageStatsRefreshFreq", refresh_freq); + } else { + return; + } + + } else { + // get last results available + monthlyTotals = (Map) this.getServletConfig().getServletContext().getAttribute("usageStatsResults"); + timeStamp = (String) this.getServletConfig().getServletContext().getAttribute("usageStatsTimestamp"); + startTime = (Long) this.getServletConfig().getServletContext().getAttribute("usageStatsStart"); + endTime = (Long) this.getServletConfig().getServletContext().getAttribute("usageStatsEnd"); + refresh_freq = (String) this.getServletConfig().getServletContext().getAttribute("usageStatsRefreshFreq"); + } + + req.setAttribute("stat", monthlyTotals); + req.setAttribute("total", Totals.sumOfTotals(monthlyTotals)); + req.setAttribute("timeexec", (endTime - startTime) / 100000000); + req.setAttribute("timestamp", timeStamp); + req.setAttribute("refreshfreq", refresh_freq); + RequestDispatcher dispatcher = req.getRequestDispatcher("statpages/MonthlySummary.jsp"); + req.setAttribute("isAdmin", isAdmin(req)); + dispatcher.forward(req, resp); + + } + + static boolean isAdmin(final HttpServletRequest request) { + return request.isUserInRole("admin"); + } } diff --git a/webservices/compbio/stat/servlet/ServiceStatus.java b/webservices/compbio/stat/servlet/ServiceStatus.java index 0b82be9..8b29ded 100644 --- a/webservices/compbio/stat/servlet/ServiceStatus.java +++ b/webservices/compbio/stat/servlet/ServiceStatus.java @@ -25,11 +25,8 @@ import java.io.StringWriter; import java.lang.management.ManagementFactory; import java.net.InetAddress; import java.net.UnknownHostException; -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; -import java.util.Set; -import java.util.Calendar; +import java.util.*; +import java.net.*; import java.text.SimpleDateFormat; import javax.management.AttributeNotFoundException; @@ -46,6 +43,12 @@ import javax.servlet.http.HttpServlet; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; +import compbio.engine.conf.PropertyHelperManager; +import compbio.stat.servlet.util.RefreshIterator; +import compbio.stat.servlet.util.Scheduler; +import compbio.stat.servlet.util.SchedulerTask; +import compbio.util.PropertyHelper; +import compbio.util.Util; import org.apache.log4j.Logger; import compbio.ws.client.Services; @@ -56,106 +59,242 @@ import compbio.ws.client.WSTester; *
*
  • Test web services and display results on the web page
    • *
    - * + * * @author pvtroshin - * */ public class ServiceStatus extends HttpServlet { - private final static Logger log = Logger.getLogger(ServiceStatus.class); - - private List getEndPoints() throws MalformedObjectNameException, NullPointerException, UnknownHostException, - AttributeNotFoundException, InstanceNotFoundException, MBeanException, ReflectionException { - MBeanServer mbs = ManagementFactory.getPlatformMBeanServer(); - Set objs = mbs.queryNames(new ObjectName("*:type=Connector,*"), - Query.match(Query.attr("protocol"), Query.value("HTTP/1.1"))); - List endPoints = new ArrayList(); - for (Iterator i = objs.iterator(); i.hasNext();) { - ObjectName obj = i.next(); - String scheme = mbs.getAttribute(obj, "scheme").toString(); - String port = obj.getKeyProperty("port"); - String hostname = InetAddress.getLocalHost().getHostName(); - endPoints.add(scheme + "://" + hostname + ":" + port); - } - return endPoints; - } - - @Override - protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException { - /** - * // PROBLEM: the code tries to test not WS endpoints on an internal - * Tomcat, but on an external // endpoints with wrong info on the - * proxing StringBuffer jabawspath = req.getRequestURL(); jabawspath = - * jabawspath.delete(jabawspath.lastIndexOf("/"), jabawspath.length()); - * String serverPath = jabawspath.toString(); - */ - List eps = new ArrayList(); - try { - eps = getEndPoints(); - } catch (MalformedObjectNameException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } catch (AttributeNotFoundException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } catch (InstanceNotFoundException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } catch (NullPointerException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } catch (MBeanException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } catch (ReflectionException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } - if (1 != eps.size()) { - log.info(eps.size() + "EndPoints found"); - //System.out.println(" " + eps.size() + " EndPoints found"); - for (String endpoint : eps) { - log.info(eps.size() + "EndPoint is " + endpoint); - //System.out.println(" EndPoint is " + endpoint); - } - } - - String serverPath = new String(); - for (String endpoint : eps) { - serverPath = endpoint + req.getContextPath(); - } - //System.out.println("Testing services at " + serverPath); - - List testResults = new ArrayList(); - long startTime = System.nanoTime(); - for (Services service : Services.values()) { - StringWriter testres = new StringWriter(); - PrintWriter writer = new PrintWriter(testres, true); - WSTester tester = new WSTester(serverPath, writer); - ServiceTestResult result = new ServiceTestResult(service); - try { - result.failed = tester.checkService(service); - } catch (Exception e) { - log.info(e, e.getCause()); - String mess = "Fails to connect to the web service: " + service + ". Reason: "; - writer.println(mess + e.getLocalizedMessage() + "\nDetails: "); - e.printStackTrace(writer); - } finally { - writer.close(); - } - result.details = testres.toString(); - testResults.add(result); - } - req.setAttribute("results", testResults); - long endTime = System.nanoTime(); -// req.setAttribute("timeexec", (endTime - startTime) / 1000000); - req.setAttribute("timeexec", (endTime - startTime) / 100000000); - String timeStamp = new SimpleDateFormat("dd/MM/yyyy HH:mm:ss").format( - Calendar.getInstance().getTime()); - req.setAttribute("timestamp", timeStamp); - RequestDispatcher rd = req.getRequestDispatcher("statpages/ServicesStatus.jsp"); - rd.forward(req, resp); - } + private static final Logger log = Logger.getLogger(ServiceStatus.class); + + private static PropertyHelper ph = PropertyHelperManager.getPropertyHelper(); + + private final Scheduler scheduler = new Scheduler(); + + private List getEndPoints() throws MalformedObjectNameException, NullPointerException, UnknownHostException, + AttributeNotFoundException, InstanceNotFoundException, MBeanException, ReflectionException { + MBeanServer mbs = ManagementFactory.getPlatformMBeanServer(); + Set objs = mbs.queryNames(new ObjectName("*:type=Connector,*"), + Query.match(Query.attr("protocol"), Query.value("HTTP/1.1"))); + List endPoints = new ArrayList(); + for (Iterator i = objs.iterator(); i.hasNext(); ) { + ObjectName obj = i.next(); + String scheme = mbs.getAttribute(obj, "scheme").toString(); + String port = obj.getKeyProperty("port"); + String hostname = InetAddress.getLocalHost().getHostName(); + endPoints.add(scheme + "://" + hostname + ":" + port); + } + return endPoints; + } + + private static int getIntProperty(String propValue) { + int value = 0; + if (!Util.isEmpty(propValue)) { + propValue = propValue.trim(); + value = Integer.parseInt(propValue); + } + return value; + } + + private static int refreshServiceStatusFrequency() { + return getIntProperty(ph.getProperty("local.service.status.refresh.frequency")); + } + + static int refreshUsageStatsFrequency() { + return getIntProperty(ph.getProperty("local.usage.stats.refresh.frequency")); + } + + @Override + public void init() { + // startup the scheduler with current time + int hour = Calendar.getInstance().get(Calendar.HOUR_OF_DAY); + int min = Calendar.getInstance().get(Calendar.MINUTE); + int sec = Calendar.getInstance().get(Calendar.SECOND); + // refresh sleep time (in between requests) - minutes + final int refresh_freq = refreshServiceStatusFrequency(); + + scheduler.schedule(new SchedulerTask() { + public void run() { + refreshCache(); + } + private void refreshCache() { + hitEndpointForRefresh(); + log.info("Refreshing the In Memory Cache..."); + log.info(String.format("Refreshing again in %d minutes", refresh_freq)); + } + }, new RefreshIterator(hour, min, sec, refresh_freq)); + } + + public void clearContextCache() { + // try remove the current values from the context + try { + this.getServletConfig().getServletContext().removeAttribute("serviceStatusResults"); + this.getServletConfig().getServletContext().removeAttribute("serviceStatusTimestamp"); + this.getServletConfig().getServletContext().removeAttribute("serviceStatusStart"); + this.getServletConfig().getServletContext().removeAttribute("serviceStatusEnd"); + this.getServletConfig().getServletContext().removeAttribute("serviceStatusAllGood"); + log.info("In Memory Cache Cleared!"); + } catch (Exception e) { + log.warn("In Memory Cache Not Cleared. Perhaps not available yet!"); + } + } + + private void hitEndpointForRefresh() { + String endpointURL; + endpointURL = (String) this.getServletConfig().getServletContext().getAttribute("serviceStatusURL"); + if (endpointURL != null){ + try { + URL url = new URL(endpointURL); + System.out.println(endpointURL); + HttpURLConnection connection = (HttpURLConnection) url.openConnection(); + connection.setRequestProperty("User-Agent", "JABAWS Service Status Refreshing the Context Cache"); + int status = connection.getResponseCode(); + log.info(String.format("Hitting %s with an internal user-agent! status code: %s", + endpointURL, status)); + } catch (IOException e) { + log.warn("Something wrong when hitting " + endpointURL); + log.warn(e); + } + } + } + + public ArrayList testServiceStatus(HttpServletRequest req) throws ServletException, IOException { + + ArrayList testResults = new ArrayList(); + + // run the service status checkup + List eps = new ArrayList(); + try { + eps = getEndPoints(); + } catch (MalformedObjectNameException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } catch (AttributeNotFoundException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } catch (InstanceNotFoundException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } catch (NullPointerException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } catch (MBeanException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } catch (ReflectionException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + if (1 != eps.size()) { + log.info(eps.size() + "EndPoints found"); + //System.out.println(" " + eps.size() + " EndPoints found"); + for (String endpoint : eps) { + log.info(eps.size() + "EndPoint is " + endpoint); + //System.out.println(" EndPoint is " + endpoint); + } + } + + String serverPath = new String(); + for (String endpoint : eps) { + serverPath = endpoint + req.getContextPath(); + } + + for (Services service : Services.values()) { + StringWriter testres = new StringWriter(); + PrintWriter writer = new PrintWriter(testres, true); + WSTester tester = new WSTester(serverPath, writer); + ServiceTestResult result = new ServiceTestResult(service); + try { + result.failed = tester.checkService(service); + } catch (Exception e) { + log.info(e, e.getCause()); + String mess = "Fails to connect to the web service: " + service + ". Reason: "; + writer.println(mess + e.getLocalizedMessage() + "\nDetails: "); + e.printStackTrace(writer); + } finally { + writer.close(); + } + result.details = testres.toString(); + testResults.add(result); + } + return testResults; + } + + public String overallStatusGood(ArrayList testResults) { + // assumes all good but returns as soon as some service is down + String success = "true"; + for (int i = 0; i < testResults.size(); i++) { + if (!testResults.get(i).failed) { + success = "false"; + break; + } + } + return success; + } + + @Override + protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException { + /** + * // PROBLEM: the code tries to test not WS endpoints on an internal + * Tomcat, but on an external // endpoints with wrong info on the + * proxing StringBuffer jabawspath = req.getRequestURL(); jabawspath = + * jabawspath.delete(jabawspath.lastIndexOf("/"), jabawspath.length()); + * String serverPath = jabawspath.toString(); + */ + + ArrayList testResults = new ArrayList(); + String timeStamp = new String(); + long startTime; + long endTime; + String refresh_freq; + String goodStatus; + + // save the main caller URL in the context + String url = new String(); + url = req.getRequestURL().toString(); + System.out.println("UR: " + url); + this.getServletConfig().getServletContext().setAttribute("serviceStatusURL", url); + + // check user-agent: this is a trick to get the cache refreshed periodically + String useragent = new String(); + useragent = req.getHeader("user-agent"); + // check if there are values available in the context + if (this.getServletConfig().getServletContext().getAttribute("serviceStatusResults") == null || + useragent.equals("JABAWS Service Status Refreshing the Context Cache")) { + + // test the services and timeit + startTime = System.nanoTime(); + testResults = testServiceStatus(req); + endTime = System.nanoTime(); + // try clear the previous values if any + clearContextCache(); + // add the current values to the context + this.getServletConfig().getServletContext().setAttribute("serviceStatusResults", testResults); + timeStamp = new SimpleDateFormat("dd/MM/yyyy HH:mm:ss").format( + Calendar.getInstance().getTime()); + this.getServletConfig().getServletContext().setAttribute("serviceStatusTimestamp", timeStamp); + this.getServletConfig().getServletContext().setAttribute("serviceStatusStart", startTime); + this.getServletConfig().getServletContext().setAttribute("serviceStatusEnd", endTime); + refresh_freq = String.valueOf(refreshServiceStatusFrequency()); + this.getServletConfig().getServletContext().setAttribute("serviceStatusRefreshFreq", refresh_freq); + goodStatus = overallStatusGood(testResults); + this.getServletConfig().getServletContext().setAttribute("serviceStatusAllGood", goodStatus); + + } else { + // get last results available + testResults = (ArrayList) this.getServletConfig().getServletContext().getAttribute("serviceStatusResults"); + timeStamp = (String) this.getServletConfig().getServletContext().getAttribute("serviceStatusTimestamp"); + startTime = (Long) this.getServletConfig().getServletContext().getAttribute("serviceStatusStart"); + endTime = (Long) this.getServletConfig().getServletContext().getAttribute("serviceStatusEnd"); + refresh_freq = (String) this.getServletConfig().getServletContext().getAttribute("serviceStatusRefreshFreq"); + } + + req.setAttribute("results", testResults); + req.setAttribute("timestamp", timeStamp); + req.setAttribute("timeexec", (endTime - startTime) / 100000000); + req.setAttribute("refreshfreq", refresh_freq); + RequestDispatcher rd = req.getRequestDispatcher("statpages/ServicesStatus.jsp"); + rd.forward(req, resp); + } } diff --git a/website/statpages/MonthlySummary.jsp b/website/statpages/MonthlySummary.jsp index a6cac63..01e96ea 100644 --- a/website/statpages/MonthlySummary.jsp +++ b/website/statpages/MonthlySummary.jsp @@ -44,8 +44,8 @@ Date: May 2011 ${pageContext.request.scheme}://${pageContext.request.serverName}:${pageContext.request.serverPort}${pageContext.request.contextPath}
  • Time of execution: ${timeexec} sec
    • -
  • Usage Statistics as of: ${timestamp}
    • -
  • Usage Statistics as of: ${timestamp} (Refreshes every ${refreshfreq} minutes)
    • +
  • Refresh Usage Statistics

    • diff --git a/website/statpages/ServicesStatus.jsp b/website/statpages/ServicesStatus.jsp index 97735f8..80f9425 100644 --- a/website/statpages/ServicesStatus.jsp +++ b/website/statpages/ServicesStatus.jsp @@ -46,7 +46,6 @@ TODO refactor
    -