From 987fe386182874a1d4a9b47df14db5da3f3d0023 Mon Sep 17 00:00:00 2001 From: Robin Bygrave Date: Tue, 1 Sep 2015 07:55:25 +1200 Subject: [PATCH 1/9] WIP AutoTune refactor --- .../java/com/avaje/ebean/AdminAutofetch.java | 120 +--- .../avaje/ebean/bean/NodeUsageCollector.java | 6 +- .../ebeaninternal/api/SpiEbeanServer.java | 6 - .../com/avaje/ebeaninternal/api/SpiQuery.java | 14 +- .../server/autofetch/AutoFetchManager.java | 241 ------- .../server/autofetch/AutoTuneService.java | 63 ++ .../autofetch/DefaultAutoFetchManager.java | 587 ------------------ .../server/autofetch/ProfilingListener.java | 13 + .../server/autofetch/package.html | 8 - .../AutoTuneServiceFactory.java} | 25 +- .../service/BaseAutoTuneService.java | 207 ++++++ .../autofetch/service/BaseQueryTuner.java | 150 +++++ .../DefaultAutoFetchManagerLogging.java | 6 +- .../autofetch/service/ProfileManager.java | 112 ++++ .../ProfileOrigin.java} | 119 ++-- .../ProfileOriginNodeUsage.java} | 10 +- .../ProfileOriginQuery.java} | 6 +- .../{ => service}/TunedQueryInfo.java | 2 +- .../server/core/DefaultContainer.java | 28 +- .../server/core/DefaultServer.java | 75 +-- .../server/core/InternalConfiguration.java | 8 +- .../server/jmx/MAdminAutofetch.java | 121 ---- .../server/jmx/MAdminAutofetchMBean.java | 145 ----- .../server/jmx/package-info.java | 1 - .../server/loadcontext/DLoadBeanContext.java | 2 +- .../server/loadcontext/DLoadContext.java | 8 +- .../server/loadcontext/DLoadManyContext.java | 2 +- .../ebeaninternal/server/query/CQuery.java | 24 +- .../server/querydefn/DefaultOrmQuery.java | 35 +- .../ebeaninternal/api/TDSpiEbeanServer.java | 6 - .../server/autofetch/TunedQueryInfoTest.java | 2 +- .../autofetch/AutofetchEmbeddedTest.java | 48 +- .../cache/TestL2CacheWithSharedBean.java | 2 +- .../query/TestAutofetchTuneWithJoin.java | 7 +- src/test/resources/ebean.properties | 7 +- src/test/resources/logback-test.xml | 2 + 36 files changed, 740 insertions(+), 1478 deletions(-) delete mode 100644 src/main/java/com/avaje/ebeaninternal/server/autofetch/AutoFetchManager.java create mode 100644 src/main/java/com/avaje/ebeaninternal/server/autofetch/AutoTuneService.java delete mode 100644 src/main/java/com/avaje/ebeaninternal/server/autofetch/DefaultAutoFetchManager.java create mode 100644 src/main/java/com/avaje/ebeaninternal/server/autofetch/ProfilingListener.java delete mode 100644 src/main/java/com/avaje/ebeaninternal/server/autofetch/package.html rename src/main/java/com/avaje/ebeaninternal/server/autofetch/{AutoFetchManagerFactory.java => service/AutoTuneServiceFactory.java} (69%) create mode 100644 src/main/java/com/avaje/ebeaninternal/server/autofetch/service/BaseAutoTuneService.java create mode 100644 src/main/java/com/avaje/ebeaninternal/server/autofetch/service/BaseQueryTuner.java rename src/main/java/com/avaje/ebeaninternal/server/autofetch/{ => service}/DefaultAutoFetchManagerLogging.java (92%) create mode 100644 src/main/java/com/avaje/ebeaninternal/server/autofetch/service/ProfileManager.java rename src/main/java/com/avaje/ebeaninternal/server/autofetch/{Statistics.java => service/ProfileOrigin.java} (52%) rename src/main/java/com/avaje/ebeaninternal/server/autofetch/{StatisticsNodeUsage.java => service/ProfileOriginNodeUsage.java} (91%) rename src/main/java/com/avaje/ebeaninternal/server/autofetch/{StatisticsQuery.java => service/ProfileOriginQuery.java} (81%) rename src/main/java/com/avaje/ebeaninternal/server/autofetch/{ => service}/TunedQueryInfo.java (98%) delete mode 100644 src/main/java/com/avaje/ebeaninternal/server/jmx/MAdminAutofetch.java delete mode 100644 src/main/java/com/avaje/ebeaninternal/server/jmx/MAdminAutofetchMBean.java delete mode 100644 src/main/java/com/avaje/ebeaninternal/server/jmx/package-info.java diff --git a/src/main/java/com/avaje/ebean/AdminAutofetch.java b/src/main/java/com/avaje/ebean/AdminAutofetch.java index 15e19f9c1..49ab9287e 100644 --- a/src/main/java/com/avaje/ebean/AdminAutofetch.java +++ b/src/main/java/com/avaje/ebean/AdminAutofetch.java @@ -5,80 +5,11 @@ package com.avaje.ebean; */ public interface AdminAutofetch { - /** - * Return true if profiling is enabled. - */ - boolean isProfiling(); - - /** - * Set to true to enable profiling. - */ - void setProfiling(boolean enable); - - /** - * Return true if autoFetch automatic query tuning is enabled. - */ - boolean isQueryTuning(); - - /** - * Set to true to enable autoFetch automatic query tuning. - */ - void setQueryTuning(boolean enable); - - /** - * Returns the rate which profiling is collected. This is an int between 0 and - * 100. - */ - double getProfilingRate(); - - /** - * Set the rate at which profiling is collected after the base. - * - * @param rate - * a int between 0 and 100. - */ - void setProfilingRate(double rate); - - /** - * Return the number of queries profiled after which profiling is collected at - * a percentage rate. - */ - int getProfilingBase(); - - /** - * Set a base number of queries to profile per query point. - *

- * After this amount of profiling has been obtained profiling is collected at - * the Profiling Percentage rate. - *

- */ - void setProfilingBase(int profilingBase); - - /** - * Return the minimum number of queries profiled before autoFetch will start - * automatically tuning the queries. - *

- * This could be one which means start autoFetch tuning after the first - * profiling information is collected. - *

- */ - int getProfilingMin(); - - /** - * Set the minimum number of queries profiled per query point before autoFetch - * will automatically tune the queries. - *

- * Increasing this number will mean more profiling is collected before - * autoFetch starts tuning the query. - *

- */ - void setProfilingMin(int autoFetchMinThreshold); - /** * Fire a garbage collection (hint to the JVM). Assuming garbage collection * fires this will gather the usage profiling information. */ - String collectUsageViaGC(); + void collectUsageViaGC(); /** * This will take the current profiling information and update the "tuned @@ -86,52 +17,7 @@ public interface AdminAutofetch { *

* This is done periodically and can also be manually invoked. *

- * - * @return a summary of the updates that occurred */ - String updateTunedQueryInfo(); + void updateTunedQueryInfo(); - /** - * Clear all the tuned query info. - *

- * Should only need do this for testing and playing around. - *

- * - * @return the amount of tuned query information cleared. - */ - int clearTunedQueryInfo(); - - /** - * Clear all the profiling information. - *

- * This means the profiling information will need to be re-gathered. - *

- *

- * Should only need do this for testing and playing around. - *

- * - * @return the amount of profiled information cleared. - */ - int clearProfilingInfo(); - - /** - * Clear the query execution statistics. - */ - void clearQueryStatistics(); - - /** - * Return the number of queries tuned by AutoFetch. - */ - int getTotalTunedQueryCount(); - - /** - * Return the size of the TuneQuery map. - */ - int getTotalTunedQuerySize(); - - /** - * Return the size of the profile map. - */ - int getTotalProfileSize(); - -} \ No newline at end of file +} diff --git a/src/main/java/com/avaje/ebean/bean/NodeUsageCollector.java b/src/main/java/com/avaje/ebean/bean/NodeUsageCollector.java index 42dc52c8c..00c1f7d0f 100644 --- a/src/main/java/com/avaje/ebean/bean/NodeUsageCollector.java +++ b/src/main/java/com/avaje/ebean/bean/NodeUsageCollector.java @@ -2,6 +2,8 @@ package com.avaje.ebean.bean; import java.lang.ref.WeakReference; import java.util.HashSet; +import java.util.LinkedHashSet; +import java.util.Set; /** * Collects profile information for a bean (or reference/proxy bean) at a given @@ -29,7 +31,7 @@ public final class NodeUsageCollector { /** * The properties used at this profile point. */ - private final HashSet used = new HashSet(); + private final Set used = new LinkedHashSet(); /** * set to true if the bean is modified (setter called) @@ -105,7 +107,7 @@ public final class NodeUsageCollector { /** * Return the set of used properties. */ - public HashSet getUsed() { + public Set getUsed() { return used; } diff --git a/src/main/java/com/avaje/ebeaninternal/api/SpiEbeanServer.java b/src/main/java/com/avaje/ebeaninternal/api/SpiEbeanServer.java index 290a6e4fa..31dbc82de 100644 --- a/src/main/java/com/avaje/ebeaninternal/api/SpiEbeanServer.java +++ b/src/main/java/com/avaje/ebeaninternal/api/SpiEbeanServer.java @@ -11,7 +11,6 @@ import com.avaje.ebean.bean.CallStack; import com.avaje.ebean.bean.ObjectGraphNode; import com.avaje.ebean.config.ServerConfig; import com.avaje.ebean.config.dbplatform.DatabasePlatform; -import com.avaje.ebeaninternal.server.autofetch.AutoFetchManager; import com.avaje.ebeaninternal.server.core.SpiOrmQueryRequest; import com.avaje.ebean.dbmigration.DdlGenerator; import com.avaje.ebeaninternal.server.deploy.BeanDescriptor; @@ -70,11 +69,6 @@ public interface SpiEbeanServer extends EbeanServer, BeanLoader, BeanCollectionL */ DdlGenerator getDdlGenerator(); - /** - * Return the AutoFetchListener. - */ - AutoFetchManager getAutoFetchManager(); - /** * Clear the query execution statistics. */ diff --git a/src/main/java/com/avaje/ebeaninternal/api/SpiQuery.java b/src/main/java/com/avaje/ebeaninternal/api/SpiQuery.java index 1f496d319..ebb5a49ae 100644 --- a/src/main/java/com/avaje/ebeaninternal/api/SpiQuery.java +++ b/src/main/java/com/avaje/ebeaninternal/api/SpiQuery.java @@ -10,7 +10,7 @@ import com.avaje.ebean.bean.CallStack; import com.avaje.ebean.bean.ObjectGraphNode; import com.avaje.ebean.bean.PersistenceContext; import com.avaje.ebean.event.BeanQueryRequest; -import com.avaje.ebeaninternal.server.autofetch.AutoFetchManager; +import com.avaje.ebeaninternal.server.autofetch.ProfilingListener; import com.avaje.ebeaninternal.server.deploy.BeanDescriptor; import com.avaje.ebeaninternal.server.deploy.BeanPropertyAssocMany; import com.avaje.ebeaninternal.server.deploy.TableJoin; @@ -370,16 +370,16 @@ public interface SpiQuery extends Query { void logSecondaryQuery(SpiQuery query); /** - * If return null then no autoFetch profiling for this query. If a - * AutoFetchManager is returned this implies that profiling is turned on for - * this query (and all the objects this query creates). + * If return null then no profiling for this query. If a ProfilingListener is + * returned this implies that profiling is turned on for this query (and all + * the objects this query creates). */ - AutoFetchManager getAutoFetchManager(); + ProfilingListener getProfilingListener(); /** - * This has the effect of turning on autoFetch profiling for this query. + * This has the effect of turning on profiling for this query. */ - void setAutoFetchManager(AutoFetchManager manager); + void setProfilingListener(ProfilingListener manager); /** * Return the origin point for the query. diff --git a/src/main/java/com/avaje/ebeaninternal/server/autofetch/AutoFetchManager.java b/src/main/java/com/avaje/ebeaninternal/server/autofetch/AutoFetchManager.java deleted file mode 100644 index 7073bf30d..000000000 --- a/src/main/java/com/avaje/ebeaninternal/server/autofetch/AutoFetchManager.java +++ /dev/null @@ -1,241 +0,0 @@ -package com.avaje.ebeaninternal.server.autofetch; - -import com.avaje.ebean.Query; -import com.avaje.ebean.bean.NodeUsageListener; -import com.avaje.ebean.bean.ObjectGraphNode; -import com.avaje.ebean.config.AutofetchMode; -import com.avaje.ebean.config.ServerConfig; -import com.avaje.ebeaninternal.api.SpiEbeanServer; -import com.avaje.ebeaninternal.api.SpiQuery; - -import java.util.Iterator; - -/** - * Collects and manages the the profile information. - *

- * The profile information is periodically converted into "tuned query details" - - * which is used to automatically tune the queries that use autoFetch. - *

- *

- * The "tuned query details" effectively are part of the query that has the - * select() and join() information (but not the where clause, order by, limits - * etc). These are applied to the query when tuneQuery() is called. - *

- */ -public interface AutoFetchManager extends NodeUsageListener { - - /** - * Set the owning ebean server. - */ - void setOwner(SpiEbeanServer server, ServerConfig serverConfig); - - /** - * Clear the query execution statistics. - */ - void clearQueryStatistics(); - - /** - * Clear all the tuned query info. - *

- * Should only need do this for testing and playing around. - *

- */ - int clearTunedQueryInfo(); - - /** - * Clear all the profiling information. - *

- * This means the profiling information will need to be re-gathered. - *

- *

- * Should only need do this for testing and playing around. - *

- */ - int clearProfilingInfo(); - - /** - * On shutdown fire garbage collection and collect statistics. Note that - * usually we add a little delay (100 milliseconds) to give the garbage - * collector plenty of time to do its thing and collect the profile - * information. - */ - void shutdown(); - - /** - * Return the current tuned fetch information for a given queryPoint key. - */ - TunedQueryInfo getTunedQueryInfo(String queryPointKey); - - /** - * Return the current Statistics for a given queryPoint key. - */ - Statistics getStatistics(String queryPointKey); - - /** - * Iterate the tuned fetch info. - *

- * This should be a read only iteration. - *

- */ - Iterator iterateTunedQueryInfo(); - - /** - * Iterate the node usage statistics. - *

- * This should be a read only iteration. - *

- */ - Iterator iterateStatistics(); - - /** - * Return true if profiling is enabled. - */ - boolean isProfiling(); - - /** - * Set to true to enable profiling. - *

- * We rely on garbage collection to collect the profiling information. This - * means there is a unknown delay between when a query is executed and when - * we actually collect the usage profile information. - *

- *

- * Due to this garbage collection delay, when turning off profiling while - * the application is running you should consider calling - * collectUsageViaGC() BEFORE setProfiling(false). This hints to - * the JVM to perform garbage collection, and hopefully collects the - * profiling information. - *

- */ - void setProfiling(boolean enable); - - /** - * Return true if automatic query tuning is enabled. - */ - boolean isQueryTuning(); - - /** - * Set to true to enable automatic query tuning. - */ - void setQueryTuning(boolean enable); - - /** - * This controls whether autoFetch is used when it has not been explicitly - * set on a query via {@link Query#setAutofetch(boolean)}. - */ - AutofetchMode getMode(); - - /** - * Set the auto fetch mode used when a query has not had - * {@link Query#setAutofetch(boolean)}. - */ - void setMode(AutofetchMode Mode); - - /** - * Return the profiling rate (int between 0 and 100). - */ - double getProfilingRate(); - - /** - * Set the profiling rate (int between 0 and 100). - */ - void setProfilingRate(double rate); - - /** - * Return the max number of queries profiled (per query point). - *

- * The number of queries profiled is collected per query point. Once a query - * point has profiled this number of queries it does not profile any more. - *

- */ - int getProfilingBase(); - - /** - * Set a max number of queries to profile per query point. - *

- * This number should provide a level of confidence that no more profiling - * is required for this query point. - *

- */ - void setProfilingBase(int profilingMax); - - /** - * Return the minimum number of queries profiled before autoFetch will start - * automatically tuning the queries. - *

- * This could be one which means start autoFetch tuning after the first - * profiling information is collected. - *

- */ - int getProfilingMin(); - - /** - * Set the minimum number of queries profiled per query point before - * autoFetch will automatically tune the queries. - *

- * Increasing this number will mean more profiling is collected before - * autoFetch starts tuning the query. - *

- */ - void setProfilingMin(int autoFetchMinThreshold); - - /** - * Fire a garbage collection (hint to the JVM). Assuming garbage collection - * fires this will gather the usage profiling information. - */ - String collectUsageViaGC(long waitMillis); - - /** - * This will take the current profiling information and update the "tuned - * query detail". - *

- * This is done periodically and can also be manually invoked. - *

- *

- * This returns a string summary of the updates that occurred. - *

- */ - String updateTunedQueryInfo(); - - /** - * Called when a query thinks it should be automatically tuned by autoFetch. - *

- * This internally checks that autoFetch is enabled, there is a "tuned query - * detail" to tune the query with and that the autoFetchMinThreshold has - * been reached. - *

- *

- * This will also determine if the query should be profiled. - *

- */ - boolean tuneQuery(SpiQuery query); - - /** - * Collect query profiling information. - *

- * This is for the original query as well as any subsequent lazy loading - * queries that are required as the object graph is traversed. - *

- * - * @param node the node path in the object graph. - * @param beans the number of beans loaded by the query. - * @param micros the query executing time in microseconds - */ - void collectQueryInfo(ObjectGraphNode node, long beans, long micros); - - - /** - * Return the number of queries tuned by AutoFetch. - */ - int getTotalTunedQueryCount(); - - /** - * Return the size of the TuneQuery map. - */ - int getTotalTunedQuerySize(); - - /** - * Return the size of the profile map. - */ - int getTotalProfileSize(); -} diff --git a/src/main/java/com/avaje/ebeaninternal/server/autofetch/AutoTuneService.java b/src/main/java/com/avaje/ebeaninternal/server/autofetch/AutoTuneService.java new file mode 100644 index 000000000..291cfe9f3 --- /dev/null +++ b/src/main/java/com/avaje/ebeaninternal/server/autofetch/AutoTuneService.java @@ -0,0 +1,63 @@ +package com.avaje.ebeaninternal.server.autofetch; + +import com.avaje.ebean.AdminAutofetch; +import com.avaje.ebean.config.ServerConfig; +import com.avaje.ebeaninternal.api.SpiEbeanServer; +import com.avaje.ebeaninternal.api.SpiQuery; + +/** + * Collects and manages the the profile information. + *

+ * The profile information is periodically converted into "tuned query details" - + * which is used to automatically tune the queries that use autoFetch. + *

+ *

+ * The "tuned query details" effectively are part of the query that has the + * select() and join() information (but not the where clause, order by, limits + * etc). These are applied to the query when tuneQuery() is called. + *

+ */ +public interface AutoTuneService extends AdminAutofetch { + + /** + * Load the query tuning information. + */ + void startup(); + + /** + * Called when a query thinks it should be automatically tuned by autoFetch. + *

+ * This internally checks that autoFetch is enabled, there is a "tuned query + * detail" to tune the query with and that the autoFetchMinThreshold has + * been reached. + *

+ *

+ * This will also determine if the query should be profiled. + *

+ */ + boolean tuneQuery(SpiQuery query); + + /** + * Fire a garbage collection (hint to the JVM). Assuming garbage collection + * fires this will gather the usage profiling information. + */ + void collectUsageViaGC(); + + /** + * This will take the current profiling information and update the "tuned + * query detail". + *

+ * This is done periodically and can also be manually invoked. + *

+ */ + void updateTunedQueryInfo(); + + /** + * On shutdown fire garbage collection and collect statistics. Note that + * usually we add a little delay (100 milliseconds) to give the garbage + * collector plenty of time to do its thing and collect the profile + * information. + */ + void shutdown(); + +} diff --git a/src/main/java/com/avaje/ebeaninternal/server/autofetch/DefaultAutoFetchManager.java b/src/main/java/com/avaje/ebeaninternal/server/autofetch/DefaultAutoFetchManager.java deleted file mode 100644 index 35dd210ba..000000000 --- a/src/main/java/com/avaje/ebeaninternal/server/autofetch/DefaultAutoFetchManager.java +++ /dev/null @@ -1,587 +0,0 @@ -package com.avaje.ebeaninternal.server.autofetch; - -import com.avaje.ebean.bean.CallStack; -import com.avaje.ebean.bean.NodeUsageCollector; -import com.avaje.ebean.bean.ObjectGraphNode; -import com.avaje.ebean.bean.ObjectGraphOrigin; -import com.avaje.ebean.config.AutofetchConfig; -import com.avaje.ebean.config.AutofetchMode; -import com.avaje.ebean.config.ServerConfig; -import com.avaje.ebeaninternal.api.ClassUtil; -import com.avaje.ebeaninternal.api.SpiEbeanServer; -import com.avaje.ebeaninternal.api.SpiQuery; -import com.avaje.ebeaninternal.server.deploy.BeanDescriptor; -import com.avaje.ebeaninternal.server.querydefn.OrmQueryDetail; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import javax.persistence.PersistenceException; -import java.io.File; -import java.io.FileOutputStream; -import java.io.ObjectOutputStream; -import java.io.Serializable; -import java.util.Iterator; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; - -/** - * The manager of all the usage/query statistics as well as the tuned fetch - * information. - */ -public class DefaultAutoFetchManager implements AutoFetchManager, Serializable { - - private static final Logger logger = LoggerFactory.getLogger(DefaultAutoFetchManager.class); - - private static final long serialVersionUID = -6826119882781771722L; - - @SuppressWarnings("RedundantStringConstructorCall") - private final String statisticsMonitor = new String(); - - private final String fileName; - - /** - * Map of the usage and query statistics gathered. - */ - private final Map statisticsMap = new ConcurrentHashMap(); - - /** - * Map of the tuned query details per profile query point. - */ - private final Map tunedQueryInfoMap = new ConcurrentHashMap(); - - private transient long defaultGarbageCollectionWait = 100; - - /** - * Left without synchronized for now. - */ - private transient int tunedQueryCount; - - /** - * Converted from a 0-100 int to a double. Effectively a percentage rate at - * which to collect profiling information. - */ - private transient double profilingRate = 0.1d; - - private transient int profilingBase = 10; - - private transient int profilingMin = 1; - - private transient boolean profiling; - - private transient boolean queryTuning; - - private transient boolean queryTuningAddVersion; - - private transient boolean garbageCollectionOnShutdown; - - private transient AutofetchMode mode; - - /** - * Server that owns this Profile Listener. - */ - private transient SpiEbeanServer server; - - /** - * The logger. - */ - private transient DefaultAutoFetchManagerLogging logging; - - public DefaultAutoFetchManager(String fileName) { - this.fileName = fileName; - } - - /** - * Set up this profile listener before it is active. - */ - public void setOwner(SpiEbeanServer server, ServerConfig serverConfig) { - this.server = server; - this.logging = new DefaultAutoFetchManagerLogging(serverConfig, this); - - AutofetchConfig autofetchConfig = serverConfig.getAutofetchConfig(); - - garbageCollectionOnShutdown = autofetchConfig.isGarbageCollectionOnShutdown(); - queryTuning = autofetchConfig.isQueryTuning(); - queryTuningAddVersion = autofetchConfig.isQueryTuningAddVersion(); - profiling = autofetchConfig.isProfiling(); - profilingMin = autofetchConfig.getProfilingMin(); - profilingBase = autofetchConfig.getProfilingBase(); - - setProfilingRate(autofetchConfig.getProfilingRate()); - - defaultGarbageCollectionWait = (long) autofetchConfig.getGarbageCollectionWait(); - - // determine the mode to use when Query.setAutoFetch() was - // not explicitly set - mode = autofetchConfig.getMode(); - - if (profiling || queryTuning) { - // log the guts of the autoFetch setup - String msg = "AutoFetch queryTuning[" + queryTuning + "] profiling[" + profiling - + "] mode[" + mode + "] profiling rate[" + profilingRate - + "] min[" + profilingMin + "] base[" + profilingBase + "]"; - logging.logInfo(msg, null); - - // Register a periodic update of the profiling informations - this.logging.init(server); - } - } - - - public void clearQueryStatistics() { - server.clearQueryStatistics(); - } - - /** - * Return the number of queries tuned by AutoFetch. - */ - public int getTotalTunedQueryCount() { - return tunedQueryCount; - } - - /** - * Return the size of the TuneQuery map. - */ - public int getTotalTunedQuerySize() { - return tunedQueryInfoMap.size(); - } - - /** - * Return the size of the profile map. - */ - public int getTotalProfileSize() { - return statisticsMap.size(); - } - - public int clearTunedQueryInfo() { - - // reset the rough count as well - tunedQueryCount = 0; - - // clear the map... - int size = tunedQueryInfoMap.size(); - tunedQueryInfoMap.clear(); - return size; - } - - public int clearProfilingInfo() { - int size = statisticsMap.size(); - statisticsMap.clear(); - return size; - } - - - public void serialize() { - - File autoFetchFile = new File(fileName); - - try { - FileOutputStream fout = new FileOutputStream(autoFetchFile); - - ObjectOutputStream oout = new ObjectOutputStream(fout); - oout.writeObject(this); - oout.flush(); - oout.close(); - - } catch (Exception e) { - String msg = "Error serializing autofetch file"; - logging.logError(msg, e); - } - } - - /** - * Return the current Tuned query info for a given origin key. - */ - public TunedQueryInfo getTunedQueryInfo(String originKey) { - return tunedQueryInfoMap.get(originKey); - } - - /** - * Return the current Statistics for a given originKey key. - */ - public Statistics getStatistics(String originKey) { - return statisticsMap.get(originKey); - } - - public Iterator iterateTunedQueryInfo() { - return tunedQueryInfoMap.values().iterator(); - } - - public Iterator iterateStatistics() { - return statisticsMap.values().iterator(); - } - - public boolean isProfiling() { - return profiling; - } - - /** - * When the application is running, BEFORE turning off profiling you - * probably should call collectUsageViaGC() as there is a delay (waiting for - * garbage collection) collecting usage profiling information. - */ - public void setProfiling(boolean profiling) { - this.profiling = profiling; - } - - public boolean isQueryTuning() { - return queryTuning; - } - - public void setQueryTuning(boolean queryTuning) { - this.queryTuning = queryTuning; - } - - public double getProfilingRate() { - return profilingRate; - } - - public AutofetchMode getMode() { - return mode; - } - - public void setMode(AutofetchMode mode) { - this.mode = mode; - } - - public void setProfilingRate(double rate) { - if (rate < 0) { - rate = 0d; - } else if (rate > 1) { - rate = 1d; - } - profilingRate = rate; - } - - public int getProfilingBase() { - return profilingBase; - } - - public void setProfilingBase(int profilingBase) { - this.profilingBase = profilingBase; - } - - public int getProfilingMin() { - return profilingMin; - } - - public void setProfilingMin(int profilingMin) { - this.profilingMin = profilingMin; - } - - /** - * Shutdown the listener. - *

- * We should try to collect the usage statistics by calling a System.gc(). - * This is necessary for use with short lived applications where garbage - * collection may not otherwise occur at all. - *

- */ - public void shutdown() { - if (garbageCollectionOnShutdown) { - collectUsageViaGC(-1); - serialize(); - } - } - - /** - * Ask for a System.gc() so that we gather node usage information. - *

- * Really only want to do this sparingly but useful just prior to shutdown - * for short run application where garbage collection may otherwise not - * occur at all. - *

- *

- * waitMillis will do a thread sleep to give the garbage collection a little - * time to do its thing assuming we are shutting down the VM. - *

- *

- * If waitMillis is -1 then the defaultGarbageCollectionWait is used which - * defaults to 100 milliseconds. - *

- */ - public String collectUsageViaGC(long waitMillis) { - System.gc(); - try { - if (waitMillis < 0) { - waitMillis = defaultGarbageCollectionWait; - } - Thread.sleep(waitMillis); - } catch (InterruptedException e) { - String msg = "Error while sleeping after System.gc() request."; - logging.logError(msg, e); - return msg; - } - return updateTunedQueryInfo(); - } - - /** - * Update the tuned fetch plans from the current usage information. - */ - public String updateTunedQueryInfo() { - - if (!profiling) { - // we are not collecting any profiling information at - // the moment so don't try updating the tuned query plans. - return "Not profiling"; - } - - synchronized (statisticsMonitor) { - - Counters counters = new Counters(); - - for (Statistics queryPointStatistics : statisticsMap.values()) { - if (!queryPointStatistics.hasUsage()) { - // no usage statistics collected yet... - counters.incrementNoUsage(); - } else { - updateTunedQueryFromUsage(counters, queryPointStatistics); - } - } - - String summaryInfo = counters.toString(); - - if (counters.isInteresting()) { - // only log it if its interesting - logging.logSummary(summaryInfo); - } - - return summaryInfo; - } - } - - private static class Counters { - - int newPlan; - int modified; - int unchanged; - int noUsage; - - void incrementNoUsage() { - noUsage++; - } - - void incrementNew() { - newPlan++; - } - - void incrementModified() { - modified++; - } - - void incrementUnchanged() { - unchanged++; - } - - boolean isInteresting() { - return newPlan > 0 || modified > 0; - } - - public String toString() { - return "new[" + newPlan + "] modified[" + modified + "] unchanged[" + unchanged + "] nousage[" + noUsage + "]"; - } - } - - private void updateTunedQueryFromUsage(Counters counters, Statistics statistics) { - - ObjectGraphOrigin queryPoint = statistics.getOrigin(); - String beanType = queryPoint.getBeanType(); - - try { - Class beanClass = ClassUtil.forName(beanType, this.getClass()); - BeanDescriptor beanDescriptor = server.getBeanDescriptor(beanClass); - if (beanDescriptor != null) { - - // Determine the fetch plan from the latest statistics. - // Use this to compare with current "tuned fetch plan". - OrmQueryDetail newFetchDetail = statistics.buildTunedFetch(beanDescriptor); - - // get the current tuned fetch info... - TunedQueryInfo currentFetch = tunedQueryInfoMap.get(queryPoint.getKey()); - - if (currentFetch == null) { - // its a new fetch plan, add it. - counters.incrementNew(); - - currentFetch = statistics.createTunedFetch(newFetchDetail); - logging.logNew(currentFetch); - tunedQueryInfoMap.put(queryPoint.getKey(), currentFetch); - - } else if (!currentFetch.isSame(newFetchDetail)) { - // the fetch plan has changed, update it. - counters.incrementModified(); - - logging.logChanged(currentFetch, newFetchDetail); - currentFetch.setTunedDetail(newFetchDetail); - - } else { - // the fetch plan has not changed... - counters.incrementUnchanged(); - } - - currentFetch.setProfileCount(statistics.getCounter()); - } - - } catch (ClassNotFoundException e) { - // expected after renaming/moving an entity bean - String msg = e.toString() + " updating autoFetch tuned query for " + beanType - + ". It isLikely this bean has been renamed or moved"; - logging.logInfo(msg, null); - statisticsMap.remove(statistics.getOrigin().getKey()); - } - } - - /** - * Return true if we should try to use autoFetch for this query. - */ - private boolean useAutoFetch(SpiQuery query) { - - if (query.isLoadBeanCache()) { - // when loading the cache don't tune the query - // as we want full objects loaded into the cache - return false; - } - - Boolean autoFetch = query.isAutofetch(); - if (autoFetch != null) { - // explicitly set... - return autoFetch; - - } else { - // determine using implicit mode... - switch (mode) { - case DEFAULT_ON: - return true; - - case DEFAULT_OFF: - return false; - - case DEFAULT_ONIFEMPTY: - return query.isDetailEmpty(); - - default: - throw new PersistenceException("Invalid autoFetchMode " + mode); - } - } - } - - /** - * Auto tune the query and enable profiling. - */ - public boolean tuneQuery(SpiQuery query) { - - if (!queryTuning && !profiling) { - return false; - } - - if (!useAutoFetch(query)) { - // not using autoFetch for this query - return false; - } - - ObjectGraphNode parentAutoFetchNode = query.getParentNode(); - if (parentAutoFetchNode != null) { - // This is a +lazy/+query query with profiling on. - // We continue to collect the profiling information. - query.setAutoFetchManager(this); - return true; - } - - // create a query point to identify the query - CallStack stack = server.createCallStack(); - ObjectGraphNode origin = query.setOrigin(stack); - - // get current "tuned fetch" for this query point - TunedQueryInfo tunedFetch = tunedQueryInfoMap.get(origin.getOriginQueryPoint().getKey()); - - // get the number of times we have collected profiling information - int profileCount = tunedFetch == null ? 0 : tunedFetch.getProfileCount(); - - if (profiling) { - // we want more profiling information? - if (tunedFetch == null) { - query.setAutoFetchManager(this); - - } else if (profileCount < profilingBase) { - query.setAutoFetchManager(this); - - } else if (tunedFetch.isPercentageProfile(profilingRate)) { - query.setAutoFetchManager(this); - } - } - - if (queryTuning) { - if (tunedFetch != null && profileCount >= profilingMin) { - // deemed to have enough profiling - // information for automatic tuning - if (tunedFetch.autoFetchTune(query)) { - // tunedQueryCount++ not thread-safe, could use AtomicInteger. - // But I'm happy if this statistic is a little wrong - // and this is a VERY HOT method - tunedQueryCount++; - } - return true; - } - } - - return false; - } - - /** - * Gather query execution statistics. This could either be the originating - * query in which case the parentNode will be null, or a lazy loading query - * resulting from traversal of the object graph. - */ - public void collectQueryInfo(ObjectGraphNode node, long beans, long micros) { - - if (node != null) { - ObjectGraphOrigin origin = node.getOriginQueryPoint(); - if (origin != null) { - Statistics stats = getQueryPointStats(origin); - stats.collectQueryInfo(node, beans, micros); - } - } - } - - /** - * Collect usage statistics from a node in the object graph. - *

- * This is sent to use from a EntityBeanIntercept when the finalise method - * is called on the bean. - *

- */ - public void collectNodeUsage(NodeUsageCollector usageCollector) { - - ObjectGraphOrigin origin = usageCollector.getNode().getOriginQueryPoint(); - - Statistics stats = getQueryPointStats(origin); - - if (logger.isTraceEnabled()) { - logger.trace("... NodeUsageCollector " + usageCollector); - } - - stats.collectUsageInfo(usageCollector); - - if (logger.isTraceEnabled()) { - logger.trace("stats\n" + stats); - } - } - - private Statistics getQueryPointStats(ObjectGraphOrigin originQueryPoint) { - synchronized (statisticsMonitor) { - Statistics stats = statisticsMap.get(originQueryPoint.getKey()); - if (stats == null) { - stats = new Statistics(originQueryPoint, queryTuningAddVersion); - statisticsMap.put(originQueryPoint.getKey(), stats); - } - return stats; - } - } - - public String toString() { - synchronized (statisticsMonitor) { - return statisticsMap.values().toString(); - } - } - - -} diff --git a/src/main/java/com/avaje/ebeaninternal/server/autofetch/ProfilingListener.java b/src/main/java/com/avaje/ebeaninternal/server/autofetch/ProfilingListener.java new file mode 100644 index 000000000..9885f7435 --- /dev/null +++ b/src/main/java/com/avaje/ebeaninternal/server/autofetch/ProfilingListener.java @@ -0,0 +1,13 @@ +package com.avaje.ebeaninternal.server.autofetch; + +import com.avaje.ebean.bean.NodeUsageListener; +import com.avaje.ebean.bean.ObjectGraphNode; + +/** + * Profiling listener gets call backs for node usage and the associated query executions. + */ +public interface ProfilingListener extends NodeUsageListener { + + void collectQueryInfo(ObjectGraphNode node, long beans, long micros); + +} diff --git a/src/main/java/com/avaje/ebeaninternal/server/autofetch/package.html b/src/main/java/com/avaje/ebeaninternal/server/autofetch/package.html deleted file mode 100644 index f0e805a5c..000000000 --- a/src/main/java/com/avaje/ebeaninternal/server/autofetch/package.html +++ /dev/null @@ -1,8 +0,0 @@ - - - AutoFetch Implementation - - -AutoFetch Implementation - - \ No newline at end of file diff --git a/src/main/java/com/avaje/ebeaninternal/server/autofetch/AutoFetchManagerFactory.java b/src/main/java/com/avaje/ebeaninternal/server/autofetch/service/AutoTuneServiceFactory.java similarity index 69% rename from src/main/java/com/avaje/ebeaninternal/server/autofetch/AutoFetchManagerFactory.java rename to src/main/java/com/avaje/ebeaninternal/server/autofetch/service/AutoTuneServiceFactory.java index b19a4ee37..1e6c2d1cd 100644 --- a/src/main/java/com/avaje/ebeaninternal/server/autofetch/AutoFetchManagerFactory.java +++ b/src/main/java/com/avaje/ebeaninternal/server/autofetch/service/AutoTuneServiceFactory.java @@ -1,7 +1,8 @@ -package com.avaje.ebeaninternal.server.autofetch; +package com.avaje.ebeaninternal.server.autofetch.service; import com.avaje.ebean.config.ServerConfig; import com.avaje.ebeaninternal.api.SpiEbeanServer; +import com.avaje.ebeaninternal.server.autofetch.AutoTuneService; import com.avaje.ebeaninternal.server.resource.ResourceManager; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -11,29 +12,29 @@ import java.io.File; import java.io.FileInputStream; import java.io.ObjectInputStream; -public class AutoFetchManagerFactory { +public class AutoTuneServiceFactory { - private static final Logger logger = LoggerFactory.getLogger(AutoFetchManagerFactory.class); + private static final Logger logger = LoggerFactory.getLogger(AutoTuneServiceFactory.class); - public static AutoFetchManager create(SpiEbeanServer server, ServerConfig serverConfig, ResourceManager resourceManager) { + public static AutoTuneService create(SpiEbeanServer server, ServerConfig serverConfig, ResourceManager resourceManager) { - AutoFetchManagerFactory me = new AutoFetchManagerFactory(); + AutoTuneServiceFactory me = new AutoTuneServiceFactory(); return me.createAutoFetchManager(server, serverConfig, resourceManager); } - private AutoFetchManager createAutoFetchManager(SpiEbeanServer server, ServerConfig serverConfig, ResourceManager resourceManager) { + private AutoTuneService createAutoFetchManager(SpiEbeanServer server, ServerConfig serverConfig, ResourceManager resourceManager) { - AutoFetchManager manager = createAutoFetchManager(server.getName(), resourceManager); + AutoTuneService manager = createAutoFetchManager(server.getName(), resourceManager); manager.setOwner(server, serverConfig); return manager; } - private AutoFetchManager createAutoFetchManager(String serverName, ResourceManager resourceManager) { + private AutoTuneService createAutoFetchManager(String serverName, ResourceManager resourceManager) { File autoFetchFile = getAutoFetchFile(serverName, resourceManager); - AutoFetchManager autoFetchManager = null; + AutoTuneService autoFetchManager = null; boolean readFile = !"false".equalsIgnoreCase(System.getProperty("autofetch.readfromfile")); if (readFile) { @@ -44,13 +45,13 @@ public class AutoFetchManagerFactory { // not deserialized from file so create as empty // It will be populated automatically by querying the // database meta data - autoFetchManager = new DefaultAutoFetchManager(autoFetchFile.getAbsolutePath()); + autoFetchManager = new BaseAutoTuneService(autoFetchFile.getAbsolutePath()); } return autoFetchManager; } - private AutoFetchManager deserializeAutoFetch(File autoFetchFile) { + private AutoTuneService deserializeAutoFetch(File autoFetchFile) { try { if (!autoFetchFile.exists()) { @@ -58,7 +59,7 @@ public class AutoFetchManagerFactory { } FileInputStream fi = new FileInputStream(autoFetchFile); ObjectInputStream ois = new ObjectInputStream(fi); - AutoFetchManager profListener = (AutoFetchManager) ois.readObject(); + AutoTuneService profListener = (AutoTuneService) ois.readObject(); ois.close(); logger.info("AutoFetch deserialized from file [" + autoFetchFile.getAbsolutePath() + "]"); diff --git a/src/main/java/com/avaje/ebeaninternal/server/autofetch/service/BaseAutoTuneService.java b/src/main/java/com/avaje/ebeaninternal/server/autofetch/service/BaseAutoTuneService.java new file mode 100644 index 000000000..81ee660b7 --- /dev/null +++ b/src/main/java/com/avaje/ebeaninternal/server/autofetch/service/BaseAutoTuneService.java @@ -0,0 +1,207 @@ +package com.avaje.ebeaninternal.server.autofetch.service; + +import com.avaje.ebean.config.AutofetchConfig; +import com.avaje.ebean.config.ServerConfig; +import com.avaje.ebeaninternal.api.SpiEbeanServer; +import com.avaje.ebeaninternal.api.SpiQuery; +import com.avaje.ebeaninternal.server.autofetch.AutoTuneService; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.ObjectOutputStream; +import java.io.Serializable; + +/** + * Implementation of the AutoTuneService which is comprised of profiling and query tuning. + */ +public class BaseAutoTuneService implements AutoTuneService { + + private static final Logger logger = LoggerFactory.getLogger(BaseAutoTuneService.class); + + private final long defaultGarbageCollectionWait; + + private final boolean garbageCollectionOnShutdown; + + private final BaseQueryTuner queryTuner; + + public BaseAutoTuneService(SpiEbeanServer server, ServerConfig serverConfig) { + + AutofetchConfig config = serverConfig.getAutofetchConfig(); + + ProfileManager profileManager = new ProfileManager(config, server); + this.queryTuner = new BaseQueryTuner(config, server, profileManager); + + this.garbageCollectionOnShutdown = config.isGarbageCollectionOnShutdown(); + this.defaultGarbageCollectionWait = (long) config.getGarbageCollectionWait(); + } + + /** + * Load the query tuning information from it's data store. + */ + public void startup() { + +// File autoFetchFile = new File(fileName); +// +// try { +// FileOutputStream fout = new FileOutputStream(autoFetchFile); +// +// ObjectOutputStream oout = new ObjectOutputStream(fout); +// oout.writeObject(this); +// oout.flush(); +// oout.close(); +// +// } catch (Exception e) { +// String msg = "Error serializing autofetch file"; +// logging.logError(msg, e); +// } + } + + private void saveProfiling() { + + } + + /** + * Shutdown the listener. + *

+ * We should try to collect the usage statistics by calling a System.gc(). + * This is necessary for use with short lived applications where garbage + * collection may not otherwise occur at all. + *

+ */ + public void shutdown() { + if (garbageCollectionOnShutdown) { + collectUsageViaGC(-1); + saveProfiling(); + } + } + + /** + * Ask for a System.gc() so that we gather node usage information. + *

+ * Really only want to do this sparingly but useful just prior to shutdown + * for short run application where garbage collection may otherwise not + * occur at all. + *

+ *

+ * waitMillis will do a thread sleep to give the garbage collection a little + * time to do its thing assuming we are shutting down the VM. + *

+ *

+ * If waitMillis is -1 then the defaultGarbageCollectionWait is used which + * defaults to 100 milliseconds. + *

+ */ + public void collectUsageViaGC() { + collectUsageViaGC(-1); + } + + public void collectUsageViaGC(long waitMillis) { + System.gc(); + try { + if (waitMillis < 0) { + waitMillis = defaultGarbageCollectionWait; + } + Thread.sleep(waitMillis); + } catch (InterruptedException e) { + logger.warn("Error while sleeping after System.gc() request.", e); + } + updateTunedQueryInfo(); + } + + /** + * Update the tuned fetch plans from the current usage information. + */ + public void updateTunedQueryInfo() { + +// if (!profiling) { +// // we are not collecting any profiling information at +// // the moment so don't try updating the tuned query plans. +// return "Not profiling"; +// } +// +// synchronized (statisticsMonitor) { +// +// Counters counters = new Counters(); +// +// for (ProfileOrigin queryPointStatistics : statisticsMap.values()) { +// if (!queryPointStatistics.hasUsage()) { +// // no usage statistics collected yet... +// counters.incrementNoUsage(); +// } else { +// updateTunedQueryFromUsage(counters, queryPointStatistics); +// } +// } +// +// String summaryInfo = counters.toString(); +// +// if (counters.isInteresting()) { +// // only log it if its interesting +// logging.logSummary(summaryInfo); +// } +// +// return summaryInfo; +// } + } + + +// +// private void updateTunedQueryFromUsage(Counters counters, ProfileOrigin statistics) { +// +// ObjectGraphOrigin queryPoint = statistics.getOrigin(); +// String beanType = queryPoint.getBeanType(); +// +// try { +// Class beanClass = ClassUtil.forName(beanType, this.getClass()); +// BeanDescriptor beanDescriptor = server.getBeanDescriptor(beanClass); +// if (beanDescriptor != null) { +// +// // Determine the fetch plan from the latest statistics. +// // Use this to compare with current "tuned fetch plan". +// OrmQueryDetail newFetchDetail = statistics.buildTunedFetch(beanDescriptor); +// +// // get the current tuned fetch info... +// TunedQueryInfo currentFetch = tunedQueryInfoMap.get(queryPoint.getKey()); +// +// if (currentFetch == null) { +// // its a new fetch plan, add it. +// counters.incrementNew(); +// +// currentFetch = statistics.createTunedFetch(newFetchDetail); +// logging.logNew(currentFetch); +// tunedQueryInfoMap.put(queryPoint.getKey(), currentFetch); +// +// } else if (!currentFetch.isSame(newFetchDetail)) { +// // the fetch plan has changed, update it. +// counters.incrementModified(); +// +// logging.logChanged(currentFetch, newFetchDetail); +// currentFetch.setTunedDetail(newFetchDetail); +// +// } else { +// // the fetch plan has not changed... +// counters.incrementUnchanged(); +// } +// +// currentFetch.setProfileCount(statistics.getCounter()); +// } +// +// } catch (ClassNotFoundException e) { +// // expected after renaming/moving an entity bean +// String msg = e.toString() + " updating autoFetch tuned query for " + beanType +// + ". It isLikely this bean has been renamed or moved"; +// logging.logInfo(msg, null); +// statisticsMap.remove(statistics.getOrigin().getKey()); +// } +// } +// + + /** + * Auto tune the query and enable profiling. + */ + public boolean tuneQuery(SpiQuery query) { + return queryTuner.tuneQuery(query); + } + +} diff --git a/src/main/java/com/avaje/ebeaninternal/server/autofetch/service/BaseQueryTuner.java b/src/main/java/com/avaje/ebeaninternal/server/autofetch/service/BaseQueryTuner.java new file mode 100644 index 000000000..a60f73fd9 --- /dev/null +++ b/src/main/java/com/avaje/ebeaninternal/server/autofetch/service/BaseQueryTuner.java @@ -0,0 +1,150 @@ +package com.avaje.ebeaninternal.server.autofetch.service; + +import com.avaje.ebean.bean.CallStack; +import com.avaje.ebean.bean.ObjectGraphNode; +import com.avaje.ebean.config.AutofetchConfig; +import com.avaje.ebean.config.AutofetchMode; +import com.avaje.ebeaninternal.api.SpiEbeanServer; +import com.avaje.ebeaninternal.api.SpiQuery; +import com.avaje.ebeaninternal.server.autofetch.ProfilingListener; + +import javax.persistence.PersistenceException; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +/** + * + */ +public class BaseQueryTuner { + + private final boolean queryTuning; + + /** + * Converted from a 0-100 int to a double. Effectively a percentage rate at + * which to collect profiling information. + */ + private final double profilingRate; + + private final int profilingBase; + + private final int profilingMin; + + private boolean profiling; + + private final AutofetchMode mode; + + /** + * Map of the tuned query details per profile query point. + */ + private final Map tunedQueryInfoMap = new ConcurrentHashMap(); + + + private final SpiEbeanServer server; + + private final ProfilingListener profilingListener; + + public BaseQueryTuner(AutofetchConfig config, SpiEbeanServer server, ProfilingListener profilingListener) { + this.server = server; + this.profilingListener = profilingListener; + this.mode = config.getMode(); + this.queryTuning = config.isQueryTuning(); + this.profiling = config.isProfiling(); + this.profilingRate = config.getProfilingRate(); + this.profilingBase = config.getProfilingBase(); + this.profilingMin = config.getProfilingMin(); + } + + /** + * Load the tuned query information. + */ + public void load(String key, TunedQueryInfo queryInfo) { + tunedQueryInfoMap.put(key, queryInfo); + } + + /** + * Auto tune the query and enable profiling. + */ + public boolean tuneQuery(SpiQuery query) { + + if (!queryTuning && !profiling) { + return false; + } + + if (!useAutoFetch(query)) { + // not using autoFetch for this query + return false; + } + + ObjectGraphNode parentAutoFetchNode = query.getParentNode(); + if (parentAutoFetchNode != null) { + // This is a +lazy/+query query with profiling on. + // We continue to collect the profiling information. + query.setProfilingListener(profilingListener); + return true; + } + + // create a query point to identify the query + CallStack stack = server.createCallStack(); + ObjectGraphNode origin = query.setOrigin(stack); + + // get current "tuned fetch" for this query point + TunedQueryInfo tunedFetch = tunedQueryInfoMap.get(origin.getOriginQueryPoint().getKey()); + + // get the number of times we have collected profiling information + int profileCount = tunedFetch == null ? 0 : tunedFetch.getProfileCount(); + + if (profiling) { + // we want more profiling information? + if (tunedFetch == null) { + query.setProfilingListener(profilingListener); + + } else if (profileCount < profilingBase) { + query.setProfilingListener(profilingListener); + + } else if (tunedFetch.isPercentageProfile(profilingRate)) { + query.setProfilingListener(profilingListener); + } + } + + if (queryTuning && tunedFetch != null && profileCount >= profilingMin) { + // deemed to have enough profiling information for automatic tuning + return tunedFetch.autoFetchTune(query); + } + + return false; + } + + /** + * Return true if we should try to use autoFetch for this query. + */ + private boolean useAutoFetch(SpiQuery query) { + + if (query.isLoadBeanCache()) { + // when loading the cache don't tune the query + // as we want full objects loaded into the cache + return false; + } + + Boolean autoFetch = query.isAutofetch(); + if (autoFetch != null) { + // explicitly set... + return autoFetch; + + } else { + // determine using implicit mode... + switch (mode) { + case DEFAULT_ON: + return true; + + case DEFAULT_OFF: + return false; + + case DEFAULT_ONIFEMPTY: + return query.isDetailEmpty(); + + default: + throw new PersistenceException("Invalid autoFetchMode " + mode); + } + } + } +} diff --git a/src/main/java/com/avaje/ebeaninternal/server/autofetch/DefaultAutoFetchManagerLogging.java b/src/main/java/com/avaje/ebeaninternal/server/autofetch/service/DefaultAutoFetchManagerLogging.java similarity index 92% rename from src/main/java/com/avaje/ebeaninternal/server/autofetch/DefaultAutoFetchManagerLogging.java rename to src/main/java/com/avaje/ebeaninternal/server/autofetch/service/DefaultAutoFetchManagerLogging.java index f1cbb183b..8ba42bb6b 100644 --- a/src/main/java/com/avaje/ebeaninternal/server/autofetch/DefaultAutoFetchManagerLogging.java +++ b/src/main/java/com/avaje/ebeaninternal/server/autofetch/service/DefaultAutoFetchManagerLogging.java @@ -1,4 +1,4 @@ -package com.avaje.ebeaninternal.server.autofetch; +package com.avaje.ebeaninternal.server.autofetch.service; import com.avaje.ebean.config.ServerConfig; import com.avaje.ebeaninternal.api.SpiEbeanServer; @@ -19,11 +19,11 @@ public class DefaultAutoFetchManagerLogging { private static final Logger logger = LoggerFactory.getLogger(DefaultAutoFetchManagerLogging.class); - private final DefaultAutoFetchManager manager; + private final BaseAutoTuneService manager; private final int updateFreqInSecs; - public DefaultAutoFetchManagerLogging(ServerConfig serverConfig, DefaultAutoFetchManager profileListener) { + public DefaultAutoFetchManagerLogging(ServerConfig serverConfig, BaseAutoTuneService profileListener) { this.manager = profileListener; this.updateFreqInSecs = serverConfig.getAutofetchConfig().getProfileUpdateFrequency(); diff --git a/src/main/java/com/avaje/ebeaninternal/server/autofetch/service/ProfileManager.java b/src/main/java/com/avaje/ebeaninternal/server/autofetch/service/ProfileManager.java new file mode 100644 index 000000000..a9bada622 --- /dev/null +++ b/src/main/java/com/avaje/ebeaninternal/server/autofetch/service/ProfileManager.java @@ -0,0 +1,112 @@ +package com.avaje.ebeaninternal.server.autofetch.service; + +import com.avaje.ebean.bean.NodeUsageCollector; +import com.avaje.ebean.bean.ObjectGraphNode; +import com.avaje.ebean.bean.ObjectGraphOrigin; +import com.avaje.ebean.config.AutofetchConfig; +import com.avaje.ebeaninternal.api.SpiEbeanServer; +import com.avaje.ebeaninternal.server.autofetch.ProfilingListener; +import com.avaje.ebeaninternal.server.deploy.BeanDescriptor; +import com.avaje.ebeaninternal.server.querydefn.OrmQueryDetail; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +/** + * + */ +public class ProfileManager implements ProfilingListener { + + private static final Logger logger = LoggerFactory.getLogger(ProfileManager.class); + + private final boolean queryTuningAddVersion; + + private final boolean profiling; + + /** + * Map of the usage and query statistics gathered. + */ + private final Map profileMap = new ConcurrentHashMap(); + + private final Object monitor = new Object(); + + private final SpiEbeanServer server; + + public ProfileManager(AutofetchConfig config, SpiEbeanServer server) { + this.server = server; + this.profiling = config.isProfiling(); + this.queryTuningAddVersion = config.isQueryTuningAddVersion(); + } + + /** + * Gather query execution statistics. This could either be the originating + * query in which case the parentNode will be null, or a lazy loading query + * resulting from traversal of the object graph. + */ + public void collectQueryInfo(ObjectGraphNode node, long beans, long micros) { + + if (node != null) { + ObjectGraphOrigin origin = node.getOriginQueryPoint(); + if (origin != null) { + ProfileOrigin stats = getProfileOrigin(origin); + stats.collectQueryInfo(node, beans, micros); + } + } + } + + /** + * Collect usage statistics from a node in the object graph. + *

+ * This is sent to use from a EntityBeanIntercept when the finalise method + * is called on the bean. + *

+ */ + public void collectNodeUsage(NodeUsageCollector usageCollector) { + + ProfileOrigin profileOrigin = getProfileOrigin(usageCollector.getNode().getOriginQueryPoint()); + profileOrigin.collectUsageInfo(usageCollector); + } + + private ProfileOrigin getProfileOrigin(ObjectGraphOrigin originQueryPoint) { + synchronized (monitor) { + ProfileOrigin stats = profileMap.get(originQueryPoint.getKey()); + if (stats == null) { + stats = new ProfileOrigin(originQueryPoint, queryTuningAddVersion); + profileMap.put(originQueryPoint.getKey(), stats); + } + return stats; + } + } + + + /** + * Update the tuned fetch plans from the current usage information. + */ + public void updateTunedQueryInfo() { + + if (!profiling) { + // we are not collecting any profiling information at + // the moment so don't try updating the tuned query plans. + return;// "Not profiling"; + } + + synchronized (monitor) { + + for (ProfileOrigin origin : profileMap.values()) { + if (origin.hasUsage()) { + OrmQueryDetail ormQueryDetail = updateTunedQueryFromUsage(origin); + + } + } + } + } + + + private OrmQueryDetail updateTunedQueryFromUsage(ProfileOrigin statistics) { + + BeanDescriptor desc = server.getBeanDescriptorById(statistics.getOrigin().getBeanType()); + return desc == null ? null : statistics.buildTunedFetch(desc); + } +} diff --git a/src/main/java/com/avaje/ebeaninternal/server/autofetch/Statistics.java b/src/main/java/com/avaje/ebeaninternal/server/autofetch/service/ProfileOrigin.java similarity index 52% rename from src/main/java/com/avaje/ebeaninternal/server/autofetch/Statistics.java rename to src/main/java/com/avaje/ebeaninternal/server/autofetch/service/ProfileOrigin.java index a9625ca70..959b740a2 100644 --- a/src/main/java/com/avaje/ebeaninternal/server/autofetch/Statistics.java +++ b/src/main/java/com/avaje/ebeaninternal/server/autofetch/service/ProfileOrigin.java @@ -1,4 +1,4 @@ -package com.avaje.ebeaninternal.server.autofetch; +package com.avaje.ebeaninternal.server.autofetch.service; import com.avaje.ebean.bean.NodeUsageCollector; import com.avaje.ebean.bean.ObjectGraphNode; @@ -13,7 +13,7 @@ import java.util.Collection; import java.util.LinkedHashMap; import java.util.Map; -public class Statistics implements Serializable { +public class ProfileOrigin implements Serializable { private static final long serialVersionUID = -5586783791097230766L; @@ -24,14 +24,14 @@ public class Statistics implements Serializable { private int counter; - private final Map queryStatsMap = new LinkedHashMap(); + private final Map queryStatsMap = new LinkedHashMap(); - private final Map nodeUsageMap = new LinkedHashMap(); + private final Map nodeUsageMap = new LinkedHashMap(); @SuppressWarnings("RedundantStringConstructorCall") private final String monitor = new String(); - public Statistics(ObjectGraphOrigin origin, boolean queryTuningAddVersion) { + public ProfileOrigin(ObjectGraphOrigin origin, boolean queryTuningAddVersion) { this.origin = origin; this.queryTuningAddVersion = queryTuningAddVersion; } @@ -40,13 +40,13 @@ public class Statistics implements Serializable { return origin; } - public TunedQueryInfo createTunedFetch(OrmQueryDetail newFetchDetail) { - synchronized (monitor) { - // NB: create a copy of queryPoint allowing garbage - // collection of source... - return new TunedQueryInfo(origin, newFetchDetail, counter); - } - } +// public TunedQueryInfo createTunedFetch(OrmQueryDetail newFetchDetail) { +// synchronized (monitor) { +// // NB: create a copy of queryPoint allowing garbage +// // collection of source... +// return new TunedQueryInfo(origin, newFetchDetail, counter); +// } +// } /** * Return the number of times the root query has executed. @@ -77,7 +77,7 @@ public class Statistics implements Serializable { PathProperties pathProps = new PathProperties(); - for (StatisticsNodeUsage statsNode : nodeUsageMap.values()) { + for (ProfileOriginNodeUsage statsNode : nodeUsageMap.values()) { statsNode.buildTunedFetch(pathProps, rootDesc); } @@ -98,23 +98,22 @@ public class Statistics implements Serializable { public void collectQueryInfo(ObjectGraphNode node, long beansLoaded, long micros) { - synchronized (monitor) { - String key = node.getPath(); - if (key == null) { - key = ""; - // this is basically the number of times the root query - // has executed which gives us an indication of how - // much profiling information we have gathered. - counter++; - } - - StatisticsQuery stats = queryStatsMap.get(key); - if (stats == null) { - stats = new StatisticsQuery(key); - queryStatsMap.put(key, stats); - } - stats.add(beansLoaded, micros); + String key = node.getPath(); + if (key == null) { + key = ""; + // this is basically the number of times the root query + // has executed which gives us an indication of how + // much profiling information we have gathered. + counter++; } + + ProfileOriginQuery stats = queryStatsMap.get(key); + if (stats == null) { + // a race condition but we don't care + stats = new ProfileOriginQuery(key); + queryStatsMap.put(key, stats); + } + stats.add(beansLoaded, micros); } @@ -126,49 +125,49 @@ public class Statistics implements Serializable { if (!profile.isEmpty()) { ObjectGraphNode node = profile.getNode(); - StatisticsNodeUsage nodeStats = getNodeStats(node.getPath()); + ProfileOriginNodeUsage nodeStats = getNodeStats(node.getPath()); nodeStats.publish(profile); } } - private StatisticsNodeUsage getNodeStats(String path) { + private ProfileOriginNodeUsage getNodeStats(String path) { synchronized (monitor) { - StatisticsNodeUsage nodeStats = nodeUsageMap.get(path); + ProfileOriginNodeUsage nodeStats = nodeUsageMap.get(path); if (nodeStats == null) { - nodeStats = new StatisticsNodeUsage(path, queryTuningAddVersion); + nodeStats = new ProfileOriginNodeUsage(path, queryTuningAddVersion); nodeUsageMap.put(path, nodeStats); } return nodeStats; } } - public String getUsageDebug() { - synchronized (monitor) { - StringBuilder sb = new StringBuilder(); - sb.append("root[").append(origin.getBeanType()).append("] "); - for (StatisticsNodeUsage node : nodeUsageMap.values()) { - sb.append(node.toString()).append("\n"); - } - return sb.toString(); - } - } - - public String getQueryStatDebug() { - synchronized (monitor) { - StringBuilder sb = new StringBuilder(); - for (StatisticsQuery queryStat : queryStatsMap.values()) { - sb.append(queryStat.toString()).append("\n"); - } - return sb.toString(); - } - } - - public String toString() { - - synchronized (monitor) { - return getUsageDebug(); - } - } +// public String getUsageDebug() { +// synchronized (monitor) { +// StringBuilder sb = new StringBuilder(); +// sb.append("root[").append(origin.getBeanType()).append("] "); +// for (ProfileOriginNodeUsage node : nodeUsageMap.values()) { +// sb.append(node.toString()).append("\n"); +// } +// return sb.toString(); +// } +// } +// +// public String getQueryStatDebug() { +// synchronized (monitor) { +// StringBuilder sb = new StringBuilder(); +// for (ProfileOriginQuery queryStat : queryStatsMap.values()) { +// sb.append(queryStat.toString()).append("\n"); +// } +// return sb.toString(); +// } +// } +// +// public String toString() { +// +// synchronized (monitor) { +// return getUsageDebug(); +// } +// } } diff --git a/src/main/java/com/avaje/ebeaninternal/server/autofetch/StatisticsNodeUsage.java b/src/main/java/com/avaje/ebeaninternal/server/autofetch/service/ProfileOriginNodeUsage.java similarity index 91% rename from src/main/java/com/avaje/ebeaninternal/server/autofetch/StatisticsNodeUsage.java rename to src/main/java/com/avaje/ebeaninternal/server/autofetch/service/ProfileOriginNodeUsage.java index 868f8f356..a051bc31f 100644 --- a/src/main/java/com/avaje/ebeaninternal/server/autofetch/StatisticsNodeUsage.java +++ b/src/main/java/com/avaje/ebeaninternal/server/autofetch/service/ProfileOriginNodeUsage.java @@ -1,4 +1,4 @@ -package com.avaje.ebeaninternal.server.autofetch; +package com.avaje.ebeaninternal.server.autofetch.service; import com.avaje.ebean.bean.NodeUsageCollector; import com.avaje.ebean.text.PathProperties; @@ -18,11 +18,11 @@ import java.util.Set; /** * Collects usages statistics for a given node in the object graph. */ -public class StatisticsNodeUsage implements Serializable { +public class ProfileOriginNodeUsage implements Serializable { private static final long serialVersionUID = -1663951463963779547L; - private static final Logger logger = LoggerFactory.getLogger(StatisticsNodeUsage.class); + private static final Logger logger = LoggerFactory.getLogger(ProfileOriginNodeUsage.class); @SuppressWarnings("RedundantStringConstructorCall") private final String monitor = new String(); @@ -39,7 +39,7 @@ public class StatisticsNodeUsage implements Serializable { private final Set aggregateUsed = new LinkedHashSet(); - public StatisticsNodeUsage(String path, boolean queryTuningAddVersion) { + public ProfileOriginNodeUsage(String path, boolean queryTuningAddVersion) { this.path = path; this.queryTuningAddVersion = queryTuningAddVersion; } @@ -99,7 +99,7 @@ public class StatisticsNodeUsage implements Serializable { synchronized (monitor) { - HashSet used = profile.getUsed(); + Set used = profile.getUsed(); profileCount++; if (!used.isEmpty()) { diff --git a/src/main/java/com/avaje/ebeaninternal/server/autofetch/StatisticsQuery.java b/src/main/java/com/avaje/ebeaninternal/server/autofetch/service/ProfileOriginQuery.java similarity index 81% rename from src/main/java/com/avaje/ebeaninternal/server/autofetch/StatisticsQuery.java rename to src/main/java/com/avaje/ebeaninternal/server/autofetch/service/ProfileOriginQuery.java index 506087b31..b4d2b6bfa 100644 --- a/src/main/java/com/avaje/ebeaninternal/server/autofetch/StatisticsQuery.java +++ b/src/main/java/com/avaje/ebeaninternal/server/autofetch/service/ProfileOriginQuery.java @@ -1,11 +1,11 @@ -package com.avaje.ebeaninternal.server.autofetch; +package com.avaje.ebeaninternal.server.autofetch.service; import java.io.Serializable; /** * Used to accumulate query execution statistics. */ -public class StatisticsQuery implements Serializable { +public class ProfileOriginQuery implements Serializable { private static final long serialVersionUID = -1133958958072778811L; @@ -17,7 +17,7 @@ public class StatisticsQuery implements Serializable { private long totalMicros; - public StatisticsQuery(String path) { + public ProfileOriginQuery(String path) { this.path = path; } diff --git a/src/main/java/com/avaje/ebeaninternal/server/autofetch/TunedQueryInfo.java b/src/main/java/com/avaje/ebeaninternal/server/autofetch/service/TunedQueryInfo.java similarity index 98% rename from src/main/java/com/avaje/ebeaninternal/server/autofetch/TunedQueryInfo.java rename to src/main/java/com/avaje/ebeaninternal/server/autofetch/service/TunedQueryInfo.java index dec85ebee..8a6a5d046 100644 --- a/src/main/java/com/avaje/ebeaninternal/server/autofetch/TunedQueryInfo.java +++ b/src/main/java/com/avaje/ebeaninternal/server/autofetch/service/TunedQueryInfo.java @@ -1,4 +1,4 @@ -package com.avaje.ebeaninternal.server.autofetch; +package com.avaje.ebeaninternal.server.autofetch.service; import com.avaje.ebean.bean.ObjectGraphOrigin; import com.avaje.ebeaninternal.api.SpiQuery; diff --git a/src/main/java/com/avaje/ebeaninternal/server/core/DefaultContainer.java b/src/main/java/com/avaje/ebeaninternal/server/core/DefaultContainer.java index 2f0a2db78..e09e3873e 100644 --- a/src/main/java/com/avaje/ebeaninternal/server/core/DefaultContainer.java +++ b/src/main/java/com/avaje/ebeaninternal/server/core/DefaultContainer.java @@ -50,7 +50,7 @@ public class DefaultContainer implements SpiContainer { private final JndiDataSourceLookup jndiDataSourceFactory; - private final AtomicInteger serverId = new AtomicInteger(1); +// private final AtomicInteger serverId = new AtomicInteger(1); public DefaultContainer(ContainerConfig containerConfig) { @@ -119,7 +119,7 @@ public class DefaultContainer implements SpiContainer { ServerCacheManager cacheManager = getCacheManager(serverConfig); - int uniqueServerId = serverId.incrementAndGet(); +// int uniqueServerId = serverId.incrementAndGet(); SpiBackgroundExecutor bgExecutor = createBackgroundExecutor(serverConfig); XmlConfigLoader xmlConfigLoader = new XmlConfigLoader(null); @@ -131,18 +131,18 @@ public class DefaultContainer implements SpiContainer { cacheManager.init(server); - if (serverConfig.isRegisterJmxMBeans()) { - MBeanServer mbeanServer; - ArrayList list = MBeanServerFactory.findMBeanServer(null); - if (list.size() == 0) { - // probably not running in a server - mbeanServer = MBeanServerFactory.createMBeanServer(); - } else { - // use the first MBeanServer - mbeanServer = (MBeanServer) list.get(0); - } - server.registerMBeans(mbeanServer, uniqueServerId); - } +// if (serverConfig.isRegisterJmxMBeans()) { +// MBeanServer mbeanServer; +// ArrayList list = MBeanServerFactory.findMBeanServer(null); +// if (list.size() == 0) { +// // probably not running in a server +// mbeanServer = MBeanServerFactory.createMBeanServer(); +// } else { +// // use the first MBeanServer +// mbeanServer = (MBeanServer) list.get(0); +// } +// server.registerMBeans(mbeanServer, uniqueServerId); +// } // generate and run DDL if required // if there are any other tasks requiring action in their plugins, do them as well diff --git a/src/main/java/com/avaje/ebeaninternal/server/core/DefaultServer.java b/src/main/java/com/avaje/ebeaninternal/server/core/DefaultServer.java index a716516c5..4b14fc42b 100644 --- a/src/main/java/com/avaje/ebeaninternal/server/core/DefaultServer.java +++ b/src/main/java/com/avaje/ebeaninternal/server/core/DefaultServer.java @@ -33,7 +33,7 @@ import com.avaje.ebeaninternal.api.SpiQuery.Type; import com.avaje.ebeaninternal.api.SpiSqlQuery; import com.avaje.ebeaninternal.api.SpiTransaction; import com.avaje.ebeaninternal.api.TransactionEventTable; -import com.avaje.ebeaninternal.server.autofetch.AutoFetchManager; +import com.avaje.ebeaninternal.server.autofetch.AutoTuneService; import com.avaje.ebean.dbmigration.DdlGenerator; import com.avaje.ebeaninternal.server.deploy.BeanDescriptor; import com.avaje.ebeaninternal.server.deploy.BeanDescriptorManager; @@ -43,7 +43,6 @@ import com.avaje.ebeaninternal.server.deploy.DeployNamedQuery; import com.avaje.ebeaninternal.server.deploy.DeployNamedUpdate; import com.avaje.ebeaninternal.server.deploy.InheritInfo; import com.avaje.ebeaninternal.server.el.ElFilter; -import com.avaje.ebeaninternal.server.jmx.MAdminAutofetch; import com.avaje.ebeaninternal.server.lib.ShutdownManager; import com.avaje.ebeaninternal.server.query.CQuery; import com.avaje.ebeaninternal.server.query.CQueryEngine; @@ -69,9 +68,6 @@ import com.avaje.ebeaninternal.util.ParamTypeHelper.TypeInfo; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import javax.management.InstanceAlreadyExistsException; -import javax.management.MBeanServer; -import javax.management.ObjectName; import javax.persistence.OptimisticLockException; import javax.persistence.PersistenceException; import java.util.ArrayList; @@ -103,8 +99,6 @@ public final class DefaultServer implements SpiServer, SpiEbeanServer { private final DatabasePlatform databasePlatform; - private final AdminAutofetch adminAutofetch; - private final TransactionManager transactionManager; private final TransactionScopeManager transactionScopeManager; @@ -132,7 +126,7 @@ public final class DefaultServer implements SpiServer, SpiEbeanServer { private final DiffHelp diffHelp; - private final AutoFetchManager autoFetchManager; + private final AutoTuneService autoTuneService; private final CQueryEngine cqueryEngine; @@ -152,21 +146,11 @@ public final class DefaultServer implements SpiServer, SpiEbeanServer { private final MetaInfoManager metaInfoManager; - /** - * The MBean name used to register Ebean. - */ - private String mbeanName; - /** * The default PersistenceContextScope used if it is not explicitly set on a query. */ private final PersistenceContextScope defaultPersistenceContextScope; - /** - * The MBeanServer Ebean is registered with. - */ - private MBeanServer mbeanServer; - /** * Flag set when the server has shutdown. */ @@ -235,8 +219,7 @@ public final class DefaultServer implements SpiServer, SpiEbeanServer { this.queryEngine = config.createOrmQueryEngine(); this.relationalQueryEngine = config.createRelationalQueryEngine(); - this.autoFetchManager = config.createAutoFetchManager(this); - this.adminAutofetch = new MAdminAutofetch(autoFetchManager); + this.autoTuneService = config.createAutoFetchManager(this); this.beanLoader = new DefaultBeanLoader(this); this.jsonContext = config.createJsonContext(this); @@ -346,11 +329,7 @@ public final class DefaultServer implements SpiServer, SpiEbeanServer { } public AdminAutofetch getAdminAutofetch() { - return adminAutofetch; - } - - public AutoFetchManager getAutoFetchManager() { - return autoFetchManager; + return autoTuneService; } /** @@ -373,41 +352,6 @@ public final class DefaultServer implements SpiServer, SpiEbeanServer { public void start() { } - public void registerMBeans(MBeanServer mbeanServer, int uniqueServerId) { - - this.mbeanServer = mbeanServer; - this.mbeanName = "Ebean:server=" + serverName + uniqueServerId; - - ObjectName autofetchName; - try { - autofetchName = new ObjectName(mbeanName + ",key=AutoFetch"); - } catch (Exception e) { - String msg = "Failed to register the JMX beans for Ebean server [" + serverName + "]."; - logger.error(msg, e); - return; - } - - try { - mbeanServer.registerMBean(adminAutofetch, autofetchName); - - } catch (InstanceAlreadyExistsException e) { - // tomcat webapp reloading - String msg = "JMX beans for Ebean server [" + serverName + "] already registered. Will try unregister/register" + e.getMessage(); - logger.warn(msg); - try { - mbeanServer.unregisterMBean(autofetchName); - mbeanServer.registerMBean(adminAutofetch, autofetchName); - - } catch (Exception ae) { - String amsg = "Unable to unregister/register the JMX beans for Ebean server [" + serverName + "]."; - logger.error(amsg, ae); - } - } catch (Exception e) { - String msg = "Error registering MBean[" + mbeanName + "]"; - logger.error(msg, e); - } - } - /** * Shutting down via JVM Shutdown hook. */ @@ -439,16 +383,9 @@ public final class DefaultServer implements SpiServer, SpiEbeanServer { return; } shutdownPlugins(); - try { - if (mbeanServer != null) { - mbeanServer.unregisterMBean(new ObjectName(mbeanName + ",key=AutoFetch")); - } - } catch (Exception e) { - logger.error("Error unregistering Ebean " + mbeanName, e); - } // shutdown autofetch profile collection - autoFetchManager.shutdown(); + autoTuneService.shutdown(); // shutdown background threads backgroundExecutor.shutdown(); // shutdown DataSource (if its an Ebean one) @@ -1100,7 +1037,7 @@ public final class DefaultServer implements SpiServer, SpiEbeanServer { public SpiOrmQueryRequest createQueryRequest(BeanDescriptor desc, SpiQuery query, Transaction t) { - if (desc.isAutoFetchTunable() && !query.isSqlSelect() && !autoFetchManager.tuneQuery(query)) { + if (desc.isAutoFetchTunable() && !query.isSqlSelect() && !autoTuneService.tuneQuery(query)) { // use deployment FetchType.LAZY/EAGER annotations // to define the 'default' select clause query.setDefaultSelectClause(); diff --git a/src/main/java/com/avaje/ebeaninternal/server/core/InternalConfiguration.java b/src/main/java/com/avaje/ebeaninternal/server/core/InternalConfiguration.java index f0b0e0b30..c1148d0cf 100644 --- a/src/main/java/com/avaje/ebeaninternal/server/core/InternalConfiguration.java +++ b/src/main/java/com/avaje/ebeaninternal/server/core/InternalConfiguration.java @@ -13,8 +13,8 @@ import com.avaje.ebean.plugin.SpiServerPlugin; import com.avaje.ebean.text.json.JsonContext; import com.avaje.ebeaninternal.api.SpiBackgroundExecutor; import com.avaje.ebeaninternal.api.SpiEbeanServer; -import com.avaje.ebeaninternal.server.autofetch.AutoFetchManager; -import com.avaje.ebeaninternal.server.autofetch.AutoFetchManagerFactory; +import com.avaje.ebeaninternal.server.autofetch.AutoTuneService; +import com.avaje.ebeaninternal.server.autofetch.service.AutoTuneServiceFactory; import com.avaje.ebeaninternal.server.changelog.DefaultChangeLogPrepare; import com.avaje.ebeaninternal.server.changelog.DefaultChangeLogRegister; import com.avaje.ebeaninternal.server.changelog.DefaultChangeLogListener; @@ -235,8 +235,8 @@ public class InternalConfiguration { return xmlConfig; } - public AutoFetchManager createAutoFetchManager(SpiEbeanServer server) { - return AutoFetchManagerFactory.create(server, serverConfig, resourceManager); + public AutoTuneService createAutoFetchManager(SpiEbeanServer server) { + return AutoTuneServiceFactory.create(server, serverConfig, resourceManager); } public RelationalQueryEngine createRelationalQueryEngine() { diff --git a/src/main/java/com/avaje/ebeaninternal/server/jmx/MAdminAutofetch.java b/src/main/java/com/avaje/ebeaninternal/server/jmx/MAdminAutofetch.java deleted file mode 100644 index a31100d6d..000000000 --- a/src/main/java/com/avaje/ebeaninternal/server/jmx/MAdminAutofetch.java +++ /dev/null @@ -1,121 +0,0 @@ -package com.avaje.ebeaninternal.server.jmx; - -import com.avaje.ebean.AdminAutofetch; -import com.avaje.ebean.EbeanServer; -import com.avaje.ebean.config.AutofetchMode; -import com.avaje.ebeaninternal.server.autofetch.AutoFetchManager; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Implementation of the AutoFetchControl. - *

- * This is accessible via {@link EbeanServer#getAdminAutofetch()} or via JMX - * MBeans. - *

- */ -public class MAdminAutofetch implements MAdminAutofetchMBean, AdminAutofetch { - - private static final Logger logger = LoggerFactory.getLogger(MAdminAutofetch.class); - - final AutoFetchManager autoFetchManager; - - final String modeOptions; - - public MAdminAutofetch(AutoFetchManager autoFetchListener) { - this.autoFetchManager = autoFetchListener; - this.modeOptions = AutofetchMode.DEFAULT_OFF + ", " - + AutofetchMode.DEFAULT_ON + ", " - + AutofetchMode.DEFAULT_ONIFEMPTY; - } - - public boolean isQueryTuning() { - return autoFetchManager.isQueryTuning(); - } - - public void setQueryTuning(boolean enable) { - autoFetchManager.setQueryTuning(enable); - } - - public boolean isProfiling() { - return autoFetchManager.isProfiling(); - } - - public void setProfiling(boolean enable) { - autoFetchManager.setProfiling(enable); - } - - public String getModeOptions() { - return modeOptions; - } - - public String getMode() { - return autoFetchManager.getMode().name(); - } - - public void setMode(String implicitMode) { - try { - AutofetchMode mode = AutofetchMode.valueOf(implicitMode); - autoFetchManager.setMode(mode); - } catch (Exception e) { - logger.info("Invalid implicit mode attempted "+e.getMessage()); - } - } - - public String collectUsageViaGC() { - return autoFetchManager.collectUsageViaGC(-1); - } - - public double getProfilingRate() { - return autoFetchManager.getProfilingRate(); - } - - public void setProfilingRate(double rate) { - autoFetchManager.setProfilingRate(rate); - } - - public int getProfilingMin() { - return autoFetchManager.getProfilingMin(); - } - - public int getProfilingBase() { - return autoFetchManager.getProfilingBase(); - } - - public void setProfilingMin(int profilingMin) { - autoFetchManager.setProfilingMin(profilingMin); - } - - public void setProfilingBase(int profilingMax) { - autoFetchManager.setProfilingBase(profilingMax); - } - - public String updateTunedQueryInfo() { - return autoFetchManager.updateTunedQueryInfo(); - } - - public int clearProfilingInfo() { - return autoFetchManager.clearProfilingInfo(); - } - - public int clearTunedQueryInfo() { - return autoFetchManager.clearTunedQueryInfo(); - } - - public void clearQueryStatistics() { - autoFetchManager.clearQueryStatistics(); - } - - public int getTotalProfileSize() { - return autoFetchManager.getTotalProfileSize(); - } - - public int getTotalTunedQueryCount() { - return autoFetchManager.getTotalTunedQueryCount(); - } - - public int getTotalTunedQuerySize() { - return autoFetchManager.getTotalTunedQuerySize(); - } - -} diff --git a/src/main/java/com/avaje/ebeaninternal/server/jmx/MAdminAutofetchMBean.java b/src/main/java/com/avaje/ebeaninternal/server/jmx/MAdminAutofetchMBean.java deleted file mode 100644 index ee6e10e78..000000000 --- a/src/main/java/com/avaje/ebeaninternal/server/jmx/MAdminAutofetchMBean.java +++ /dev/null @@ -1,145 +0,0 @@ -package com.avaje.ebeaninternal.server.jmx; - -import com.avaje.ebean.Query; - -public interface MAdminAutofetchMBean { - - /** - * Return true if profiling is enabled. - */ - boolean isProfiling(); - - /** - * Set to true to enable profiling. - */ - void setProfiling(boolean enable); - - /** - * Return true if autoFetch is enabled. - */ - boolean isQueryTuning(); - - /** - * Set to true to enable autoFetch. - */ - void setQueryTuning(boolean enable); - - /** - * This controls whether autoFetch is used when it has not - * been explicitly set on a query via {@link Query#setAutofetch(boolean)}. - */ - String getMode(); - - /** - * List of the valid implicit modes that can be used. - */ - String getModeOptions(); - - /** - * Set the auto fetch mode used when a query has not had {@link Query#setAutofetch(boolean)}. - */ - void setMode(String mode); - - /** - * Return the max number of queries profiled (per query point). - *

- * The number of queries profiled is collected per query point. Once a query - * point has profiled this number of queries it does not profile any more. - *

- */ - int getProfilingBase(); - - /** - * Set a max number of queries to profile per query point. - *

- * This number should provide a level of confidence that no more profiling - * is required for this query point. - *

- */ - void setProfilingBase(int profilingMaxThreshold); - - /** - * Returns the rate which profiling is collected. - * This is an int between 0 and 100. - */ - double getProfilingRate(); - - /** - * Set the rate at which profiling is collected after the base. - * - * @param rate a int between 0 and 100. - */ - void setProfilingRate(double rate); - - /** - * Return the minimum number of queries profiled before autoFetch will start - * automatically tuning the queries. - *

- * This could be one which means start autoFetch tuning after the first - * profiling information is collected. - *

- */ - int getProfilingMin(); - - /** - * Set the minimum number of queries profiled per query point before - * autoFetch will automatically tune the queries. - *

- * Increasing this number will mean more profiling is collected before - * autoFetch starts tuning the query. - *

- */ - void setProfilingMin(int autoFetchMinThreshold); - - /** - * Fire a garbage collection (hint to the JVM). Assuming garbage collection - * fires this will gather the usage profiling information. - */ - String collectUsageViaGC(); - - /** - * This will take the current profiling information and update the "tuned query - * detail". - *

- * This is done periodically and can also be manually invoked. - *

- */ - String updateTunedQueryInfo(); - - /** - * Clear all the tuned query info. - *

- * Should only need do this for testing and playing around. - *

- * @return the amount of tuned query information cleared. - */ - int clearTunedQueryInfo(); - - /** - * Clear all the profiling information. - *

- * This means the profiling information will need to be re-gathered. - *

- *

- * Should only need do this for testing and playing around. - *

- * @return the amount of profiled information cleared. - */ - int clearProfilingInfo(); - - /** - * Return the number of queries tuned by AutoFetch. - */ - int getTotalTunedQueryCount(); - - /** - * Return the size of the TuneQuery map. - */ - int getTotalTunedQuerySize(); - - /** - * Return the size of the profile map. - */ - int getTotalProfileSize(); - -} diff --git a/src/main/java/com/avaje/ebeaninternal/server/jmx/package-info.java b/src/main/java/com/avaje/ebeaninternal/server/jmx/package-info.java deleted file mode 100644 index edaa2ad21..000000000 --- a/src/main/java/com/avaje/ebeaninternal/server/jmx/package-info.java +++ /dev/null @@ -1 +0,0 @@ -package com.avaje.ebeaninternal.server.jmx; \ No newline at end of file diff --git a/src/main/java/com/avaje/ebeaninternal/server/loadcontext/DLoadBeanContext.java b/src/main/java/com/avaje/ebeaninternal/server/loadcontext/DLoadBeanContext.java index b1f7adada..d7eb2d86b 100644 --- a/src/main/java/com/avaje/ebeaninternal/server/loadcontext/DLoadBeanContext.java +++ b/src/main/java/com/avaje/ebeaninternal/server/loadcontext/DLoadBeanContext.java @@ -58,7 +58,7 @@ public class DLoadBeanContext extends DLoadBaseContext implements LoadBeanContex if (queryProps != null) { queryProps.configureBeanQuery(query); } - if (parent.isUseAutofetchManager()) { + if (parent.isUseAutoTune()) { query.setAutofetch(true); } } diff --git a/src/main/java/com/avaje/ebeaninternal/server/loadcontext/DLoadContext.java b/src/main/java/com/avaje/ebeaninternal/server/loadcontext/DLoadContext.java index f9f1c1870..9a6b3fb3d 100644 --- a/src/main/java/com/avaje/ebeaninternal/server/loadcontext/DLoadContext.java +++ b/src/main/java/com/avaje/ebeaninternal/server/loadcontext/DLoadContext.java @@ -47,7 +47,7 @@ public class DLoadContext implements LoadContext { */ private final String relativePath; private final ObjectGraphOrigin origin; - private final boolean useAutofetchManager; + private final boolean useProfiling; private final Map nodePathMap = new HashMap(); @@ -67,7 +67,7 @@ public class DLoadContext implements LoadContext { this.readOnly = query.isReadOnly(); this.disableLazyLoading = query.isDisableLazyLoading(); this.excludeBeanCache = Boolean.FALSE.equals(query.isUseBeanCache()); - this.useAutofetchManager = query.getAutoFetchManager() != null; + this.useProfiling = query.getProfilingListener() != null; ObjectGraphNode parentNode = query.getParentNode(); if (parentNode != null) { @@ -197,8 +197,8 @@ public class DLoadContext implements LoadContext { return new ObjectGraphNode(origin, path); } - public boolean isUseAutofetchManager() { - return useAutofetchManager; + public boolean isUseAutoTune() { + return useProfiling; } protected String getFullPath(String path) { diff --git a/src/main/java/com/avaje/ebeaninternal/server/loadcontext/DLoadManyContext.java b/src/main/java/com/avaje/ebeaninternal/server/loadcontext/DLoadManyContext.java index 2e726ee94..fc3927b31 100644 --- a/src/main/java/com/avaje/ebeaninternal/server/loadcontext/DLoadManyContext.java +++ b/src/main/java/com/avaje/ebeaninternal/server/loadcontext/DLoadManyContext.java @@ -70,7 +70,7 @@ public class DLoadManyContext extends DLoadBaseContext implements LoadManyContex queryProps.configureBeanQuery(query); } - if (parent.isUseAutofetchManager()) { + if (parent.isUseAutoTune()) { query.setAutofetch(true); } } diff --git a/src/main/java/com/avaje/ebeaninternal/server/query/CQuery.java b/src/main/java/com/avaje/ebeaninternal/server/query/CQuery.java index 1f0518c1c..923d4ebed 100644 --- a/src/main/java/com/avaje/ebeaninternal/server/query/CQuery.java +++ b/src/main/java/com/avaje/ebeaninternal/server/query/CQuery.java @@ -6,7 +6,7 @@ import com.avaje.ebean.bean.*; import com.avaje.ebeaninternal.api.SpiQuery; import com.avaje.ebeaninternal.api.SpiQuery.Mode; import com.avaje.ebeaninternal.api.SpiTransaction; -import com.avaje.ebeaninternal.server.autofetch.AutoFetchManager; +import com.avaje.ebeaninternal.server.autofetch.ProfilingListener; import com.avaje.ebeaninternal.server.core.Message; import com.avaje.ebeaninternal.server.core.OrmQueryRequest; import com.avaje.ebeaninternal.server.core.SpiOrmQueryRequest; @@ -149,13 +149,13 @@ public class CQuery implements DbReadContext, CancelableQuery { private final Mode queryMode; - private final boolean autoFetchProfiling; + private final boolean autoTuneProfiling; private final ObjectGraphNode objectGraphNode; - private final AutoFetchManager autoFetchManager; + private final ProfilingListener profilingListener; - private final WeakReference autoFetchManagerRef; + private final WeakReference profilingListenerRef; private final Boolean readOnly; @@ -176,11 +176,10 @@ public class CQuery implements DbReadContext, CancelableQuery { this.readOnly = request.isReadOnly(); - this.autoFetchManager = query.getAutoFetchManager(); - this.autoFetchProfiling = autoFetchManager != null; this.objectGraphNode = query.getParentNode(); - this.autoFetchManagerRef = autoFetchProfiling ? new WeakReference( - autoFetchManager) : null; + this.profilingListener = query.getProfilingListener(); + this.autoTuneProfiling = profilingListener != null; + this.profilingListenerRef = autoTuneProfiling ? new WeakReference(profilingListener) : null; // set the generated sql back to the query // so its available to the user... @@ -518,8 +517,8 @@ public class CQuery implements DbReadContext, CancelableQuery { long exeNano = System.nanoTime() - startNano; executionTimeMicros = TimeUnit.NANOSECONDS.toMicros(exeNano); - if (autoFetchProfiling) { - autoFetchManager.collectQueryInfo(objectGraphNode, loadedBeanCount, executionTimeMicros); + if (autoTuneProfiling) { + profilingListener.collectQueryInfo(objectGraphNode, loadedBeanCount, executionTimeMicros); } queryPlan.executionTime(loadedBeanCount, executionTimeMicros, objectGraphNode); @@ -649,7 +648,7 @@ public class CQuery implements DbReadContext, CancelableQuery { // need query.isProfiling() because we just take the data // from the lazy loaded or refreshed beans and put it into the already // existing beans which are already collecting usage information - return autoFetchProfiling && query.isUsageProfiling(); + return autoTuneProfiling && query.isUsageProfiling(); } private String getPath(String propertyName) { @@ -671,8 +670,7 @@ public class CQuery implements DbReadContext, CancelableQuery { public void profileBean(EntityBeanIntercept ebi, String prefix) { ObjectGraphNode node = request.getGraphContext().getObjectGraphNode(prefix); - - ebi.setNodeUsageCollector(new NodeUsageCollector(node, autoFetchManagerRef)); + ebi.setNodeUsageCollector(new NodeUsageCollector(node, profilingListenerRef)); } public void setCurrentPrefix(String currentPrefix, Map currentPathMap) { diff --git a/src/main/java/com/avaje/ebeaninternal/server/querydefn/DefaultOrmQuery.java b/src/main/java/com/avaje/ebeaninternal/server/querydefn/DefaultOrmQuery.java index db2dfb342..b8543b0e1 100644 --- a/src/main/java/com/avaje/ebeaninternal/server/querydefn/DefaultOrmQuery.java +++ b/src/main/java/com/avaje/ebeaninternal/server/querydefn/DefaultOrmQuery.java @@ -1,14 +1,5 @@ package com.avaje.ebeaninternal.server.querydefn; -import java.sql.Timestamp; -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Set; - -import javax.persistence.PersistenceException; - import com.avaje.ebean.*; import com.avaje.ebean.OrderBy.Property; import com.avaje.ebean.bean.BeanCollectionTouched; @@ -26,7 +17,7 @@ import com.avaje.ebeaninternal.api.ManyWhereJoins; import com.avaje.ebeaninternal.api.SpiExpression; import com.avaje.ebeaninternal.api.SpiExpressionList; import com.avaje.ebeaninternal.api.SpiQuery; -import com.avaje.ebeaninternal.server.autofetch.AutoFetchManager; +import com.avaje.ebeaninternal.server.autofetch.ProfilingListener; import com.avaje.ebeaninternal.server.deploy.BeanDescriptor; import com.avaje.ebeaninternal.server.deploy.BeanPropertyAssocMany; import com.avaje.ebeaninternal.server.deploy.DRawSqlSelect; @@ -36,6 +27,14 @@ import com.avaje.ebeaninternal.server.expression.SimpleExpression; import com.avaje.ebeaninternal.server.query.CancelableQuery; import com.avaje.ebeaninternal.util.DefaultExpressionList; +import javax.persistence.PersistenceException; +import java.sql.Timestamp; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; + /** * Default implementation of an Object Relational query. */ @@ -57,7 +56,7 @@ public class DefaultOrmQuery implements SpiQuery { */ private transient TableJoin includeTableJoin; - private transient AutoFetchManager autoFetchManager; + private transient ProfilingListener profilingListener; private transient BeanDescriptor beanDescriptor; @@ -481,7 +480,7 @@ public class DefaultOrmQuery implements SpiQuery { DefaultOrmQuery copy = new DefaultOrmQuery(beanType, server, expressionFactory, (String) null); copy.name = name; copy.includeTableJoin = includeTableJoin; - copy.autoFetchManager = autoFetchManager; + copy.profilingListener = profilingListener; copy.query = query; copy.additionalWhere = additionalWhere; @@ -616,19 +615,23 @@ public class DefaultOrmQuery implements SpiQuery { return this; } + @Override public DefaultOrmQuery setForUpdate(boolean forUpdate) { this.forUpdate = forUpdate; return this; } - public AutoFetchManager getAutoFetchManager() { - return autoFetchManager; + @Override + public ProfilingListener getProfilingListener() { + return profilingListener; } - public void setAutoFetchManager(AutoFetchManager autoFetchManager) { - this.autoFetchManager = autoFetchManager; + @Override + public void setProfilingListener(ProfilingListener profilingListener) { + this.profilingListener = profilingListener; } + @Override public Mode getMode() { return mode; } diff --git a/src/test/java/com/avaje/ebeaninternal/api/TDSpiEbeanServer.java b/src/test/java/com/avaje/ebeaninternal/api/TDSpiEbeanServer.java index 9304cf8ec..adbcc84d8 100644 --- a/src/test/java/com/avaje/ebeaninternal/api/TDSpiEbeanServer.java +++ b/src/test/java/com/avaje/ebeaninternal/api/TDSpiEbeanServer.java @@ -13,7 +13,6 @@ import com.avaje.ebean.meta.MetaInfoManager; import com.avaje.ebean.plugin.SpiServer; import com.avaje.ebean.text.csv.CsvReader; import com.avaje.ebean.text.json.JsonContext; -import com.avaje.ebeaninternal.server.autofetch.AutoFetchManager; import com.avaje.ebeaninternal.server.core.SpiOrmQueryRequest; import com.avaje.ebeaninternal.server.deploy.BeanDescriptor; import com.avaje.ebeaninternal.server.query.CQuery; @@ -86,11 +85,6 @@ public class TDSpiEbeanServer implements SpiEbeanServer { return null; } - @Override - public AutoFetchManager getAutoFetchManager() { - return null; - } - @Override public void clearQueryStatistics() { diff --git a/src/test/java/com/avaje/ebeaninternal/server/autofetch/TunedQueryInfoTest.java b/src/test/java/com/avaje/ebeaninternal/server/autofetch/TunedQueryInfoTest.java index b66df26fb..eaf1ec8be 100644 --- a/src/test/java/com/avaje/ebeaninternal/server/autofetch/TunedQueryInfoTest.java +++ b/src/test/java/com/avaje/ebeaninternal/server/autofetch/TunedQueryInfoTest.java @@ -15,7 +15,7 @@ import com.avaje.ebean.bean.EntityBean; import com.avaje.ebean.bean.EntityBeanIntercept; import com.avaje.ebean.cache.ServerCacheManager; import com.avaje.ebeaninternal.api.SpiQuery; -import com.avaje.ebeaninternal.server.autofetch.TunedQueryInfo; +import com.avaje.ebeaninternal.server.autofetch.service.TunedQueryInfo; import com.avaje.ebeaninternal.server.querydefn.OrmQueryDetail; import com.avaje.tests.model.basic.Order; import com.avaje.tests.model.basic.ResetBasicData; diff --git a/src/test/java/com/avaje/tests/autofetch/AutofetchEmbeddedTest.java b/src/test/java/com/avaje/tests/autofetch/AutofetchEmbeddedTest.java index 2a201f256..ba59ae6cb 100644 --- a/src/test/java/com/avaje/tests/autofetch/AutofetchEmbeddedTest.java +++ b/src/test/java/com/avaje/tests/autofetch/AutofetchEmbeddedTest.java @@ -37,30 +37,30 @@ public class AutofetchEmbeddedTest extends BaseTestCase { @Test public void testEmbeddedBeanQueryTuning() { - Ebean.getServer(null).getAdminAutofetch().setProfiling(true); - Ebean.getServer(null).getAdminAutofetch().setQueryTuning(true); - Ebean.getServer(null).getAdminAutofetch().setProfilingBase(1); - - EMain testBean = new EMain(); - testBean.setName("test"); - testBean.getEmbeddable().setDescription("test description"); - Ebean.save(testBean); - - //This should not throw an exception - for (int i = 0; i < 5; i++) { - Ebean.beginTransaction(); - try { - List result = Ebean.find(EMain.class).setAutofetch(true).findList(); - for (EMain e : result) { - e.getEmbeddable().setDescription("Test" + i); - Ebean.save(e); - } - Ebean.commitTransaction(); - } finally { - Ebean.endTransaction(); - logger.debug(Ebean.getServer(null).getAdminAutofetch().collectUsageViaGC()); - } - } +// Ebean.getServer(null).getAdminAutofetch().setProfiling(true); +// Ebean.getServer(null).getAdminAutofetch().setQueryTuning(true); +// Ebean.getServer(null).getAdminAutofetch().setProfilingBase(1); +// +// EMain testBean = new EMain(); +// testBean.setName("test"); +// testBean.getEmbeddable().setDescription("test description"); +// Ebean.save(testBean); +// +// //This should not throw an exception +// for (int i = 0; i < 5; i++) { +// Ebean.beginTransaction(); +// try { +// List result = Ebean.find(EMain.class).setAutofetch(true).findList(); +// for (EMain e : result) { +// e.getEmbeddable().setDescription("Test" + i); +// Ebean.save(e); +// } +// Ebean.commitTransaction(); +// } finally { +// Ebean.endTransaction(); +// logger.debug(Ebean.getServer(null).getAdminAutofetch().collectUsageViaGC()); +// } +// } } @Test diff --git a/src/test/java/com/avaje/tests/cache/TestL2CacheWithSharedBean.java b/src/test/java/com/avaje/tests/cache/TestL2CacheWithSharedBean.java index 381da8236..a31184917 100644 --- a/src/test/java/com/avaje/tests/cache/TestL2CacheWithSharedBean.java +++ b/src/test/java/com/avaje/tests/cache/TestL2CacheWithSharedBean.java @@ -8,7 +8,7 @@ import com.avaje.ebean.Ebean; import com.avaje.ebean.Query; import com.avaje.ebean.cache.ServerCache; import com.avaje.ebeaninternal.api.SpiQuery; -import com.avaje.ebeaninternal.server.autofetch.TunedQueryInfo; +import com.avaje.ebeaninternal.server.autofetch.service.TunedQueryInfo; import com.avaje.ebeaninternal.server.querydefn.OrmQueryDetail; import com.avaje.tests.model.basic.FeatureDescription; diff --git a/src/test/java/com/avaje/tests/query/TestAutofetchTuneWithJoin.java b/src/test/java/com/avaje/tests/query/TestAutofetchTuneWithJoin.java index 0693038fe..f9fcfe399 100644 --- a/src/test/java/com/avaje/tests/query/TestAutofetchTuneWithJoin.java +++ b/src/test/java/com/avaje/tests/query/TestAutofetchTuneWithJoin.java @@ -28,8 +28,11 @@ public class TestAutofetchTuneWithJoin extends BaseTestCase { ResetBasicData.reset(); - Query q = Ebean.find(Order.class).setAutofetch(true).fetch("customer") - .fetch("customer.contacts").where().lt("id", 3).query(); + Query q = Ebean.find(Order.class) + .setAutofetch(true) + .fetch("customer") + .fetch("customer.contacts") + .where().lt("id", 3).query(); List list = q.findList(); diff --git a/src/test/resources/ebean.properties b/src/test/resources/ebean.properties index 86562c599..10323b913 100644 --- a/src/test/resources/ebean.properties +++ b/src/test/resources/ebean.properties @@ -11,14 +11,15 @@ ebean.encryptKeyManager=com.avaje.tests.basic.encrypt.BasicEncyptKeyManager -ebean.autofetch.querytuning=false -ebean.autofetch.profiling=false +ebean.autofetch.querytuning=true +ebean.autofetch.profiling=true ebean.autofetch.implicitmode=default_off #ebean.autofetch.implicitmode=default_onifempty ebean.autofetch.profiling.min=1 ebean.autofetch.profiling.base=10 #ebean.autofetch.profiling.rate=0.05 -ebean.autofetch.traceUsageCollection=false +ebean.autofetch.garbageCollectionOnShutdown=true +ebean.autofetch.traceUsageCollection=true ebean.ddl.generate=true diff --git a/src/test/resources/logback-test.xml b/src/test/resources/logback-test.xml index b52efc14d..39dcc78bd 100644 --- a/src/test/resources/logback-test.xml +++ b/src/test/resources/logback-test.xml @@ -42,6 +42,8 @@ + + From 5c1de0a886c5b4ece437f6e74de698eb12ff7608 Mon Sep 17 00:00:00 2001 From: Robin Bygrave Date: Wed, 2 Sep 2015 09:28:17 +1200 Subject: [PATCH 2/9] WIP Autotune refactor --- .../server/autofetch/AutoTuneCollection.java | 101 ++++++++++ .../server/autofetch/AutoTuneStorage.java | 17 ++ .../server/autofetch/ProfilingListener.java | 12 ++ .../service/AutoTuneServiceFactory.java | 86 +------- .../service/BaseAutoTuneService.java | 26 +-- .../autofetch/service/BaseQueryTuner.java | 46 +---- .../DefaultAutoFetchManagerLogging.java | 67 ------- .../autofetch/service/ProfileManager.java | 51 ++--- .../autofetch/service/ProfileOrigin.java | 186 +++++++++--------- .../service/ProfileOriginNodeUsage.java | 18 +- .../autofetch/service/ProfileOriginQuery.java | 29 +-- .../autofetch/service/TunedQueryInfo.java | 136 +------------ .../server/core/InternalConfiguration.java | 2 +- .../server/querydefn/OrmQueryDetail.java | 2 +- .../server/autofetch/TunedQueryInfoTest.java | 20 +- .../cache/TestL2CacheWithSharedBean.java | 4 +- 16 files changed, 307 insertions(+), 496 deletions(-) create mode 100644 src/main/java/com/avaje/ebeaninternal/server/autofetch/AutoTuneCollection.java create mode 100644 src/main/java/com/avaje/ebeaninternal/server/autofetch/AutoTuneStorage.java delete mode 100644 src/main/java/com/avaje/ebeaninternal/server/autofetch/service/DefaultAutoFetchManagerLogging.java diff --git a/src/main/java/com/avaje/ebeaninternal/server/autofetch/AutoTuneCollection.java b/src/main/java/com/avaje/ebeaninternal/server/autofetch/AutoTuneCollection.java new file mode 100644 index 000000000..e26b892f1 --- /dev/null +++ b/src/main/java/com/avaje/ebeaninternal/server/autofetch/AutoTuneCollection.java @@ -0,0 +1,101 @@ +package com.avaje.ebeaninternal.server.autofetch; + +import com.avaje.ebean.bean.ObjectGraphOrigin; +import com.avaje.ebeaninternal.server.querydefn.OrmQueryDetail; + +import java.util.ArrayList; +import java.util.List; + +/** + * Profiling information collected. + */ +public class AutoTuneCollection { + + List entries = new ArrayList(); + + public Entry add(ObjectGraphOrigin origin, OrmQueryDetail detail) { + Entry entry = new Entry(origin, detail); + entries.add(entry); + return entry; + } + + /** + * Profiling entry at a given origin point. + */ + public static class Entry { + + /** + * Profiling origin point. + */ + private final ObjectGraphOrigin origin; + + /** + * The tuned query detail. + */ + private final OrmQueryDetail detail; + + /** + * Summary execution statistics for queries related to this origin point. + */ + private final List queries = new ArrayList(); + + public Entry(ObjectGraphOrigin origin, OrmQueryDetail detail) { + this.origin = origin; + this.detail = detail; + } + + public void addQuery(EntryQuery entryQuery) { + queries.add(entryQuery); + } + + public ObjectGraphOrigin getOrigin() { + return origin; + } + + public OrmQueryDetail getDetail() { + return detail; + } + + public List getQueries() { + return queries; + } + + } + + /** + * Summary query execution statistics for the origin point. + */ + public static class EntryQuery { + + final String path; + final long exeCount; + final long totalBeanLoaded; + final long totalMicros; + + public EntryQuery(String path, long exeCount, long totalBeanLoaded, long totalMicros) { + this.path = path; + this.exeCount = exeCount; + this.totalBeanLoaded = totalBeanLoaded; + this.totalMicros = totalMicros; + } + + /** + * Return the relative path with empty string for the origin query. + */ + public String getPath() { + return path; + } + + public long getExeCount() { + return exeCount; + } + + public long getTotalBeanLoaded() { + return totalBeanLoaded; + } + + public long getTotalMicros() { + return totalMicros; + } + } +} diff --git a/src/main/java/com/avaje/ebeaninternal/server/autofetch/AutoTuneStorage.java b/src/main/java/com/avaje/ebeaninternal/server/autofetch/AutoTuneStorage.java new file mode 100644 index 000000000..831cdeee2 --- /dev/null +++ b/src/main/java/com/avaje/ebeaninternal/server/autofetch/AutoTuneStorage.java @@ -0,0 +1,17 @@ +package com.avaje.ebeaninternal.server.autofetch; + +/** + * + */ +public interface AutoTuneStorage { + + /** + * Load and return the tuning information. + */ + AutoTuneCollection load(); + + /** + * Store the collected profiling information. + */ + void store(AutoTuneCollection profiling); +} diff --git a/src/main/java/com/avaje/ebeaninternal/server/autofetch/ProfilingListener.java b/src/main/java/com/avaje/ebeaninternal/server/autofetch/ProfilingListener.java index 9885f7435..ca496456f 100644 --- a/src/main/java/com/avaje/ebeaninternal/server/autofetch/ProfilingListener.java +++ b/src/main/java/com/avaje/ebeaninternal/server/autofetch/ProfilingListener.java @@ -8,6 +8,18 @@ import com.avaje.ebean.bean.ObjectGraphNode; */ public interface ProfilingListener extends NodeUsageListener { + /** + * Collect summary statistics for a query executed for the given node. + * + * @param node the node relative to the origin point + * @param beans the number of beans loaded by the query + * @param micros the query execution in microseconds + */ void collectQueryInfo(ObjectGraphNode node, long beans, long micros); + /** + * Return true if this request should be profiled (based on the + * profiling ratio and collection count for this origin). + */ + boolean isProfileRequest(ObjectGraphNode origin); } diff --git a/src/main/java/com/avaje/ebeaninternal/server/autofetch/service/AutoTuneServiceFactory.java b/src/main/java/com/avaje/ebeaninternal/server/autofetch/service/AutoTuneServiceFactory.java index 1e6c2d1cd..c0327c4fb 100644 --- a/src/main/java/com/avaje/ebeaninternal/server/autofetch/service/AutoTuneServiceFactory.java +++ b/src/main/java/com/avaje/ebeaninternal/server/autofetch/service/AutoTuneServiceFactory.java @@ -3,94 +3,12 @@ package com.avaje.ebeaninternal.server.autofetch.service; import com.avaje.ebean.config.ServerConfig; import com.avaje.ebeaninternal.api.SpiEbeanServer; import com.avaje.ebeaninternal.server.autofetch.AutoTuneService; -import com.avaje.ebeaninternal.server.resource.ResourceManager; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import javax.persistence.PersistenceException; -import java.io.File; -import java.io.FileInputStream; -import java.io.ObjectInputStream; public class AutoTuneServiceFactory { - private static final Logger logger = LoggerFactory.getLogger(AutoTuneServiceFactory.class); + public static AutoTuneService create(SpiEbeanServer server, ServerConfig serverConfig) { - public static AutoTuneService create(SpiEbeanServer server, ServerConfig serverConfig, ResourceManager resourceManager) { - - AutoTuneServiceFactory me = new AutoTuneServiceFactory(); - return me.createAutoFetchManager(server, serverConfig, resourceManager); - } - - private AutoTuneService createAutoFetchManager(SpiEbeanServer server, ServerConfig serverConfig, ResourceManager resourceManager) { - - AutoTuneService manager = createAutoFetchManager(server.getName(), resourceManager); - manager.setOwner(server, serverConfig); - - return manager; - } - - private AutoTuneService createAutoFetchManager(String serverName, ResourceManager resourceManager) { - - File autoFetchFile = getAutoFetchFile(serverName, resourceManager); - - AutoTuneService autoFetchManager = null; - - boolean readFile = !"false".equalsIgnoreCase(System.getProperty("autofetch.readfromfile")); - if (readFile) { - autoFetchManager = deserializeAutoFetch(autoFetchFile); - } - - if (autoFetchManager == null) { - // not deserialized from file so create as empty - // It will be populated automatically by querying the - // database meta data - autoFetchManager = new BaseAutoTuneService(autoFetchFile.getAbsolutePath()); - } - - return autoFetchManager; - } - - private AutoTuneService deserializeAutoFetch(File autoFetchFile) { - try { - - if (!autoFetchFile.exists()) { - return null; - } - FileInputStream fi = new FileInputStream(autoFetchFile); - ObjectInputStream ois = new ObjectInputStream(fi); - AutoTuneService profListener = (AutoTuneService) ois.readObject(); - ois.close(); - - logger.info("AutoFetch deserialized from file [" + autoFetchFile.getAbsolutePath() + "]"); - - return profListener; - - } catch (Exception ex) { - logger.error("Error loading autofetch file " + autoFetchFile.getAbsolutePath(), ex); - return null; - } - } - - /** - * Return the file name of the autoFetch meta data. - */ - private File getAutoFetchFile(String serverName, ResourceManager resourceManager) { - - String fileName = ".ebean." + serverName + ".autofetch"; - - File dir = resourceManager.getAutofetchDirectory(); - - if (!dir.exists()) { - // automatically create the directory if it does not exist. - // this is probably a fairly reasonable thing to do - if (!dir.mkdirs()) { - String m = "Unable to create directory [" + dir + "] for autofetch file [" + fileName + "]"; - throw new PersistenceException(m); - } - } - - return new File(dir, fileName); + return new BaseAutoTuneService(server, serverConfig); } } diff --git a/src/main/java/com/avaje/ebeaninternal/server/autofetch/service/BaseAutoTuneService.java b/src/main/java/com/avaje/ebeaninternal/server/autofetch/service/BaseAutoTuneService.java index 81ee660b7..9b4923276 100644 --- a/src/main/java/com/avaje/ebeaninternal/server/autofetch/service/BaseAutoTuneService.java +++ b/src/main/java/com/avaje/ebeaninternal/server/autofetch/service/BaseAutoTuneService.java @@ -5,14 +5,10 @@ import com.avaje.ebean.config.ServerConfig; import com.avaje.ebeaninternal.api.SpiEbeanServer; import com.avaje.ebeaninternal.api.SpiQuery; import com.avaje.ebeaninternal.server.autofetch.AutoTuneService; +import com.avaje.ebeaninternal.server.autofetch.AutoTuneCollection; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.File; -import java.io.FileOutputStream; -import java.io.ObjectOutputStream; -import java.io.Serializable; - /** * Implementation of the AutoTuneService which is comprised of profiling and query tuning. */ @@ -26,11 +22,13 @@ public class BaseAutoTuneService implements AutoTuneService { private final BaseQueryTuner queryTuner; + private final ProfileManager profileManager; + public BaseAutoTuneService(SpiEbeanServer server, ServerConfig serverConfig) { AutofetchConfig config = serverConfig.getAutofetchConfig(); - ProfileManager profileManager = new ProfileManager(config, server); + this.profileManager = new ProfileManager(config, server); this.queryTuner = new BaseQueryTuner(config, server, profileManager); this.garbageCollectionOnShutdown = config.isGarbageCollectionOnShutdown(); @@ -42,24 +40,12 @@ public class BaseAutoTuneService implements AutoTuneService { */ public void startup() { -// File autoFetchFile = new File(fileName); -// -// try { -// FileOutputStream fout = new FileOutputStream(autoFetchFile); -// -// ObjectOutputStream oout = new ObjectOutputStream(fout); -// oout.writeObject(this); -// oout.flush(); -// oout.close(); -// -// } catch (Exception e) { -// String msg = "Error serializing autofetch file"; -// logging.logError(msg, e); -// } } private void saveProfiling() { + AutoTuneCollection autoTuneCollection = profileManager.profilingCollection(false); + } /** diff --git a/src/main/java/com/avaje/ebeaninternal/server/autofetch/service/BaseQueryTuner.java b/src/main/java/com/avaje/ebeaninternal/server/autofetch/service/BaseQueryTuner.java index a60f73fd9..6ad1fe2c0 100644 --- a/src/main/java/com/avaje/ebeaninternal/server/autofetch/service/BaseQueryTuner.java +++ b/src/main/java/com/avaje/ebeaninternal/server/autofetch/service/BaseQueryTuner.java @@ -19,16 +19,6 @@ public class BaseQueryTuner { private final boolean queryTuning; - /** - * Converted from a 0-100 int to a double. Effectively a percentage rate at - * which to collect profiling information. - */ - private final double profilingRate; - - private final int profilingBase; - - private final int profilingMin; - private boolean profiling; private final AutofetchMode mode; @@ -49,9 +39,6 @@ public class BaseQueryTuner { this.mode = config.getMode(); this.queryTuning = config.isQueryTuning(); this.profiling = config.isProfiling(); - this.profilingRate = config.getProfilingRate(); - this.profilingBase = config.getProfilingBase(); - this.profilingMin = config.getProfilingMin(); } /** @@ -70,13 +57,12 @@ public class BaseQueryTuner { return false; } - if (!useAutoFetch(query)) { + if (!useAutoTune(query)) { // not using autoFetch for this query return false; } - ObjectGraphNode parentAutoFetchNode = query.getParentNode(); - if (parentAutoFetchNode != null) { + if (query.getParentNode() != null) { // This is a +lazy/+query query with profiling on. // We continue to collect the profiling information. query.setProfilingListener(profilingListener); @@ -87,37 +73,25 @@ public class BaseQueryTuner { CallStack stack = server.createCallStack(); ObjectGraphNode origin = query.setOrigin(stack); - // get current "tuned fetch" for this query point - TunedQueryInfo tunedFetch = tunedQueryInfoMap.get(origin.getOriginQueryPoint().getKey()); - - // get the number of times we have collected profiling information - int profileCount = tunedFetch == null ? 0 : tunedFetch.getProfileCount(); - if (profiling) { - // we want more profiling information? - if (tunedFetch == null) { - query.setProfilingListener(profilingListener); - - } else if (profileCount < profilingBase) { - query.setProfilingListener(profilingListener); - - } else if (tunedFetch.isPercentageProfile(profilingRate)) { + if (profilingListener.isProfileRequest(origin)) { + // collect more profiling based on profiling rate etc query.setProfilingListener(profilingListener); } } - if (queryTuning && tunedFetch != null && profileCount >= profilingMin) { - // deemed to have enough profiling information for automatic tuning - return tunedFetch.autoFetchTune(query); + if (queryTuning) { + // get current "tuned fetch" for this query point + TunedQueryInfo tuneInfo = tunedQueryInfoMap.get(origin.getOriginQueryPoint().getKey()); + return tuneInfo != null && tuneInfo.tuneQuery(query); } - return false; } /** - * Return true if we should try to use autoFetch for this query. + * Return true if we should try to tune this query. */ - private boolean useAutoFetch(SpiQuery query) { + private boolean useAutoTune(SpiQuery query) { if (query.isLoadBeanCache()) { // when loading the cache don't tune the query diff --git a/src/main/java/com/avaje/ebeaninternal/server/autofetch/service/DefaultAutoFetchManagerLogging.java b/src/main/java/com/avaje/ebeaninternal/server/autofetch/service/DefaultAutoFetchManagerLogging.java deleted file mode 100644 index 8ba42bb6b..000000000 --- a/src/main/java/com/avaje/ebeaninternal/server/autofetch/service/DefaultAutoFetchManagerLogging.java +++ /dev/null @@ -1,67 +0,0 @@ -package com.avaje.ebeaninternal.server.autofetch.service; - -import com.avaje.ebean.config.ServerConfig; -import com.avaje.ebeaninternal.api.SpiEbeanServer; -import com.avaje.ebeaninternal.server.querydefn.OrmQueryDetail; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.concurrent.TimeUnit; - -/** - * Handles the logging aspects for the DefaultAutoFetchListener. - *

- * Note that java util logging loggers generally should not be serialised and - * that is one of the main reasons for pulling out the logging to this class. - *

- */ -public class DefaultAutoFetchManagerLogging { - - private static final Logger logger = LoggerFactory.getLogger(DefaultAutoFetchManagerLogging.class); - - private final BaseAutoTuneService manager; - - private final int updateFreqInSecs; - - public DefaultAutoFetchManagerLogging(ServerConfig serverConfig, BaseAutoTuneService profileListener) { - - this.manager = profileListener; - this.updateFreqInSecs = serverConfig.getAutofetchConfig().getProfileUpdateFrequency(); - } - - public void init(SpiEbeanServer ebeanServer) { - ebeanServer.getBackgroundExecutor().executePeriodically(new UpdateProfile(), updateFreqInSecs, TimeUnit.SECONDS); - } - - private final class UpdateProfile implements Runnable { - public void run() { - manager.updateTunedQueryInfo(); - } - } - - public void logInfo(String msg, Throwable e) { - logger.info(msg, e); - } - - public void logError(String msg, Throwable e) { - logger.error(msg, e); - } - - public void logSummary(String summaryInfo) { - - String msg = "\"Summary\",\"" + summaryInfo + "\",,,,"; - logger.debug(msg); - } - - public void logChanged(TunedQueryInfo tunedFetch, OrmQueryDetail newQueryDetail) { - - String msg = tunedFetch.getLogOutput(newQueryDetail); - logger.debug(msg); - } - - public void logNew(TunedQueryInfo tunedFetch) { - - String msg = tunedFetch.getLogOutput(null); - logger.debug(msg); - } -} diff --git a/src/main/java/com/avaje/ebeaninternal/server/autofetch/service/ProfileManager.java b/src/main/java/com/avaje/ebeaninternal/server/autofetch/service/ProfileManager.java index a9bada622..b76e66e47 100644 --- a/src/main/java/com/avaje/ebeaninternal/server/autofetch/service/ProfileManager.java +++ b/src/main/java/com/avaje/ebeaninternal/server/autofetch/service/ProfileManager.java @@ -5,9 +5,9 @@ import com.avaje.ebean.bean.ObjectGraphNode; import com.avaje.ebean.bean.ObjectGraphOrigin; import com.avaje.ebean.config.AutofetchConfig; import com.avaje.ebeaninternal.api.SpiEbeanServer; +import com.avaje.ebeaninternal.server.autofetch.AutoTuneCollection; import com.avaje.ebeaninternal.server.autofetch.ProfilingListener; import com.avaje.ebeaninternal.server.deploy.BeanDescriptor; -import com.avaje.ebeaninternal.server.querydefn.OrmQueryDetail; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -23,7 +23,13 @@ public class ProfileManager implements ProfilingListener { private final boolean queryTuningAddVersion; - private final boolean profiling; + /** + * Converted from a 0-100 int to a double. Effectively a percentage rate at + * which to collect profiling information. + */ + private final double profilingRate; + + private final int profilingBase; /** * Map of the usage and query statistics gathered. @@ -36,10 +42,18 @@ public class ProfileManager implements ProfilingListener { public ProfileManager(AutofetchConfig config, SpiEbeanServer server) { this.server = server; - this.profiling = config.isProfiling(); + this.profilingRate = config.getProfilingRate(); + this.profilingBase = config.getProfilingBase(); this.queryTuningAddVersion = config.isQueryTuningAddVersion(); } + @Override + public boolean isProfileRequest(ObjectGraphNode origin) { + + ProfileOrigin profileOrigin = profileMap.get(origin.getOriginQueryPoint().getKey()); + return profileOrigin == null || profileOrigin.isProfile(); + } + /** * Gather query execution statistics. This could either be the originating * query in which case the parentNode will be null, or a lazy loading query @@ -73,40 +87,29 @@ public class ProfileManager implements ProfilingListener { synchronized (monitor) { ProfileOrigin stats = profileMap.get(originQueryPoint.getKey()); if (stats == null) { - stats = new ProfileOrigin(originQueryPoint, queryTuningAddVersion); + stats = new ProfileOrigin(originQueryPoint, queryTuningAddVersion, profilingBase, profilingRate); profileMap.put(originQueryPoint.getKey(), stats); } return stats; } } - /** - * Update the tuned fetch plans from the current usage information. + * Collect all the profiling information. */ - public void updateTunedQueryInfo() { + public AutoTuneCollection profilingCollection(boolean reset) { - if (!profiling) { - // we are not collecting any profiling information at - // the moment so don't try updating the tuned query plans. - return;// "Not profiling"; - } + AutoTuneCollection req = new AutoTuneCollection(); - synchronized (monitor) { + for (ProfileOrigin origin : profileMap.values()) { - for (ProfileOrigin origin : profileMap.values()) { - if (origin.hasUsage()) { - OrmQueryDetail ormQueryDetail = updateTunedQueryFromUsage(origin); - - } + BeanDescriptor desc = server.getBeanDescriptorById(origin.getOrigin().getBeanType()); + if (desc != null) { + origin.profilingCollection(desc, req, reset); } } + + return req; } - - private OrmQueryDetail updateTunedQueryFromUsage(ProfileOrigin statistics) { - - BeanDescriptor desc = server.getBeanDescriptorById(statistics.getOrigin().getBeanType()); - return desc == null ? null : statistics.buildTunedFetch(desc); - } } diff --git a/src/main/java/com/avaje/ebeaninternal/server/autofetch/service/ProfileOrigin.java b/src/main/java/com/avaje/ebeaninternal/server/autofetch/service/ProfileOrigin.java index 959b740a2..84f758c0b 100644 --- a/src/main/java/com/avaje/ebeaninternal/server/autofetch/service/ProfileOrigin.java +++ b/src/main/java/com/avaje/ebeaninternal/server/autofetch/service/ProfileOrigin.java @@ -5,106 +5,127 @@ import com.avaje.ebean.bean.ObjectGraphNode; import com.avaje.ebean.bean.ObjectGraphOrigin; import com.avaje.ebean.text.PathProperties; import com.avaje.ebean.text.PathProperties.Props; +import com.avaje.ebeaninternal.server.autofetch.AutoTuneCollection; import com.avaje.ebeaninternal.server.deploy.BeanDescriptor; import com.avaje.ebeaninternal.server.querydefn.OrmQueryDetail; -import java.io.Serializable; import java.util.Collection; -import java.util.LinkedHashMap; import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicLong; -public class ProfileOrigin implements Serializable { +public class ProfileOrigin { - - private static final long serialVersionUID = -5586783791097230766L; + private static final long RESET_COUNT = -1000000000L; private final ObjectGraphOrigin origin; private final boolean queryTuningAddVersion; - private int counter; + private final int profilingBase; - private final Map queryStatsMap = new LinkedHashMap(); + private final double profilingRate; - private final Map nodeUsageMap = new LinkedHashMap(); + private final Map queryStatsMap = new ConcurrentHashMap(); - @SuppressWarnings("RedundantStringConstructorCall") - private final String monitor = new String(); + private final Map nodeUsageMap = new ConcurrentHashMap(); - public ProfileOrigin(ObjectGraphOrigin origin, boolean queryTuningAddVersion) { + private final Object monitor = new Object(); + + private final AtomicLong requestCount = new AtomicLong(); + + private final AtomicLong profileCount = new AtomicLong(); + + public ProfileOrigin(ObjectGraphOrigin origin, boolean queryTuningAddVersion, int profilingBase, double profilingRate) { this.origin = origin; this.queryTuningAddVersion = queryTuningAddVersion; + this.profilingBase = profilingBase; + this.profilingRate = profilingRate; } + /** + * Return true if this query should be profiled based on a percentage rate. + */ + public boolean isProfile() { + + long count = requestCount.incrementAndGet(); + if (count < profilingBase) { + return true; + } + long hits = profileCount.get(); + if (profilingRate > (double) hits / count) { + profileCount.incrementAndGet(); + return true; + } else { + return false; + } + } + + /** + * Collect profiling information with the option to reset the underlying profiling detail. + */ + public void profilingCollection(BeanDescriptor rootDesc, AutoTuneCollection req, boolean reset) { + + synchronized (monitor) { + if (nodeUsageMap.isEmpty()) { + return; + } + + OrmQueryDetail detail = buildDetail(rootDesc); + AutoTuneCollection.Entry entry = req.add(origin, detail); + + Collection values = queryStatsMap.values(); + for (ProfileOriginQuery queryEntry : values) { + entry.addQuery(queryEntry.createEntryQuery(reset)); + } + if (reset) { + nodeUsageMap.clear(); + if (requestCount.get() > RESET_COUNT) { + requestCount.set(profilingBase); + profileCount.set(0); + } + } + } + } + + private OrmQueryDetail buildDetail(BeanDescriptor rootDesc) { + PathProperties pathProps = new PathProperties(); + + for (ProfileOriginNodeUsage statsNode : nodeUsageMap.values()) { + statsNode.buildTunedFetch(pathProps, rootDesc); + } + + OrmQueryDetail detail = new OrmQueryDetail(); + + Collection pathProperties = pathProps.getPathProps(); + for (Props props : pathProperties) { + if (!props.isEmpty()) { + detail.addFetch(props.getPath(), props.getPropertiesAsString(), null); + } + } + + detail.sortFetchPaths(rootDesc); + return detail; + } + + /** + * Return the origin. + */ public ObjectGraphOrigin getOrigin() { return origin; } -// public TunedQueryInfo createTunedFetch(OrmQueryDetail newFetchDetail) { -// synchronized (monitor) { -// // NB: create a copy of queryPoint allowing garbage -// // collection of source... -// return new TunedQueryInfo(origin, newFetchDetail, counter); -// } -// } - /** - * Return the number of times the root query has executed. + * Collect query execution summary statistics. *

- * This tells us how much profiling we have done for this query. - * For example, after 100 times we may stop collecting more profiling info. + * This can give us a quick overview into bad lazy loading areas etc. *

*/ - public int getCounter() { - return counter; - } - - /** - * Return true if this has usage statistics. - */ - public boolean hasUsage() { - synchronized (monitor) { - return !nodeUsageMap.isEmpty(); - } - } - - public OrmQueryDetail buildTunedFetch(BeanDescriptor rootDesc) { - - synchronized (monitor) { - if (nodeUsageMap.isEmpty()) { - return null; - } - - PathProperties pathProps = new PathProperties(); - - for (ProfileOriginNodeUsage statsNode : nodeUsageMap.values()) { - statsNode.buildTunedFetch(pathProps, rootDesc); - } - - OrmQueryDetail detail = new OrmQueryDetail(); - - Collection pathProperties = pathProps.getPathProps(); - for (Props props : pathProperties) { - if (!props.isEmpty()) { - detail.addFetch(props.getPath(), props.getPropertiesAsString(), null); - } - } - - detail.sortFetchPaths(rootDesc); - return detail; - } - } - - public void collectQueryInfo(ObjectGraphNode node, long beansLoaded, long micros) { String key = node.getPath(); if (key == null) { key = ""; - // this is basically the number of times the root query - // has executed which gives us an indication of how - // much profiling information we have gathered. - counter++; } ProfileOriginQuery stats = queryStatsMap.get(key); @@ -116,7 +137,6 @@ public class ProfileOrigin implements Serializable { stats.add(beansLoaded, micros); } - /** * Collect the usage information for from a instance for this node. */ @@ -126,7 +146,7 @@ public class ProfileOrigin implements Serializable { ObjectGraphNode node = profile.getNode(); ProfileOriginNodeUsage nodeStats = getNodeStats(node.getPath()); - nodeStats.publish(profile); + nodeStats.collectUsageInfo(profile); } } @@ -142,32 +162,4 @@ public class ProfileOrigin implements Serializable { } } -// public String getUsageDebug() { -// synchronized (monitor) { -// StringBuilder sb = new StringBuilder(); -// sb.append("root[").append(origin.getBeanType()).append("] "); -// for (ProfileOriginNodeUsage node : nodeUsageMap.values()) { -// sb.append(node.toString()).append("\n"); -// } -// return sb.toString(); -// } -// } -// -// public String getQueryStatDebug() { -// synchronized (monitor) { -// StringBuilder sb = new StringBuilder(); -// for (ProfileOriginQuery queryStat : queryStatsMap.values()) { -// sb.append(queryStat.toString()).append("\n"); -// } -// return sb.toString(); -// } -// } -// -// public String toString() { -// -// synchronized (monitor) { -// return getUsageDebug(); -// } -// } - } diff --git a/src/main/java/com/avaje/ebeaninternal/server/autofetch/service/ProfileOriginNodeUsage.java b/src/main/java/com/avaje/ebeaninternal/server/autofetch/service/ProfileOriginNodeUsage.java index a051bc31f..632ac8097 100644 --- a/src/main/java/com/avaje/ebeaninternal/server/autofetch/service/ProfileOriginNodeUsage.java +++ b/src/main/java/com/avaje/ebeaninternal/server/autofetch/service/ProfileOriginNodeUsage.java @@ -10,22 +10,17 @@ import com.avaje.ebeaninternal.server.query.SplitName; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.Serializable; -import java.util.HashSet; import java.util.LinkedHashSet; import java.util.Set; /** * Collects usages statistics for a given node in the object graph. */ -public class ProfileOriginNodeUsage implements Serializable { - - private static final long serialVersionUID = -1663951463963779547L; +public class ProfileOriginNodeUsage { private static final Logger logger = LoggerFactory.getLogger(ProfileOriginNodeUsage.class); - @SuppressWarnings("RedundantStringConstructorCall") - private final String monitor = new String(); + private final Object monitor = new Object(); private final String path; @@ -53,7 +48,7 @@ public class ProfileOriginNodeUsage implements Serializable { ElPropertyValue elGetValue = rootDesc.getElGetValue(path); if (elGetValue == null) { desc = null; - logger.warn("Autofetch: Can't find join for path[" + path + "] for " + rootDesc.getName()); + logger.warn("AutoTune: Can't find join for path[" + path + "] for " + rootDesc.getName()); } else { BeanProperty beanProperty = elGetValue.getBeanProperty(); @@ -66,7 +61,7 @@ public class ProfileOriginNodeUsage implements Serializable { for (String propName : aggregateUsed) { BeanProperty beanProp = desc.getBeanPropertyFromPath(propName); if (beanProp == null) { - logger.warn("Autofetch: Can't find property[" + propName + "] for " + desc.getName()); + logger.warn("AutoTune: Can't find property[" + propName + "] for " + desc.getName()); } else { if (beanProp instanceof BeanPropertyAssoc) { @@ -95,7 +90,10 @@ public class ProfileOriginNodeUsage implements Serializable { } } - public void publish(NodeUsageCollector profile) { + /** + * Collect usage from a node. + */ + public void collectUsageInfo(NodeUsageCollector profile) { synchronized (monitor) { diff --git a/src/main/java/com/avaje/ebeaninternal/server/autofetch/service/ProfileOriginQuery.java b/src/main/java/com/avaje/ebeaninternal/server/autofetch/service/ProfileOriginQuery.java index b4d2b6bfa..344d040dc 100644 --- a/src/main/java/com/avaje/ebeaninternal/server/autofetch/service/ProfileOriginQuery.java +++ b/src/main/java/com/avaje/ebeaninternal/server/autofetch/service/ProfileOriginQuery.java @@ -1,9 +1,12 @@ package com.avaje.ebeaninternal.server.autofetch.service; +import com.avaje.ebeaninternal.server.autofetch.AutoTuneCollection; +import com.avaje.ebeaninternal.server.util.LongAdder; + import java.io.Serializable; /** - * Used to accumulate query execution statistics. + * Used to accumulate query execution statistics for paths relative to the origin query. */ public class ProfileOriginQuery implements Serializable { @@ -11,26 +14,30 @@ public class ProfileOriginQuery implements Serializable { private final String path; - private long exeCount; + private final LongAdder exeCount = new LongAdder(); - private long totalBeanLoaded; + private final LongAdder totalBeanLoaded = new LongAdder(); - private long totalMicros; + private final LongAdder totalMicros = new LongAdder(); public ProfileOriginQuery(String path) { this.path = path; } public void add(long beansLoaded, long micros) { - exeCount++; - totalBeanLoaded += beansLoaded; - totalMicros += micros; + exeCount.increment(); + totalBeanLoaded.add(beansLoaded); + totalMicros.add(micros); } - public String toString() { - long avgMicros = exeCount == 0 ? 0 : totalMicros / exeCount; + public AutoTuneCollection.EntryQuery createEntryQuery(boolean reset){ - return "queryExe path[" + path + "] count[" + exeCount + "] totalBeansLoaded[" + totalBeanLoaded + "] avgMicros[" - + avgMicros + "] totalMicros[" + totalMicros + "]"; + if (reset) { + return new AutoTuneCollection.EntryQuery(path, exeCount.sumThenReset(), totalBeanLoaded.sumThenReset(), totalMicros.sumThenReset()); + + } else { + return new AutoTuneCollection.EntryQuery(path, exeCount.sum(), totalBeanLoaded.sum(), totalMicros.sum()); + } } + } \ No newline at end of file diff --git a/src/main/java/com/avaje/ebeaninternal/server/autofetch/service/TunedQueryInfo.java b/src/main/java/com/avaje/ebeaninternal/server/autofetch/service/TunedQueryInfo.java index 8a6a5d046..b1e554cbd 100644 --- a/src/main/java/com/avaje/ebeaninternal/server/autofetch/service/TunedQueryInfo.java +++ b/src/main/java/com/avaje/ebeaninternal/server/autofetch/service/TunedQueryInfo.java @@ -12,90 +12,13 @@ import java.io.Serializable; */ public class TunedQueryInfo implements Serializable { - private static final long serialVersionUID = 7381493228797997282L; - private final ObjectGraphOrigin origin; - /** - * The tuned query details with joins and properties. - */ - private OrmQueryDetail tunedDetail; + private final OrmQueryDetail tunedDetail; - /** - * The number of times profiling has been collected for this query point. - */ - private int profileCount; - - private Long lastTuneTime = (long) 0; - - @SuppressWarnings("RedundantStringConstructorCall") - private final String rateMonitor = new String(); - - /** - * The number of queries tuned by this object. - * Could use AtomicInteger perhaps. - */ - private transient int tunedCount; - - private transient int rateTotal; - - private transient int rateHits; - - private transient double lastRate; - - public TunedQueryInfo(ObjectGraphOrigin queryPoint, OrmQueryDetail tunedDetail, int profileCount) { + public TunedQueryInfo(ObjectGraphOrigin queryPoint, OrmQueryDetail tunedDetail) { this.origin = queryPoint; this.tunedDetail = tunedDetail; - this.profileCount = profileCount; - } - - /** - * Return true if this query should be profiled based on a percentage rate. - */ - public boolean isPercentageProfile(double rate) { - - synchronized (rateMonitor) { - - if (lastRate != rate) { - // the rate has changed so resetting - lastRate = rate; - rateTotal = 0; - rateHits = 0; - } - - rateTotal++; - if (rate > (double) rateHits / rateTotal) { - rateHits++; - return true; - } else { - return false; - } - } - } - - /** - * Set the number of times profiling has been collected for this query - * point. - */ - public void setProfileCount(int profileCount) { - // int assignment is atomic - this.profileCount = profileCount; - } - - /** - * Set the tuned query detail. - */ - public void setTunedDetail(OrmQueryDetail tunedDetail) { - // assignment is atomic - this.tunedDetail = tunedDetail; - this.lastTuneTime = System.currentTimeMillis(); - } - - /** - * Return true if the fetches are essentially the same. - */ - public boolean isSame(OrmQueryDetail newQueryDetail) { - return tunedDetail != null && tunedDetail.isAutoFetchEqual(newQueryDetail); } /** @@ -103,13 +26,12 @@ public class TunedQueryInfo implements Serializable { * * @return true if the query was tuned, otherwise false. */ - public boolean autoFetchTune(SpiQuery query) { + public boolean tuneQuery(SpiQuery query) { if (tunedDetail == null) { return false; } boolean tuned; - //Note: tunedDetail is immutable by convention if (query.isDetailEmpty()) { tuned = true; // tune by 'replacement' @@ -120,62 +42,10 @@ public class TunedQueryInfo implements Serializable { } if (tuned) { query.setAutoFetchTuned(true); - // a case for AtomicInteger but good enough for statistics - tunedCount++; } return tuned; } - /** - * Return the time of the last tune. - */ - public Long getLastTuneTime() { - return lastTuneTime; - } - - /** - * Return the number of queries tuned by this object. - */ - public int getTunedCount() { - return tunedCount; - } - - /** - * Return the number of times profiling has been collected for this query - * point. - */ - public int getProfileCount() { - return profileCount; - } - - public OrmQueryDetail getTunedDetail() { - return tunedDetail; - } - - public ObjectGraphOrigin getOrigin() { - return origin; - } - - public String getLogOutput(OrmQueryDetail newQueryDetail) { - - boolean changed = newQueryDetail != null; - - StringBuilder sb = new StringBuilder(150); - sb.append(changed ? "\"Changed\"," : "\"New\","); - sb.append("\"").append(origin.getBeanType()).append("\","); - sb.append("\"").append(origin.getKey()).append("\","); - if (changed) { - sb.append("\"to: ").append(newQueryDetail.toString()).append("\","); - sb.append("\"from: ").append(tunedDetail.toString()).append("\","); - } else { - sb.append("\"to: ").append(tunedDetail.toString()).append("\","); - sb.append("\"\","); - } - sb.append("\"").append(origin.getFirstStackElement()).append("\""); - - return sb.toString(); - } - public String toString() { return origin.getBeanType() + " " + origin.getKey() + " " + tunedDetail; } diff --git a/src/main/java/com/avaje/ebeaninternal/server/core/InternalConfiguration.java b/src/main/java/com/avaje/ebeaninternal/server/core/InternalConfiguration.java index c1148d0cf..62c1e2470 100644 --- a/src/main/java/com/avaje/ebeaninternal/server/core/InternalConfiguration.java +++ b/src/main/java/com/avaje/ebeaninternal/server/core/InternalConfiguration.java @@ -236,7 +236,7 @@ public class InternalConfiguration { } public AutoTuneService createAutoFetchManager(SpiEbeanServer server) { - return AutoTuneServiceFactory.create(server, serverConfig, resourceManager); + return AutoTuneServiceFactory.create(server, serverConfig); } public RelationalQueryEngine createRelationalQueryEngine() { diff --git a/src/main/java/com/avaje/ebeaninternal/server/querydefn/OrmQueryDetail.java b/src/main/java/com/avaje/ebeaninternal/server/querydefn/OrmQueryDetail.java index b7a2885d8..a65edfa16 100644 --- a/src/main/java/com/avaje/ebeaninternal/server/querydefn/OrmQueryDetail.java +++ b/src/main/java/com/avaje/ebeaninternal/server/querydefn/OrmQueryDetail.java @@ -82,7 +82,7 @@ public class OrmQueryDetail implements Serializable { /** * Return true if equal in terms of autofetch (select and joins). */ - public boolean isAutoFetchEqual(OrmQueryDetail otherDetail) { + public boolean isAutoTuneEqual(OrmQueryDetail otherDetail) { return autofetchPlanHash() == otherDetail.autofetchPlanHash(); } diff --git a/src/test/java/com/avaje/ebeaninternal/server/autofetch/TunedQueryInfoTest.java b/src/test/java/com/avaje/ebeaninternal/server/autofetch/TunedQueryInfoTest.java index eaf1ec8be..a27881bc5 100644 --- a/src/test/java/com/avaje/ebeaninternal/server/autofetch/TunedQueryInfoTest.java +++ b/src/test/java/com/avaje/ebeaninternal/server/autofetch/TunedQueryInfoTest.java @@ -41,11 +41,11 @@ public class TunedQueryInfoTest extends BaseTestCase { OrmQueryDetail tunedDetail = new OrmQueryDetail(); tunedDetail.select(null); - TunedQueryInfo tunedInfo = new TunedQueryInfo(null, tunedDetail, 0); + TunedQueryInfo tunedInfo = new TunedQueryInfo(null, tunedDetail); Query query = server.find(Order.class).setId(1); - tunedInfo.autoFetchTune((SpiQuery)query); + tunedInfo.tuneQuery((SpiQuery) query); Order order = query.findUnique(); EntityBean eb = (EntityBean)order; @@ -69,11 +69,11 @@ public class TunedQueryInfoTest extends BaseTestCase { OrmQueryDetail tunedDetail = new OrmQueryDetail(); tunedDetail.select(""); - TunedQueryInfo tunedInfo = new TunedQueryInfo(null, tunedDetail, 0); + TunedQueryInfo tunedInfo = new TunedQueryInfo(null, tunedDetail); Query query = server.find(Order.class).setId(1); - tunedInfo.autoFetchTune((SpiQuery)query); + tunedInfo.tuneQuery((SpiQuery) query); Order order = query.findUnique(); EntityBean eb = (EntityBean)order; @@ -96,11 +96,11 @@ public class TunedQueryInfoTest extends BaseTestCase { OrmQueryDetail tunedDetail = new OrmQueryDetail(); tunedDetail.select("somethingThatDoesNotExist"); - TunedQueryInfo tunedInfo = new TunedQueryInfo(null, tunedDetail, 0); + TunedQueryInfo tunedInfo = new TunedQueryInfo(null, tunedDetail); Query query = server.find(Order.class).setId(1); - tunedInfo.autoFetchTune((SpiQuery)query); + tunedInfo.tuneQuery((SpiQuery) query); LoggedSqlCollector.start(); @@ -132,11 +132,11 @@ public class TunedQueryInfoTest extends BaseTestCase { OrmQueryDetail tunedDetail = new OrmQueryDetail(); tunedDetail.select("status, customer"); - TunedQueryInfo tunedInfo = new TunedQueryInfo(null, tunedDetail, 0); + TunedQueryInfo tunedInfo = new TunedQueryInfo(null, tunedDetail); Query query = server.find(Order.class).setId(1); - tunedInfo.autoFetchTune((SpiQuery)query); + tunedInfo.tuneQuery((SpiQuery) query); LoggedSqlCollector.start(); @@ -167,11 +167,11 @@ public class TunedQueryInfoTest extends BaseTestCase { OrmQueryDetail tunedDetail = new OrmQueryDetail(); tunedDetail.select("status"); - TunedQueryInfo tunedInfo = new TunedQueryInfo(null, tunedDetail, 0); + TunedQueryInfo tunedInfo = new TunedQueryInfo(null, tunedDetail); Query query = server.find(Order.class).setId(1); - tunedInfo.autoFetchTune((SpiQuery)query); + tunedInfo.tuneQuery((SpiQuery) query); LoggedSqlCollector.start(); diff --git a/src/test/java/com/avaje/tests/cache/TestL2CacheWithSharedBean.java b/src/test/java/com/avaje/tests/cache/TestL2CacheWithSharedBean.java index a31184917..c3367d686 100644 --- a/src/test/java/com/avaje/tests/cache/TestL2CacheWithSharedBean.java +++ b/src/test/java/com/avaje/tests/cache/TestL2CacheWithSharedBean.java @@ -28,11 +28,11 @@ public class TestL2CacheWithSharedBean extends BaseTestCase { OrmQueryDetail tunedDetail = new OrmQueryDetail(); tunedDetail.select("name"); - TunedQueryInfo tunedInfo = new TunedQueryInfo(null, tunedDetail, 0); + TunedQueryInfo tunedInfo = new TunedQueryInfo(null, tunedDetail); Query query = Ebean.find(FeatureDescription.class).setId(f1.getId()); - tunedInfo.autoFetchTune((SpiQuery) query); + tunedInfo.tuneQuery((SpiQuery) query); query.findUnique(); // PUT into cache From 5b8ce8eaa0d95022b98dcb7cecc47a563145b058 Mon Sep 17 00:00:00 2001 From: Robin Bygrave Date: Fri, 4 Sep 2015 08:48:25 +1200 Subject: [PATCH 3/9] WIP AutoTune refactor --- ebean-autotune-profiling.xml | 32 ++++ ebean-autotune.xml | 31 ++++ .../java/com/avaje/ebean/bean/CallStack.java | 11 ++ .../server/autofetch/AutoTuneCollection.java | 4 + .../server/autofetch/model/Autotune.java | 123 +++++++++++++ .../server/autofetch/model/ObjectFactory.java | 64 +++++++ .../server/autofetch/model/Origin.java | 171 ++++++++++++++++++ .../server/autofetch/model/ProfileDiff.java | 69 +++++++ .../server/autofetch/model/ProfileNew.java | 69 +++++++ .../server/autofetch/model/package-info.java | 2 + .../autofetch/service/AutoTuneXmlReader.java | 57 ++++++ .../autofetch/service/AutoTuneXmlWriter.java | 32 ++++ .../service/BaseAutoTuneService.java | 66 ++++++- .../autofetch/service/BaseQueryTuner.java | 28 ++- .../autofetch/service/ProfileOrigin.java | 16 +- .../service/ProfileOriginNodeUsage.java | 15 +- .../autofetch/service/TunedQueryInfo.java | 14 +- .../server/core/DefaultServer.java | 2 + .../server/querydefn/OrmQueryDetail.java | 50 +++-- .../querydefn/OrmQueryDetailParser.java | 20 +- .../server/querydefn/OrmQueryProperties.java | 7 + src/main/resources/ebean-autotune-1.0.xsd | 46 +++++ .../querydefn/OrmQueryDetailParserTest.java | 80 ++++++++ .../server/querydefn/OrmQueryDetailTest.java | 21 +++ .../query/TestAutofetchTuneWithJoin.java | 4 +- .../query/autotune/TestAutoTuneProfiling.java | 82 +++++++++ .../resources/autotune/test-profiling.xml | 10 + 27 files changed, 1092 insertions(+), 34 deletions(-) create mode 100644 ebean-autotune-profiling.xml create mode 100644 ebean-autotune.xml create mode 100644 src/main/java/com/avaje/ebeaninternal/server/autofetch/model/Autotune.java create mode 100644 src/main/java/com/avaje/ebeaninternal/server/autofetch/model/ObjectFactory.java create mode 100644 src/main/java/com/avaje/ebeaninternal/server/autofetch/model/Origin.java create mode 100644 src/main/java/com/avaje/ebeaninternal/server/autofetch/model/ProfileDiff.java create mode 100644 src/main/java/com/avaje/ebeaninternal/server/autofetch/model/ProfileNew.java create mode 100644 src/main/java/com/avaje/ebeaninternal/server/autofetch/model/package-info.java create mode 100644 src/main/java/com/avaje/ebeaninternal/server/autofetch/service/AutoTuneXmlReader.java create mode 100644 src/main/java/com/avaje/ebeaninternal/server/autofetch/service/AutoTuneXmlWriter.java create mode 100644 src/main/resources/ebean-autotune-1.0.xsd create mode 100644 src/test/java/com/avaje/ebeaninternal/server/querydefn/OrmQueryDetailParserTest.java create mode 100644 src/test/java/com/avaje/ebeaninternal/server/querydefn/OrmQueryDetailTest.java create mode 100644 src/test/java/com/avaje/tests/query/autotune/TestAutoTuneProfiling.java create mode 100644 src/test/resources/autotune/test-profiling.xml diff --git a/ebean-autotune-profiling.xml b/ebean-autotune-profiling.xml new file mode 100644 index 000000000..3911ea279 --- /dev/null +++ b/ebean-autotune-profiling.xml @@ -0,0 +1,32 @@ + + + + + + + com.avaje.tests.query.autotune.TestAutoTuneProfiling.findById(TestAutoTuneProfiling.java:45) + com.avaje.tests.query.autotune.TestAutoTuneProfiling.useOrderDate(TestAutoTuneProfiling.java:49) + com.avaje.tests.query.autotune.TestAutoTuneProfiling.execute(TestAutoTuneProfiling.java:35) + com.avaje.tests.query.autotune.TestAutoTuneProfiling.test(TestAutoTuneProfiling.java:26) + sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) + + + + + + + + + + com.avaje.tests.query.autotune.TestAutoTuneProfiling.findById(TestAutoTuneProfiling.java:45) + com.avaje.tests.query.autotune.TestAutoTuneProfiling.useOrderDateCustomerName(TestAutoTuneProfiling.java:55) + com.avaje.tests.query.autotune.TestAutoTuneProfiling.execute(TestAutoTuneProfiling.java:36) + com.avaje.tests.query.autotune.TestAutoTuneProfiling.test(TestAutoTuneProfiling.java:26) + sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) + + + + + diff --git a/ebean-autotune.xml b/ebean-autotune.xml new file mode 100644 index 000000000..73563dec3 --- /dev/null +++ b/ebean-autotune.xml @@ -0,0 +1,31 @@ + + + + + com.avaje.tests.query.autotune.TestAutoTuneProfiling.findById(TestAutoTuneProfiling.java:45) + com.avaje.tests.query.autotune.TestAutoTuneProfiling.useLots(TestAutoTuneProfiling.java:62) + com.avaje.tests.query.autotune.TestAutoTuneProfiling.execute(TestAutoTuneProfiling.java:37) + com.avaje.tests.query.autotune.TestAutoTuneProfiling.test(TestAutoTuneProfiling.java:26) + sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) + + + + com.avaje.tests.query.autotune.TestAutoTuneProfiling.findById(TestAutoTuneProfiling.java:45) + com.avaje.tests.query.autotune.TestAutoTuneProfiling.useOrderDate(TestAutoTuneProfiling.java:49) + com.avaje.tests.query.autotune.TestAutoTuneProfiling.execute(TestAutoTuneProfiling.java:35) + com.avaje.tests.query.autotune.TestAutoTuneProfiling.test(TestAutoTuneProfiling.java:26) + sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) + + + + + + + + + + + + + diff --git a/src/main/java/com/avaje/ebean/bean/CallStack.java b/src/main/java/com/avaje/ebean/bean/CallStack.java index 0f4b78f0c..f283ea591 100644 --- a/src/main/java/com/avaje/ebean/bean/CallStack.java +++ b/src/main/java/com/avaje/ebean/bean/CallStack.java @@ -87,6 +87,17 @@ public final class CallStack implements Serializable { return zeroHash + ":" + pathHash + ":" + callStack[0]; } + /** + * Return the call stack lines appended with the given newLine string. + */ + public String description(String newLine) { + StringBuilder sb = new StringBuilder(400); + for (int i = 0; i < callStack.length; i++) { + sb.append(callStack[i].toString()).append(newLine); + } + return sb.toString(); + } + public String getOriginKey(int queryHash) { return zeroHash + "." + enc(queryHash) + "." + pathHash; } diff --git a/src/main/java/com/avaje/ebeaninternal/server/autofetch/AutoTuneCollection.java b/src/main/java/com/avaje/ebeaninternal/server/autofetch/AutoTuneCollection.java index e26b892f1..eb62059b9 100644 --- a/src/main/java/com/avaje/ebeaninternal/server/autofetch/AutoTuneCollection.java +++ b/src/main/java/com/avaje/ebeaninternal/server/autofetch/AutoTuneCollection.java @@ -19,6 +19,10 @@ public class AutoTuneCollection { return entry; } + public List getEntries() { + return entries; + } + /** * Profiling entry at a given origin point. */ diff --git a/src/main/java/com/avaje/ebeaninternal/server/autofetch/model/Autotune.java b/src/main/java/com/avaje/ebeaninternal/server/autofetch/model/Autotune.java new file mode 100644 index 000000000..4be03eb68 --- /dev/null +++ b/src/main/java/com/avaje/ebeaninternal/server/autofetch/model/Autotune.java @@ -0,0 +1,123 @@ + +package com.avaje.ebeaninternal.server.autofetch.model; + +import java.util.ArrayList; +import java.util.List; +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlRootElement; +import javax.xml.bind.annotation.XmlType; + + +/** + *

Java class for anonymous complex type. + * + *

The following schema fragment specifies the expected content contained within this class. + * + * 

+ * <complexType>
+ *   <complexContent>
+ *     <restriction base="{http://www.w3.org/2001/XMLSchema}anyType">
+ *       <sequence>
+ *         <element ref="{http://ebean-orm.github.io/xml/ns/autotune}origin" maxOccurs="unbounded" minOccurs="0"/>
+ *         <element ref="{http://ebean-orm.github.io/xml/ns/autotune}profileDiff" minOccurs="0"/>
+ *         <element ref="{http://ebean-orm.github.io/xml/ns/autotune}profileNew" minOccurs="0"/>
+ *       </sequence>
+ *     </restriction>
+ *   </complexContent>
+ * </complexType>
+ *
+ * + * + */ +@XmlAccessorType(XmlAccessType.FIELD) +@XmlType(name = "", propOrder = { + "origin", + "profileDiff", + "profileNew" +}) +@XmlRootElement(name = "autotune") +public class Autotune { + + protected List origin; + protected ProfileDiff profileDiff; + protected ProfileNew profileNew; + + /** + * Gets the value of the origin property. + * + *

+ * This accessor method returns a reference to the live list, + * not a snapshot. Therefore any modification you make to the + * returned list will be present inside the JAXB object. + * This is why there is not a set method for the origin property. + * + *

+ * For example, to add a new item, do as follows: + * 

+     *    getOrigin().add(newItem);
+     *
+ * + * + *

+ * Objects of the following type(s) are allowed in the list + * {@link Origin } + * + * + */ + public List getOrigin() { + if (origin == null) { + origin = new ArrayList(); + } + return this.origin; + } + + /** + * Gets the value of the profileDiff property. + * + * @return + * possible object is + * {@link ProfileDiff } + * + */ + public ProfileDiff getProfileDiff() { + return profileDiff; + } + + /** + * Sets the value of the profileDiff property. + * + * @param value + * allowed object is + * {@link ProfileDiff } + * + */ + public void setProfileDiff(ProfileDiff value) { + this.profileDiff = value; + } + + /** + * Gets the value of the profileNew property. + * + * @return + * possible object is + * {@link ProfileNew } + * + */ + public ProfileNew getProfileNew() { + return profileNew; + } + + /** + * Sets the value of the profileNew property. + * + * @param value + * allowed object is + * {@link ProfileNew } + * + */ + public void setProfileNew(ProfileNew value) { + this.profileNew = value; + } + +} diff --git a/src/main/java/com/avaje/ebeaninternal/server/autofetch/model/ObjectFactory.java b/src/main/java/com/avaje/ebeaninternal/server/autofetch/model/ObjectFactory.java new file mode 100644 index 000000000..a6fc83ab8 --- /dev/null +++ b/src/main/java/com/avaje/ebeaninternal/server/autofetch/model/ObjectFactory.java @@ -0,0 +1,64 @@ + +package com.avaje.ebeaninternal.server.autofetch.model; + +import javax.xml.bind.annotation.XmlRegistry; + + +/** + * This object contains factory methods for each + * Java content interface and Java element interface + * generated in the com.avaje.ebeaninternal.server.autofetch.model package. + *

An ObjectFactory allows you to programatically + * construct new instances of the Java representation + * for XML content. The Java representation of XML + * content can consist of schema derived interfaces + * and classes representing the binding of schema + * type definitions, element declarations and model + * groups. Factory methods for each of these are + * provided in this class. + * + */ +@XmlRegistry +public class ObjectFactory { + + + /** + * Create a new ObjectFactory that can be used to create new instances of schema derived classes for package: com.avaje.ebeaninternal.server.autofetch.model + * + */ + public ObjectFactory() { + } + + /** + * Create an instance of {@link ProfileNew } + * + */ + public ProfileNew createProfileNew() { + return new ProfileNew(); + } + + /** + * Create an instance of {@link Origin } + * + */ + public Origin createOrigin() { + return new Origin(); + } + + /** + * Create an instance of {@link Autotune } + * + */ + public Autotune createAutotune() { + return new Autotune(); + } + + /** + * Create an instance of {@link ProfileDiff } + * + */ + public ProfileDiff createProfileDiff() { + return new ProfileDiff(); + } + +} diff --git a/src/main/java/com/avaje/ebeaninternal/server/autofetch/model/Origin.java b/src/main/java/com/avaje/ebeaninternal/server/autofetch/model/Origin.java new file mode 100644 index 000000000..86eebba46 --- /dev/null +++ b/src/main/java/com/avaje/ebeaninternal/server/autofetch/model/Origin.java @@ -0,0 +1,171 @@ + +package com.avaje.ebeaninternal.server.autofetch.model; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlAttribute; +import javax.xml.bind.annotation.XmlRootElement; +import javax.xml.bind.annotation.XmlType; + + +/** + *

Java class for anonymous complex type. + * + *

The following schema fragment specifies the expected content contained within this class. + * + * 

+ * <complexType>
+ *   <complexContent>
+ *     <restriction base="{http://www.w3.org/2001/XMLSchema}anyType">
+ *       <sequence>
+ *         <element name="callStack" type="{http://www.w3.org/2001/XMLSchema}string" minOccurs="0"/>
+ *       </sequence>
+ *       <attribute name="key" use="required" type="{http://www.w3.org/2001/XMLSchema}string" />
+ *       <attribute name="beanType" type="{http://www.w3.org/2001/XMLSchema}string" />
+ *       <attribute name="detail" type="{http://www.w3.org/2001/XMLSchema}string" />
+ *       <attribute name="tuneDetail" type="{http://www.w3.org/2001/XMLSchema}string" />
+ *     </restriction>
+ *   </complexContent>
+ * </complexType>
+ *
+ * + * + */ +@XmlAccessorType(XmlAccessType.FIELD) +@XmlType(name = "", propOrder = { + "callStack" +}) +@XmlRootElement(name = "origin") +public class Origin { + + protected String callStack; + @XmlAttribute(name = "key", required = true) + protected String key; + @XmlAttribute(name = "beanType") + protected String beanType; + @XmlAttribute(name = "detail") + protected String detail; + @XmlAttribute(name = "tuneDetail") + protected String tuneDetail; + + /** + * Gets the value of the callStack property. + * + * @return + * possible object is + * {@link String } + * + */ + public String getCallStack() { + return callStack; + } + + /** + * Sets the value of the callStack property. + * + * @param value + * allowed object is + * {@link String } + * + */ + public void setCallStack(String value) { + this.callStack = value; + } + + /** + * Gets the value of the key property. + * + * @return + * possible object is + * {@link String } + * + */ + public String getKey() { + return key; + } + + /** + * Sets the value of the key property. + * + * @param value + * allowed object is + * {@link String } + * + */ + public void setKey(String value) { + this.key = value; + } + + /** + * Gets the value of the beanType property. + * + * @return + * possible object is + * {@link String } + * + */ + public String getBeanType() { + return beanType; + } + + /** + * Sets the value of the beanType property. + * + * @param value + * allowed object is + * {@link String } + * + */ + public void setBeanType(String value) { + this.beanType = value; + } + + /** + * Gets the value of the detail property. + * + * @return + * possible object is + * {@link String } + * + */ + public String getDetail() { + return detail; + } + + /** + * Sets the value of the detail property. + * + * @param value + * allowed object is + * {@link String } + * + */ + public void setDetail(String value) { + this.detail = value; + } + + /** + * Gets the value of the tuneDetail property. + * + * @return + * possible object is + * {@link String } + * + */ + public String getTuneDetail() { + return tuneDetail; + } + + /** + * Sets the value of the tuneDetail property. + * + * @param value + * allowed object is + * {@link String } + * + */ + public void setTuneDetail(String value) { + this.tuneDetail = value; + } + +} diff --git a/src/main/java/com/avaje/ebeaninternal/server/autofetch/model/ProfileDiff.java b/src/main/java/com/avaje/ebeaninternal/server/autofetch/model/ProfileDiff.java new file mode 100644 index 000000000..a1bc1c91b --- /dev/null +++ b/src/main/java/com/avaje/ebeaninternal/server/autofetch/model/ProfileDiff.java @@ -0,0 +1,69 @@ + +package com.avaje.ebeaninternal.server.autofetch.model; + +import java.util.ArrayList; +import java.util.List; +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlRootElement; +import javax.xml.bind.annotation.XmlType; + + +/** + *

Java class for anonymous complex type. + * + *

The following schema fragment specifies the expected content contained within this class. + * + * 

+ * <complexType>
+ *   <complexContent>
+ *     <restriction base="{http://www.w3.org/2001/XMLSchema}anyType">
+ *       <sequence>
+ *         <element ref="{http://ebean-orm.github.io/xml/ns/autotune}origin" maxOccurs="unbounded" minOccurs="0"/>
+ *       </sequence>
+ *     </restriction>
+ *   </complexContent>
+ * </complexType>
+ *
+ * + * + */ +@XmlAccessorType(XmlAccessType.FIELD) +@XmlType(name = "", propOrder = { + "origin" +}) +@XmlRootElement(name = "profileDiff") +public class ProfileDiff { + + protected List origin; + + /** + * Gets the value of the origin property. + * + *

+ * This accessor method returns a reference to the live list, + * not a snapshot. Therefore any modification you make to the + * returned list will be present inside the JAXB object. + * This is why there is not a set method for the origin property. + * + *

+ * For example, to add a new item, do as follows: + * 

+     *    getOrigin().add(newItem);
+     *
+ * + * + *

+ * Objects of the following type(s) are allowed in the list + * {@link Origin } + * + * + */ + public List getOrigin() { + if (origin == null) { + origin = new ArrayList(); + } + return this.origin; + } + +} diff --git a/src/main/java/com/avaje/ebeaninternal/server/autofetch/model/ProfileNew.java b/src/main/java/com/avaje/ebeaninternal/server/autofetch/model/ProfileNew.java new file mode 100644 index 000000000..0a058fb29 --- /dev/null +++ b/src/main/java/com/avaje/ebeaninternal/server/autofetch/model/ProfileNew.java @@ -0,0 +1,69 @@ + +package com.avaje.ebeaninternal.server.autofetch.model; + +import java.util.ArrayList; +import java.util.List; +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlRootElement; +import javax.xml.bind.annotation.XmlType; + + +/** + *

Java class for anonymous complex type. + * + *

The following schema fragment specifies the expected content contained within this class. + * + * 

+ * <complexType>
+ *   <complexContent>
+ *     <restriction base="{http://www.w3.org/2001/XMLSchema}anyType">
+ *       <sequence>
+ *         <element ref="{http://ebean-orm.github.io/xml/ns/autotune}origin" maxOccurs="unbounded" minOccurs="0"/>
+ *       </sequence>
+ *     </restriction>
+ *   </complexContent>
+ * </complexType>
+ *
+ * + * + */ +@XmlAccessorType(XmlAccessType.FIELD) +@XmlType(name = "", propOrder = { + "origin" +}) +@XmlRootElement(name = "profileNew") +public class ProfileNew { + + protected List origin; + + /** + * Gets the value of the origin property. + * + *

+ * This accessor method returns a reference to the live list, + * not a snapshot. Therefore any modification you make to the + * returned list will be present inside the JAXB object. + * This is why there is not a set method for the origin property. + * + *

+ * For example, to add a new item, do as follows: + * 

+     *    getOrigin().add(newItem);
+     *
+ * + * + *

+ * Objects of the following type(s) are allowed in the list + * {@link Origin } + * + * + */ + public List getOrigin() { + if (origin == null) { + origin = new ArrayList(); + } + return this.origin; + } + +} diff --git a/src/main/java/com/avaje/ebeaninternal/server/autofetch/model/package-info.java b/src/main/java/com/avaje/ebeaninternal/server/autofetch/model/package-info.java new file mode 100644 index 000000000..7204153e4 --- /dev/null +++ b/src/main/java/com/avaje/ebeaninternal/server/autofetch/model/package-info.java @@ -0,0 +1,2 @@ +@javax.xml.bind.annotation.XmlSchema(namespace = "http://ebean-orm.github.io/xml/ns/autotune", elementFormDefault = javax.xml.bind.annotation.XmlNsForm.QUALIFIED) +package com.avaje.ebeaninternal.server.autofetch.model; diff --git a/src/main/java/com/avaje/ebeaninternal/server/autofetch/service/AutoTuneXmlReader.java b/src/main/java/com/avaje/ebeaninternal/server/autofetch/service/AutoTuneXmlReader.java new file mode 100644 index 000000000..8ff62af16 --- /dev/null +++ b/src/main/java/com/avaje/ebeaninternal/server/autofetch/service/AutoTuneXmlReader.java @@ -0,0 +1,57 @@ +package com.avaje.ebeaninternal.server.autofetch.service; + + +import com.avaje.ebeaninternal.server.autofetch.model.Autotune; + +import javax.xml.bind.JAXBContext; +import javax.xml.bind.JAXBException; +import javax.xml.bind.Unmarshaller; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; + +/** + * Reads a profiling xml document. + */ +public class AutoTuneXmlReader { + + /** + * Read and return a Profiling from an xml file. + */ + public Autotune read(File file) { + + try { + return readFile(file); + } catch (IOException e) { + throw new IllegalStateException(e); + } + } + + protected Autotune readFile(File file) throws IOException { + if (!file.exists()) { + return new Autotune(); + } + FileInputStream is = new FileInputStream(file); + try { + return read(is); + } finally { + is.close(); + } + } + + /** + * Read and return a Profiling from an xml document. + */ + public Autotune read(InputStream is) { + + try { + JAXBContext jaxbContext = JAXBContext.newInstance(Autotune.class); + Unmarshaller unmarshaller = jaxbContext.createUnmarshaller(); + return (Autotune) unmarshaller.unmarshal(is); + } catch (JAXBException e) { + throw new IllegalStateException(e); + } + } + +} diff --git a/src/main/java/com/avaje/ebeaninternal/server/autofetch/service/AutoTuneXmlWriter.java b/src/main/java/com/avaje/ebeaninternal/server/autofetch/service/AutoTuneXmlWriter.java new file mode 100644 index 000000000..7770bafb1 --- /dev/null +++ b/src/main/java/com/avaje/ebeaninternal/server/autofetch/service/AutoTuneXmlWriter.java @@ -0,0 +1,32 @@ +package com.avaje.ebeaninternal.server.autofetch.service; + + +import com.avaje.ebeaninternal.server.autofetch.model.Autotune; + +import javax.xml.bind.JAXBContext; +import javax.xml.bind.JAXBException; +import javax.xml.bind.Marshaller; +import java.io.File; + +/** + * Simple writer for output of the AutoTune Profiling as an XML document. + */ +public class AutoTuneXmlWriter { + + /** + * Write Profiling to a file as xml. + */ + public void write(Autotune profiling, File file) { + + try { + JAXBContext jaxbContext = JAXBContext.newInstance(Autotune.class); + Marshaller marshaller = jaxbContext.createMarshaller(); + marshaller.setProperty( Marshaller.JAXB_FORMATTED_OUTPUT, Boolean.TRUE); + marshaller.marshal(profiling, file); + + } catch (JAXBException e) { + throw new RuntimeException(e); + } + } + +} diff --git a/src/main/java/com/avaje/ebeaninternal/server/autofetch/service/BaseAutoTuneService.java b/src/main/java/com/avaje/ebeaninternal/server/autofetch/service/BaseAutoTuneService.java index 9b4923276..5dc886cfe 100644 --- a/src/main/java/com/avaje/ebeaninternal/server/autofetch/service/BaseAutoTuneService.java +++ b/src/main/java/com/avaje/ebeaninternal/server/autofetch/service/BaseAutoTuneService.java @@ -1,14 +1,25 @@ package com.avaje.ebeaninternal.server.autofetch.service; +import com.avaje.ebean.bean.ObjectGraphOrigin; import com.avaje.ebean.config.AutofetchConfig; import com.avaje.ebean.config.ServerConfig; import com.avaje.ebeaninternal.api.SpiEbeanServer; import com.avaje.ebeaninternal.api.SpiQuery; -import com.avaje.ebeaninternal.server.autofetch.AutoTuneService; import com.avaje.ebeaninternal.server.autofetch.AutoTuneCollection; +import com.avaje.ebeaninternal.server.autofetch.AutoTuneService; +import com.avaje.ebeaninternal.server.autofetch.model.Autotune; +import com.avaje.ebeaninternal.server.autofetch.model.Origin; +import com.avaje.ebeaninternal.server.autofetch.model.ProfileDiff; +import com.avaje.ebeaninternal.server.autofetch.model.ProfileNew; +import com.avaje.ebeaninternal.server.querydefn.OrmQueryDetail; +import com.avaje.ebeaninternal.server.querydefn.OrmQueryDetailParser; +import org.jetbrains.annotations.NotNull; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.File; +import java.util.List; + /** * Implementation of the AutoTuneService which is comprised of profiling and query tuning. */ @@ -40,12 +51,65 @@ public class BaseAutoTuneService implements AutoTuneService { */ public void startup() { + File file = new File("ebean-autotune.xml"); + AutoTuneXmlReader reader = new AutoTuneXmlReader(); + Autotune profiling = reader.read(file); + List originList = profiling.getOrigin(); + for (Origin origin : originList) { + String key = origin.getKey(); + String detail = origin.getDetail(); + OrmQueryDetailParser parser = new OrmQueryDetailParser(detail); + OrmQueryDetail fetchDetail = parser.parse(); + TunedQueryInfo tunedQueryInfo = new TunedQueryInfo(fetchDetail); + queryTuner.load(key, tunedQueryInfo); + } } private void saveProfiling() { + Autotune document = new Autotune(); + AutoTuneCollection autoTuneCollection = profileManager.profilingCollection(false); + List entries = autoTuneCollection.getEntries(); + for (AutoTuneCollection.Entry entry : entries) { + ObjectGraphOrigin point = entry.getOrigin(); + OrmQueryDetail profileDetail = entry.getDetail(); + + OrmQueryDetail tuneDetail = queryTuner.get(point.getKey()); + if (tuneDetail == null) { + ProfileNew profileNew = document.getProfileNew(); + if (profileNew == null) { + profileNew = new ProfileNew(); + document.setProfileNew(profileNew); + } + profileNew.getOrigin().add( createOrigin(entry, point)); + + } else if (!tuneDetail.isAutoTuneEqual(profileDetail)) { + Origin origin1 = createOrigin(entry, point); + origin1.setTuneDetail(tuneDetail.toString()); + ProfileDiff diff = document.getProfileDiff(); + if (diff == null) { + diff = new ProfileDiff(); + document.setProfileDiff(diff); + } + diff.getOrigin().add(origin1); + } + } + + File file = new File("ebean-autotune-profiling.xml"); + AutoTuneXmlWriter writer = new AutoTuneXmlWriter(); + writer.write(document, file); + } + + @NotNull + private Origin createOrigin(AutoTuneCollection.Entry entry, ObjectGraphOrigin point) { + Origin origin1 = new Origin(); + origin1.setKey(point.getKey()); + origin1.setBeanType(point.getBeanType()); + origin1.setDetail(entry.getDetail().toString()); + origin1.setCallStack(point.getCallStack().description("\n")); + return origin1; } /** diff --git a/src/main/java/com/avaje/ebeaninternal/server/autofetch/service/BaseQueryTuner.java b/src/main/java/com/avaje/ebeaninternal/server/autofetch/service/BaseQueryTuner.java index 6ad1fe2c0..9536197d5 100644 --- a/src/main/java/com/avaje/ebeaninternal/server/autofetch/service/BaseQueryTuner.java +++ b/src/main/java/com/avaje/ebeaninternal/server/autofetch/service/BaseQueryTuner.java @@ -7,6 +7,7 @@ import com.avaje.ebean.config.AutofetchMode; import com.avaje.ebeaninternal.api.SpiEbeanServer; import com.avaje.ebeaninternal.api.SpiQuery; import com.avaje.ebeaninternal.server.autofetch.ProfilingListener; +import com.avaje.ebeaninternal.server.querydefn.OrmQueryDetail; import javax.persistence.PersistenceException; import java.util.Map; @@ -48,6 +49,17 @@ public class BaseQueryTuner { tunedQueryInfoMap.put(key, queryInfo); } + /** + * Return the detail currently used for tuning. + * This returns null if there is currently no matching tuning. + */ + public OrmQueryDetail get(String key) { + TunedQueryInfo info = tunedQueryInfoMap.get(key); + return (info == null) ? null : info.getTunedDetail(); + } + + boolean fullProfiling = true; + /** * Auto tune the query and enable profiling. */ @@ -58,7 +70,11 @@ public class BaseQueryTuner { } if (!useAutoTune(query)) { - // not using autoFetch for this query + // not tuning this query but maybe profiling + if (fullProfiling) { + CallStack stack = server.createCallStack(); + profiling(query, stack); + } return false; } @@ -88,6 +104,16 @@ public class BaseQueryTuner { return false; } + private void profiling(SpiQuery query, CallStack stack) { + + // create a query point to identify the query + ObjectGraphNode origin = query.setOrigin(stack); + if (profilingListener.isProfileRequest(origin)) { + // collect more profiling based on profiling rate etc + query.setProfilingListener(profilingListener); + } + } + /** * Return true if we should try to tune this query. */ diff --git a/src/main/java/com/avaje/ebeaninternal/server/autofetch/service/ProfileOrigin.java b/src/main/java/com/avaje/ebeaninternal/server/autofetch/service/ProfileOrigin.java index 84f758c0b..411f9756f 100644 --- a/src/main/java/com/avaje/ebeaninternal/server/autofetch/service/ProfileOrigin.java +++ b/src/main/java/com/avaje/ebeaninternal/server/autofetch/service/ProfileOrigin.java @@ -16,6 +16,8 @@ import java.util.concurrent.atomic.AtomicLong; public class ProfileOrigin { + //private static final Logger logger = LoggerFactory.getLogger(ProfileOrigin.class); + private static final long RESET_COUNT = -1000000000L; private final ObjectGraphOrigin origin; @@ -89,10 +91,10 @@ public class ProfileOrigin { } private OrmQueryDetail buildDetail(BeanDescriptor rootDesc) { - PathProperties pathProps = new PathProperties(); + PathProperties pathProps = new PathProperties(); for (ProfileOriginNodeUsage statsNode : nodeUsageMap.values()) { - statsNode.buildTunedFetch(pathProps, rootDesc); + statsNode.buildTunedFetch(pathProps, rootDesc, queryTuningAddVersion); } OrmQueryDetail detail = new OrmQueryDetail(); @@ -142,10 +144,10 @@ public class ProfileOrigin { */ public void collectUsageInfo(NodeUsageCollector profile) { - if (!profile.isEmpty()) { - ObjectGraphNode node = profile.getNode(); + //logger.info("COLLECT USAGE {}", profile.toString()); - ProfileOriginNodeUsage nodeStats = getNodeStats(node.getPath()); + if (!profile.isEmpty()) { + ProfileOriginNodeUsage nodeStats = getNodeStats(profile.getNode().getPath()); nodeStats.collectUsageInfo(profile); } } @@ -153,9 +155,11 @@ public class ProfileOrigin { private ProfileOriginNodeUsage getNodeStats(String path) { synchronized (monitor) { + // handle null paths as using ConcurrentHashMap + path = (path == null) ? "" : path; ProfileOriginNodeUsage nodeStats = nodeUsageMap.get(path); if (nodeStats == null) { - nodeStats = new ProfileOriginNodeUsage(path, queryTuningAddVersion); + nodeStats = new ProfileOriginNodeUsage(path); nodeUsageMap.put(path, nodeStats); } return nodeStats; diff --git a/src/main/java/com/avaje/ebeaninternal/server/autofetch/service/ProfileOriginNodeUsage.java b/src/main/java/com/avaje/ebeaninternal/server/autofetch/service/ProfileOriginNodeUsage.java index 632ac8097..ec8e9dc1a 100644 --- a/src/main/java/com/avaje/ebeaninternal/server/autofetch/service/ProfileOriginNodeUsage.java +++ b/src/main/java/com/avaje/ebeaninternal/server/autofetch/service/ProfileOriginNodeUsage.java @@ -24,8 +24,6 @@ public class ProfileOriginNodeUsage { private final String path; - private final boolean queryTuningAddVersion; - private int profileCount; private int profileUsedCount; @@ -34,12 +32,12 @@ public class ProfileOriginNodeUsage { private final Set aggregateUsed = new LinkedHashSet(); - public ProfileOriginNodeUsage(String path, boolean queryTuningAddVersion) { - this.path = path; - this.queryTuningAddVersion = queryTuningAddVersion; + public ProfileOriginNodeUsage(String path) { + // handle null paths as using ConcurrentHashMap + this.path = "".equals(path) ? null : path; } - public void buildTunedFetch(PathProperties pathProps, BeanDescriptor rootDesc) { + protected void buildTunedFetch(PathProperties pathProps, BeanDescriptor rootDesc, boolean addVersionProperty) { synchronized (monitor) { @@ -59,6 +57,7 @@ public class ProfileOriginNodeUsage { } for (String propName : aggregateUsed) { + //propName = "".equals(propName) ? null : propName; BeanProperty beanProp = desc.getBeanPropertyFromPath(propName); if (beanProp == null) { logger.warn("AutoTune: Can't find property[" + propName + "] for " + desc.getName()); @@ -81,7 +80,7 @@ public class ProfileOriginNodeUsage { } } - if ((modified || queryTuningAddVersion) && desc != null) { + if ((modified || addVersionProperty) && desc != null) { BeanProperty versionProp = desc.getVersionProperty(); if (versionProp != null) { pathProps.addToPath(path, versionProp.getName()); @@ -93,7 +92,7 @@ public class ProfileOriginNodeUsage { /** * Collect usage from a node. */ - public void collectUsageInfo(NodeUsageCollector profile) { + protected void collectUsageInfo(NodeUsageCollector profile) { synchronized (monitor) { diff --git a/src/main/java/com/avaje/ebeaninternal/server/autofetch/service/TunedQueryInfo.java b/src/main/java/com/avaje/ebeaninternal/server/autofetch/service/TunedQueryInfo.java index b1e554cbd..e203d5f70 100644 --- a/src/main/java/com/avaje/ebeaninternal/server/autofetch/service/TunedQueryInfo.java +++ b/src/main/java/com/avaje/ebeaninternal/server/autofetch/service/TunedQueryInfo.java @@ -21,6 +21,18 @@ public class TunedQueryInfo implements Serializable { this.tunedDetail = tunedDetail; } + public TunedQueryInfo(OrmQueryDetail tunedDetail) { + this.origin = null; + this.tunedDetail = tunedDetail; + } + + /** + * Return the tuned detail (for comparison with profiling information). + */ + public OrmQueryDetail getTunedDetail() { + return tunedDetail; + } + /** * Tune the query by replacing its OrmQueryDetail with a tuned one. * @@ -47,7 +59,7 @@ public class TunedQueryInfo implements Serializable { } public String toString() { - return origin.getBeanType() + " " + origin.getKey() + " " + tunedDetail; + return tunedDetail.toString(); } } diff --git a/src/main/java/com/avaje/ebeaninternal/server/core/DefaultServer.java b/src/main/java/com/avaje/ebeaninternal/server/core/DefaultServer.java index 4b14fc42b..cdab0e94f 100644 --- a/src/main/java/com/avaje/ebeaninternal/server/core/DefaultServer.java +++ b/src/main/java/com/avaje/ebeaninternal/server/core/DefaultServer.java @@ -236,6 +236,8 @@ public final class DefaultServer implements SpiServer, SpiEbeanServer { private void configureServerPlugins() { + autoTuneService.startup(); + for (SpiServerPlugin plugin : serverPlugins) { plugin.configure(this); } diff --git a/src/main/java/com/avaje/ebeaninternal/server/querydefn/OrmQueryDetail.java b/src/main/java/com/avaje/ebeaninternal/server/querydefn/OrmQueryDetail.java index a65edfa16..954c5f4fe 100644 --- a/src/main/java/com/avaje/ebeaninternal/server/querydefn/OrmQueryDetail.java +++ b/src/main/java/com/avaje/ebeaninternal/server/querydefn/OrmQueryDetail.java @@ -9,6 +9,7 @@ import java.util.LinkedHashMap; import java.util.LinkedHashSet; import java.util.List; import java.util.Map; +import java.util.Set; import javax.persistence.PersistenceException; @@ -83,25 +84,48 @@ public class OrmQueryDetail implements Serializable { * Return true if equal in terms of autofetch (select and joins). */ public boolean isAutoTuneEqual(OrmQueryDetail otherDetail) { - return autofetchPlanHash() == otherDetail.autofetchPlanHash(); - } - /** - * Calculate the hash for the query plan. - */ - private int autofetchPlanHash() { - - int hc = (baseProps == null ? 1 : baseProps.autofetchPlanHash()); - - if (fetchPaths != null) { - for (OrmQueryProperties p : fetchPaths.values()) { - hc = hc * 31 + p.autofetchPlanHash(); + if (!isSame(baseProps, otherDetail.baseProps)) { + return false; + } + if (fetchPaths == null) { + return otherDetail.fetchPaths == null; + } + Set> entries = fetchPaths.entrySet(); + for (Map.Entry entry : entries) { + OrmQueryProperties chunk = otherDetail.getChunk(entry.getKey(), false); + if (!isSame(entry.getValue(), chunk)) { + return false; } } - return hc; + return true; + //return autofetchPlanHash() == otherDetail.autofetchPlanHash(); } + private boolean isSame(OrmQueryProperties p1, OrmQueryProperties p2) { + if (p1 == null) { + return p2 == null; + } + return p1.isSame(p2); + } + +// /** +// * Calculate the hash for the query plan. +// */ +// private int autofetchPlanHash() { +// +// int hc = (baseProps == null ? 1 : baseProps.autofetchPlanHash()); +// +// if (fetchPaths != null) { +// for (OrmQueryProperties p : fetchPaths.values()) { +// hc = hc * 31 + p.autofetchPlanHash(); +// } +// } +// +// return hc; +// } + public String toString() { StringBuilder sb = new StringBuilder(); if (baseProps != null) { diff --git a/src/main/java/com/avaje/ebeaninternal/server/querydefn/OrmQueryDetailParser.java b/src/main/java/com/avaje/ebeaninternal/server/querydefn/OrmQueryDetailParser.java index f33572efb..9a5026693 100644 --- a/src/main/java/com/avaje/ebeaninternal/server/querydefn/OrmQueryDetailParser.java +++ b/src/main/java/com/avaje/ebeaninternal/server/querydefn/OrmQueryDetailParser.java @@ -27,10 +27,11 @@ public class OrmQueryDetailParser { this.parser = new SimpleTextParser(oql); } - public void parse() throws PersistenceException { + public OrmQueryDetail parse() throws PersistenceException { parser.nextWord(); processInitial(); + return detail; } protected void assign(DefaultOrmQuery query) { @@ -42,7 +43,9 @@ public class OrmQueryDetailParser { } private void processInitial() { - if (parser.isMatch("find")) { + if (parser.isMatch("select")) { + readSelect(); + } else if (parser.isMatch("find")) { OrmQueryProperties props = readFindFetch(); detail.setBase(props); } else { @@ -149,6 +152,19 @@ public class OrmQueryDetailParser { } } + private void readSelect() { + String path = null; + String props = parser.nextWord(); + if (props.startsWith("(")) { + props = props.substring(1, props.length() - 1); + OrmQueryProperties base = new OrmQueryProperties(path, props); + detail.setBase(base); + parser.nextWord(); + } else { + process(); + } + } + private OrmQueryProperties readFindFetch() { boolean readAlias = false; diff --git a/src/main/java/com/avaje/ebeaninternal/server/querydefn/OrmQueryProperties.java b/src/main/java/com/avaje/ebeaninternal/server/querydefn/OrmQueryProperties.java index a045b4194..23d270ce4 100644 --- a/src/main/java/com/avaje/ebeaninternal/server/querydefn/OrmQueryProperties.java +++ b/src/main/java/com/avaje/ebeaninternal/server/querydefn/OrmQueryProperties.java @@ -573,4 +573,11 @@ public class OrmQueryProperties implements Serializable { return Collections.unmodifiableSet(set); } + + public boolean isSame(OrmQueryProperties p2) { + if (included == null) { + return p2.included == null; + } + return included.equals(p2.included); + } } diff --git a/src/main/resources/ebean-autotune-1.0.xsd b/src/main/resources/ebean-autotune-1.0.xsd new file mode 100644 index 000000000..42a1ce081 --- /dev/null +++ b/src/main/resources/ebean-autotune-1.0.xsd @@ -0,0 +1,46 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/test/java/com/avaje/ebeaninternal/server/querydefn/OrmQueryDetailParserTest.java b/src/test/java/com/avaje/ebeaninternal/server/querydefn/OrmQueryDetailParserTest.java new file mode 100644 index 000000000..5ddd475b1 --- /dev/null +++ b/src/test/java/com/avaje/ebeaninternal/server/querydefn/OrmQueryDetailParserTest.java @@ -0,0 +1,80 @@ +package com.avaje.ebeaninternal.server.querydefn; + +import com.avaje.ebean.BaseTestCase; +import org.junit.Test; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.Assert.assertNull; + + +public class OrmQueryDetailParserTest extends BaseTestCase { + + @Test + public void testParseBasic() throws Exception { + + + OrmQueryDetail other = new OrmQueryDetail(); + other.select("id,name"); + + OrmQueryProperties root = other.getChunk(null, false); + assertNull(root.getPath()); + assertThat(root.getAllIncludedProperties()).contains("id", "name"); + + OrmQueryDetailParser p = new OrmQueryDetailParser("select (id,name)"); + OrmQueryDetail detail = p.parse(); + + root = detail.getChunk(null, false); + assertNull(root.getPath()); + assertThat(root.getAllIncludedProperties()).contains("id", "name"); + } + + @Test + public void testParseEmptySelect() throws Exception { + + OrmQueryDetailParser p = new OrmQueryDetailParser("select fetch customer (email)"); + OrmQueryDetail detail = p.parse(); + + OrmQueryProperties root = detail.getChunk(null, false); + assertNull(root.getPath()); + assertThat(root.getAllIncludedProperties()).isNull(); + + OrmQueryProperties chunk = detail.getChunk("customer", false); + assertThat(chunk.getPath()).isEqualTo("customer"); + assertThat(chunk.getAllIncludedProperties()).contains("email"); + } + + @Test + public void testParseSelectFetch() throws Exception { + + OrmQueryDetailParser p = new OrmQueryDetailParser("select (id,name) fetch customer (email)"); + OrmQueryDetail detail = p.parse(); + + OrmQueryProperties root = detail.getChunk(null, false); + assertNull(root.getPath()); + assertThat(root.getAllIncludedProperties()).contains("id", "name"); + + OrmQueryProperties chunk = detail.getChunk("customer", false); + assertThat(chunk.getPath()).isEqualTo("customer"); + assertThat(chunk.getAllIncludedProperties()).contains("email"); + } + + @Test + public void testParseSelectFetchMore() throws Exception { + + OrmQueryDetailParser p = new OrmQueryDetailParser("select (id,name) fetch customer (email) fetch details.product (sku,description)"); + OrmQueryDetail detail = p.parse(); + + OrmQueryProperties root = detail.getChunk(null, false); + assertNull(root.getPath()); + assertThat(root.getAllIncludedProperties()).contains("id", "name"); + + OrmQueryProperties chunk = detail.getChunk("customer", false); + assertThat(chunk.getPath()).isEqualTo("customer"); + assertThat(chunk.getAllIncludedProperties()).contains("email"); + + chunk = detail.getChunk("details.product", false); + assertThat(chunk.getPath()).isEqualTo("details.product"); + assertThat(chunk.getAllIncludedProperties()).contains("sku","description"); + } + +} \ No newline at end of file diff --git a/src/test/java/com/avaje/ebeaninternal/server/querydefn/OrmQueryDetailTest.java b/src/test/java/com/avaje/ebeaninternal/server/querydefn/OrmQueryDetailTest.java new file mode 100644 index 000000000..4a6a18ec6 --- /dev/null +++ b/src/test/java/com/avaje/ebeaninternal/server/querydefn/OrmQueryDetailTest.java @@ -0,0 +1,21 @@ +package com.avaje.ebeaninternal.server.querydefn; + +import org.junit.Test; + +import static org.junit.Assert.*; + +public class OrmQueryDetailTest { + + + @Test + public void test_isAutoTuneEqual() { + + OrmQueryDetailParser parser1 = new OrmQueryDetailParser("select order (id,name) fetch customer (name) fetch details (code)"); + OrmQueryDetail detail1 = parser1.parse(); + + OrmQueryDetailParser parser2 = new OrmQueryDetailParser("select order (id,name) fetch details (code) fetch customer (name)"); + OrmQueryDetail detail2 = parser2.parse(); + + assertTrue(detail1.isAutoTuneEqual(detail2)); + } +} \ No newline at end of file diff --git a/src/test/java/com/avaje/tests/query/TestAutofetchTuneWithJoin.java b/src/test/java/com/avaje/tests/query/TestAutofetchTuneWithJoin.java index f9fcfe399..d5e2cbdf9 100644 --- a/src/test/java/com/avaje/tests/query/TestAutofetchTuneWithJoin.java +++ b/src/test/java/com/avaje/tests/query/TestAutofetchTuneWithJoin.java @@ -30,8 +30,8 @@ public class TestAutofetchTuneWithJoin extends BaseTestCase { Query q = Ebean.find(Order.class) .setAutofetch(true) - .fetch("customer") - .fetch("customer.contacts") + //.fetch("customer") + //.fetch("customer.contacts") .where().lt("id", 3).query(); List list = q.findList(); diff --git a/src/test/java/com/avaje/tests/query/autotune/TestAutoTuneProfiling.java b/src/test/java/com/avaje/tests/query/autotune/TestAutoTuneProfiling.java new file mode 100644 index 000000000..e270607e2 --- /dev/null +++ b/src/test/java/com/avaje/tests/query/autotune/TestAutoTuneProfiling.java @@ -0,0 +1,82 @@ +package com.avaje.tests.query.autotune; + +import com.avaje.ebean.AdminAutofetch; +import com.avaje.ebean.BaseTestCase; +import com.avaje.ebean.Ebean; +import com.avaje.ebean.Query; +import com.avaje.ebean.bean.ObjectGraphNode; +import com.avaje.ebean.bean.ObjectGraphOrigin; +import com.avaje.ebeaninternal.api.SpiQuery; +import com.avaje.tests.model.basic.Address; +import com.avaje.tests.model.basic.Customer; +import com.avaje.tests.model.basic.Order; +import com.avaje.tests.model.basic.ResetBasicData; +import org.junit.Test; + +import java.util.List; + +public class TestAutoTuneProfiling extends BaseTestCase { + + @Test + public void test() { + + ResetBasicData.reset(); + + for (int i = 0; i < 1; i++) { + execute(); + } + + + collectUsage(); + } + + + private void execute() { + useOrderDate(); + useOrderDateCustomerName(); + useLots(); + } + + + private Order findById(long id) { + return Ebean.find(Order.class) + .setAutofetch(true) + .setId(id) + .findUnique(); + } + + private void useOrderDate() { + Order order = findById(3); + order.getStatus(); + order.getShipDate(); + } + + private void useOrderDateCustomerName() { + Order order = findById(3); + order.getOrderDate(); + order.getCustomer().getName(); + } + + private void useLots() { + + Order order = findById(3); + order.getOrderDate(); + order.getShipDate(); + // order.setShipDate(new Date(System.currentTimeMillis())); + Customer customer = order.getCustomer(); + customer.getName(); + Address shippingAddress = customer.getShippingAddress(); + if (shippingAddress != null) { + shippingAddress.getLine1(); + shippingAddress.getCity(); + } + } + + private static void collectUsage() { + + AdminAutofetch adminAutofetch = Ebean.getServer(null).getAdminAutofetch(); + adminAutofetch.collectUsageViaGC(); + + } + +} diff --git a/src/test/resources/autotune/test-profiling.xml b/src/test/resources/autotune/test-profiling.xml new file mode 100644 index 000000000..7bd775db9 --- /dev/null +++ b/src/test/resources/autotune/test-profiling.xml @@ -0,0 +1,10 @@ + + + + + + + + + \ No newline at end of file From 116cc3a3fee172e0b5c81f198bab9f21806f7e1e Mon Sep 17 00:00:00 2001 From: Robin Bygrave Date: Wed, 9 Sep 2015 15:21:10 +1200 Subject: [PATCH 4/9] AutoTune update - remove resources --- .../avaje/ebean/config/AutofetchConfig.java | 62 +++------- .../service/AutoTuneServiceFactory.java | 2 +- ...rvice.java => DefaultAutoTuneService.java} | 6 +- .../server/core/InternalConfiguration.java | 13 +-- .../server/deploy/DeployOrmXml.java | 31 ++--- .../server/lib/resource/DirectoryFinder.java | 108 ------------------ .../lib/resource/FileResourceContent.java | 70 ------------ .../lib/resource/FileResourceSource.java | 50 -------- .../server/lib/resource/ResourceContent.java | 34 ------ .../server/lib/resource/ResourceSource.java | 26 ----- .../server/lib/resource/package.html | 13 --- .../server/resource/ResourceManager.java | 29 ----- .../resource/ResourceManagerFactory.java | 107 ----------------- .../server/resource/package.html | 10 -- .../server/querydefn/OrmQueryDetailTest.java | 4 +- 15 files changed, 28 insertions(+), 537 deletions(-) rename src/main/java/com/avaje/ebeaninternal/server/autofetch/service/{BaseAutoTuneService.java => DefaultAutoTuneService.java} (97%) delete mode 100644 src/main/java/com/avaje/ebeaninternal/server/lib/resource/DirectoryFinder.java delete mode 100644 src/main/java/com/avaje/ebeaninternal/server/lib/resource/FileResourceContent.java delete mode 100644 src/main/java/com/avaje/ebeaninternal/server/lib/resource/FileResourceSource.java delete mode 100644 src/main/java/com/avaje/ebeaninternal/server/lib/resource/ResourceContent.java delete mode 100644 src/main/java/com/avaje/ebeaninternal/server/lib/resource/ResourceSource.java delete mode 100644 src/main/java/com/avaje/ebeaninternal/server/lib/resource/package.html delete mode 100644 src/main/java/com/avaje/ebeaninternal/server/resource/ResourceManager.java delete mode 100644 src/main/java/com/avaje/ebeaninternal/server/resource/ResourceManagerFactory.java delete mode 100644 src/main/java/com/avaje/ebeaninternal/server/resource/package.html diff --git a/src/main/java/com/avaje/ebean/config/AutofetchConfig.java b/src/main/java/com/avaje/ebean/config/AutofetchConfig.java index 02dbdda38..96aa821d5 100644 --- a/src/main/java/com/avaje/ebean/config/AutofetchConfig.java +++ b/src/main/java/com/avaje/ebean/config/AutofetchConfig.java @@ -7,19 +7,15 @@ public class AutofetchConfig { private AutofetchMode mode = AutofetchMode.DEFAULT_ONIFEMPTY; - private boolean queryTuning = false; + private boolean queryTuning; - private boolean queryTuningAddVersion = false; + private boolean queryTuningAddVersion; - private boolean profiling = false; + private boolean profiling; - private int profilingMin = 1; + private int profilingBase = 5; - private int profilingBase = 10; - - private double profilingRate = 0.05; - - private String logDirectory; + private double profilingRate = 0.01; private int profileUpdateFrequency = 60; @@ -101,22 +97,6 @@ public class AutofetchConfig { this.profiling = profiling; } - /** - * Return the minimum number of queries to profile before autofetch will start - * tuning the queries. - */ - public int getProfilingMin() { - return profilingMin; - } - - /** - * Set the minimum number of queries to profile before autofetch will start - * tuning the queries. - */ - public void setProfilingMin(int profilingMin) { - this.profilingMin = profilingMin; - } - /** * Return the base number of queries to profile before changing to profile * only a percentage of following queries (profileRate). @@ -148,20 +128,6 @@ public class AutofetchConfig { this.profilingRate = profilingRate; } - /** - * Return the log directory to put the autofetch log. - */ - public String getLogDirectory() { - return logDirectory; - } - - /** - * Set the directory to put the autofetch log in. - */ - public void setLogDirectory(String logDirectory) { - this.logDirectory = logDirectory; - } - /** * Return the frequency in seconds to update the autofetch tuned queries from * the profiled information. @@ -228,18 +194,16 @@ public class AutofetchConfig { */ public void loadSettings(PropertiesWrapper p) { - logDirectory = p.get("autofetch.logDirectory", logDirectory); - queryTuning = p.getBoolean("autofetch.querytuning", queryTuning); - queryTuningAddVersion = p.getBoolean("autofetch.queryTuningAddVersion", queryTuningAddVersion); - garbageCollectionOnShutdown = p.getBoolean("autofetch.garbageCollectionOnShutdown", garbageCollectionOnShutdown); + queryTuning = p.getBoolean("autotune.querytuning", queryTuning); + queryTuningAddVersion = p.getBoolean("autotune.queryTuningAddVersion", queryTuningAddVersion); + garbageCollectionOnShutdown = p.getBoolean("autotune.garbageCollectionOnShutdown", garbageCollectionOnShutdown); - profiling = p.getBoolean("autofetch.profiling", profiling); - mode = p.getEnum(AutofetchMode.class, "autofetch.implicitmode", mode); + profiling = p.getBoolean("autotune.profiling", profiling); + mode = p.getEnum(AutofetchMode.class, "autotune.implicitmode", mode); - profilingMin = p.getInt("autofetch.profiling.min", profilingMin); - profilingBase = p.getInt("autofetch.profiling.base", profilingBase); + profilingBase = p.getInt("autotune.profiling.base", profilingBase); - profilingRate = p.getDouble("autofetch.profiling.rate", profilingRate); - profileUpdateFrequency = p.getInt("autofetch.profiling.updatefrequency", profileUpdateFrequency); + profilingRate = p.getDouble("autotune.profiling.rate", profilingRate); + profileUpdateFrequency = p.getInt("autotune.profiling.updatefrequency", profileUpdateFrequency); } } diff --git a/src/main/java/com/avaje/ebeaninternal/server/autofetch/service/AutoTuneServiceFactory.java b/src/main/java/com/avaje/ebeaninternal/server/autofetch/service/AutoTuneServiceFactory.java index c0327c4fb..f730a3b31 100644 --- a/src/main/java/com/avaje/ebeaninternal/server/autofetch/service/AutoTuneServiceFactory.java +++ b/src/main/java/com/avaje/ebeaninternal/server/autofetch/service/AutoTuneServiceFactory.java @@ -8,7 +8,7 @@ public class AutoTuneServiceFactory { public static AutoTuneService create(SpiEbeanServer server, ServerConfig serverConfig) { - return new BaseAutoTuneService(server, serverConfig); + return new DefaultAutoTuneService(server, serverConfig); } } diff --git a/src/main/java/com/avaje/ebeaninternal/server/autofetch/service/BaseAutoTuneService.java b/src/main/java/com/avaje/ebeaninternal/server/autofetch/service/DefaultAutoTuneService.java similarity index 97% rename from src/main/java/com/avaje/ebeaninternal/server/autofetch/service/BaseAutoTuneService.java rename to src/main/java/com/avaje/ebeaninternal/server/autofetch/service/DefaultAutoTuneService.java index 5dc886cfe..3a22f1d57 100644 --- a/src/main/java/com/avaje/ebeaninternal/server/autofetch/service/BaseAutoTuneService.java +++ b/src/main/java/com/avaje/ebeaninternal/server/autofetch/service/DefaultAutoTuneService.java @@ -23,9 +23,9 @@ import java.util.List; /** * Implementation of the AutoTuneService which is comprised of profiling and query tuning. */ -public class BaseAutoTuneService implements AutoTuneService { +public class DefaultAutoTuneService implements AutoTuneService { - private static final Logger logger = LoggerFactory.getLogger(BaseAutoTuneService.class); + private static final Logger logger = LoggerFactory.getLogger(DefaultAutoTuneService.class); private final long defaultGarbageCollectionWait; @@ -35,7 +35,7 @@ public class BaseAutoTuneService implements AutoTuneService { private final ProfileManager profileManager; - public BaseAutoTuneService(SpiEbeanServer server, ServerConfig serverConfig) { + public DefaultAutoTuneService(SpiEbeanServer server, ServerConfig serverConfig) { AutofetchConfig config = serverConfig.getAutofetchConfig(); diff --git a/src/main/java/com/avaje/ebeaninternal/server/core/InternalConfiguration.java b/src/main/java/com/avaje/ebeaninternal/server/core/InternalConfiguration.java index 62c1e2470..9f77e4029 100644 --- a/src/main/java/com/avaje/ebeaninternal/server/core/InternalConfiguration.java +++ b/src/main/java/com/avaje/ebeaninternal/server/core/InternalConfiguration.java @@ -15,9 +15,9 @@ import com.avaje.ebeaninternal.api.SpiBackgroundExecutor; import com.avaje.ebeaninternal.api.SpiEbeanServer; import com.avaje.ebeaninternal.server.autofetch.AutoTuneService; import com.avaje.ebeaninternal.server.autofetch.service.AutoTuneServiceFactory; +import com.avaje.ebeaninternal.server.changelog.DefaultChangeLogListener; import com.avaje.ebeaninternal.server.changelog.DefaultChangeLogPrepare; import com.avaje.ebeaninternal.server.changelog.DefaultChangeLogRegister; -import com.avaje.ebeaninternal.server.changelog.DefaultChangeLogListener; import com.avaje.ebeaninternal.server.cluster.ClusterManager; import com.avaje.ebeaninternal.server.deploy.BeanDescriptorManager; import com.avaje.ebeaninternal.server.deploy.DeployOrmXml; @@ -32,8 +32,6 @@ import com.avaje.ebeaninternal.server.persist.DefaultPersister; import com.avaje.ebeaninternal.server.query.CQueryEngine; import com.avaje.ebeaninternal.server.query.DefaultOrmQueryEngine; import com.avaje.ebeaninternal.server.query.DefaultRelationalQueryEngine; -import com.avaje.ebeaninternal.server.resource.ResourceManager; -import com.avaje.ebeaninternal.server.resource.ResourceManagerFactory; import com.avaje.ebeaninternal.server.text.json.DJsonContext; import com.avaje.ebeaninternal.server.transaction.AutoCommitTransactionManager; import com.avaje.ebeaninternal.server.transaction.DefaultTransactionScopeManager; @@ -68,8 +66,6 @@ public class InternalConfiguration { private final DeployInherit deployInherit; - private final ResourceManager resourceManager; - private final DeployOrmXml deployOrmXml; private final TypeManager typeManager; @@ -120,8 +116,7 @@ public class InternalConfiguration { this.typeManager = new DefaultTypeManager(serverConfig, bootupClasses); - this.resourceManager = ResourceManagerFactory.createResourceManager(serverConfig); - this.deployOrmXml = new DeployOrmXml(resourceManager.getResourceSource()); + this.deployOrmXml = new DeployOrmXml(); this.deployInherit = new DeployInherit(bootupClasses); this.deployCreateProperties = new DeployCreateProperties(typeManager); @@ -288,10 +283,6 @@ public class InternalConfiguration { return deployInherit; } - public ResourceManager getResourceManager() { - return resourceManager; - } - public DeployOrmXml getDeployOrmXml() { return deployOrmXml; } diff --git a/src/main/java/com/avaje/ebeaninternal/server/deploy/DeployOrmXml.java b/src/main/java/com/avaje/ebeaninternal/server/deploy/DeployOrmXml.java index 833d1a5cc..fedc80baf 100644 --- a/src/main/java/com/avaje/ebeaninternal/server/deploy/DeployOrmXml.java +++ b/src/main/java/com/avaje/ebeaninternal/server/deploy/DeployOrmXml.java @@ -1,18 +1,16 @@ package com.avaje.ebeaninternal.server.deploy; +import com.avaje.ebeaninternal.server.lib.util.Dnode; +import com.avaje.ebeaninternal.server.lib.util.DnodeReader; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import java.io.IOException; import java.io.InputStream; import java.util.ArrayList; import java.util.HashMap; import java.util.List; -import com.avaje.ebeaninternal.server.lib.resource.ResourceContent; -import com.avaje.ebeaninternal.server.lib.resource.ResourceSource; -import com.avaje.ebeaninternal.server.lib.util.Dnode; -import com.avaje.ebeaninternal.server.lib.util.DnodeReader; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - /** * Controls the creation and caching of BeanManager's, BeanDescriptors, * BeanTable etc for both beans and tables(MapBeans). @@ -27,15 +25,10 @@ public class DeployOrmXml { private final HashMap nativeQueryCache; private final ArrayList ormXmlList; - - private final ResourceSource resSource; - public DeployOrmXml(ResourceSource resSource) { - - this.resSource = resSource; + public DeployOrmXml() { this.nativeQueryCache = new HashMap(); this.ormXmlList = findAllOrmXml(); - initialiseNativeQueries(); } @@ -106,17 +99,7 @@ public class DeployOrmXml { private void readOrmXml(String ormXmlName, ArrayList ormXmlList) { try { - Dnode ormXml; - ResourceContent content = resSource.getContent(ormXmlName); - if (content != null) { - // servlet resource or file system... - ormXml = readOrmXml(content.getInputStream()); - - } else { - // try the classpath... - ormXml = readOrmXmlFromClasspath(ormXmlName); - } - + Dnode ormXml = readOrmXmlFromClasspath(ormXmlName); if (ormXml != null) { ormXml.setAttribute("ebean.filename", ormXmlName); ormXmlList.add(ormXml); diff --git a/src/main/java/com/avaje/ebeaninternal/server/lib/resource/DirectoryFinder.java b/src/main/java/com/avaje/ebeaninternal/server/lib/resource/DirectoryFinder.java deleted file mode 100644 index 0755ba6c8..000000000 --- a/src/main/java/com/avaje/ebeaninternal/server/lib/resource/DirectoryFinder.java +++ /dev/null @@ -1,108 +0,0 @@ -package com.avaje.ebeaninternal.server.lib.resource; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.File; - -/** - * Helper object used to find directories typically from the current working - * directory. - */ -public class DirectoryFinder { - - private static final Logger logger = LoggerFactory.getLogger(DirectoryFinder.class); - - /** - * Find a directory by search through subdirectories. - *

- * For example, used to find the WEB-INF directory starting from the current - * working directory. - *

- * - * 
-	 * 
-	 * // search to a depth of 3 from the current working directory
-	 * // looking for a directory WEB-INF that contains the subdirectory
-	 * // data
-	 * 
-	 * File dir = DirectoryFinder.find(null, "WEB-INF/data", 3);
-	 * if (dir != null) {
-	 * 	//found the directory
-	 * }
-	 *
- */ - public static File find(File startDir, String match, int maxDepth) { - - String matchSub = null; - int slashPos = match.indexOf('/'); - if (slashPos > -1) { - // match has sub directories - matchSub = match.substring(slashPos + 1); - match = match.substring(0, slashPos); - } - - // search for the directory - File found = find(startDir, match, matchSub, 0, maxDepth); - - if (found != null && matchSub != null) { - // match has sub directories - return new File(found, matchSub); - } - return found; - } - - private static File find(File dir, String match, String matchSub, int depth, int maxDepth) { - - if (dir == null) { - String curDir = System.getProperty("user.dir"); - dir = new File(curDir); - } - - if (dir.exists()) { - File[] list = dir.listFiles(); - if (list != null){ - for (int i = 0; i < list.length; i++) { - if (isMatch(list[i], match, matchSub)) { - return list[i]; - } - } - - // go through the directories again - // Aka *NOT* a depth first search - if (depth < maxDepth) { - for (int i = 0; i < list.length; i++) { - if (list[i].isDirectory()) { - File found = find(list[i], match, matchSub, depth + 1, maxDepth); - if (found != null) { - return found; - } - } - } - } - } - } - return null; - } - - private static boolean isMatch(File f, String match, String matchSub) { - if (f == null) { - return false; - } - if (!f.isDirectory()) { - return false; - } - if (!f.getName().equalsIgnoreCase(match)) { - return false; - } - if (matchSub == null) { - return true; - } - File sub = new File(f, matchSub); - if (logger.isTraceEnabled()){ - logger.trace("search; " + f.getPath()); - } - return sub.exists(); - - } -} diff --git a/src/main/java/com/avaje/ebeaninternal/server/lib/resource/FileResourceContent.java b/src/main/java/com/avaje/ebeaninternal/server/lib/resource/FileResourceContent.java deleted file mode 100644 index b2f6cb07e..000000000 --- a/src/main/java/com/avaje/ebeaninternal/server/lib/resource/FileResourceContent.java +++ /dev/null @@ -1,70 +0,0 @@ -package com.avaje.ebeaninternal.server.lib.resource; - -import java.io.File; -import java.io.FileInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.util.Date; - -/** - * Content from a file system file. - */ -public class FileResourceContent implements ResourceContent { - - /** - * The underlying file. - */ - final File file; - - final String entryName; - - /** - * Create with a File and the entryName. - */ - public FileResourceContent(File file, String entryName) { - this.file = file; - this.entryName = entryName; - } - - public String toString() { - StringBuilder sb = new StringBuilder(); - sb.append("[").append(getName()); - sb.append("] size[").append(size()); - sb.append("] lastModified[").append(new Date(lastModified())); - sb.append("]"); - return sb.toString(); - } - - /** - * Returns the entry name which contains the path from the base directory. - *

- * This does not return the full path of the file, but the path relative to - * the FileIoSource directory. - *

- */ - public String getName() { - return entryName; - } - - /** - * Return the time the file was last modified. - */ - public long lastModified() { - return file.lastModified(); - } - - /** - * Return the size of the file. - */ - public long size() { - return file.length(); - } - - /** - * Return the input stream for this file. - */ - public InputStream getInputStream() throws IOException { - - return new FileInputStream(file); - } -} diff --git a/src/main/java/com/avaje/ebeaninternal/server/lib/resource/FileResourceSource.java b/src/main/java/com/avaje/ebeaninternal/server/lib/resource/FileResourceSource.java deleted file mode 100644 index 496660c1e..000000000 --- a/src/main/java/com/avaje/ebeaninternal/server/lib/resource/FileResourceSource.java +++ /dev/null @@ -1,50 +0,0 @@ -package com.avaje.ebeaninternal.server.lib.resource; - -import java.io.File; - -/** - * A file system directory represented as a FileSource. - */ -public class FileResourceSource implements ResourceSource { - - /** - * The directory name. - */ - final String directory; - - final String baseDir; - - /** - * Create the source based on a directory name. - */ - public FileResourceSource(String directory) { - this.directory = directory; - this.baseDir = directory + File.separator; - } - - /** - * Create the source based on a directory file. - */ - public FileResourceSource(File dir) { - this(dir.getPath()); - } - - - public String getRealPath() { - return directory; - } - - /** - * Search for the given file and return as IoContent. - */ - public ResourceContent getContent(String entry) { - - String fullPath = baseDir + entry; - - File f = new File(fullPath); - if (f.exists()) { - return new FileResourceContent(f, entry); - } - return null; - } -} diff --git a/src/main/java/com/avaje/ebeaninternal/server/lib/resource/ResourceContent.java b/src/main/java/com/avaje/ebeaninternal/server/lib/resource/ResourceContent.java deleted file mode 100644 index 6aab953c7..000000000 --- a/src/main/java/com/avaje/ebeaninternal/server/lib/resource/ResourceContent.java +++ /dev/null @@ -1,34 +0,0 @@ -package com.avaje.ebeaninternal.server.lib.resource; - -import java.io.IOException; -import java.io.InputStream; - -/** - * Represents content that can be read via the ResourceManager. - *

- * Typically either content from a File or a URL. - *

- */ -public interface ResourceContent { - - /** - * The name of the content. - */ - String getName(); - - /** - * The size of the content in bytes. - */ - long size(); - - /** - * The last modified timestamp of the content. - */ - long lastModified(); - - /** - * The content itself. - */ - InputStream getInputStream() throws IOException; - -} diff --git a/src/main/java/com/avaje/ebeaninternal/server/lib/resource/ResourceSource.java b/src/main/java/com/avaje/ebeaninternal/server/lib/resource/ResourceSource.java deleted file mode 100644 index 1be9338bb..000000000 --- a/src/main/java/com/avaje/ebeaninternal/server/lib/resource/ResourceSource.java +++ /dev/null @@ -1,26 +0,0 @@ -package com.avaje.ebeaninternal.server.lib.resource; - -/** - * A Source for ResourceManager. - *

- * Typically a File System Directory based source or a ServletContext URL - * resource based source (for Servlet WAR files). - *

- */ -public interface ResourceSource { - - /** - * Return the File System path of the root of the ResourceSource. - *

- * This will return null IF the ResourceSource is an unpacked WAR file. - *

- */ - String getRealPath(); - - /** - * Find the content with a given entry name. This will return null if no - * matching content was found. - */ - ResourceContent getContent(String entry); - -} diff --git a/src/main/java/com/avaje/ebeaninternal/server/lib/resource/package.html b/src/main/java/com/avaje/ebeaninternal/server/lib/resource/package.html deleted file mode 100644 index 9ca4bbdba..000000000 --- a/src/main/java/com/avaje/ebeaninternal/server/lib/resource/package.html +++ /dev/null @@ -1,13 +0,0 @@ - - - - AvajeLib - - -Service for reading content from either the file system or servlet context. -

-A service used to read deployment content such as xml files, images etc -taking into account the environment (servlet war file or file system). -

- - \ No newline at end of file diff --git a/src/main/java/com/avaje/ebeaninternal/server/resource/ResourceManager.java b/src/main/java/com/avaje/ebeaninternal/server/resource/ResourceManager.java deleted file mode 100644 index 087e1d2cd..000000000 --- a/src/main/java/com/avaje/ebeaninternal/server/resource/ResourceManager.java +++ /dev/null @@ -1,29 +0,0 @@ -package com.avaje.ebeaninternal.server.resource; - -import java.io.File; - -import com.avaje.ebeaninternal.server.lib.resource.ResourceSource; - -/** - * The ResourceManager implementation. - */ -public class ResourceManager { - - final ResourceSource resourceSource; - - final File autofetchDir; - - public ResourceManager(ResourceSource resourceSource, File autofetchDir) { - this.resourceSource = resourceSource; - this.autofetchDir = autofetchDir; - } - - public ResourceSource getResourceSource() { - return resourceSource; - } - - public File getAutofetchDirectory() { - return autofetchDir; - } - -} diff --git a/src/main/java/com/avaje/ebeaninternal/server/resource/ResourceManagerFactory.java b/src/main/java/com/avaje/ebeaninternal/server/resource/ResourceManagerFactory.java deleted file mode 100644 index a32fd5625..000000000 --- a/src/main/java/com/avaje/ebeaninternal/server/resource/ResourceManagerFactory.java +++ /dev/null @@ -1,107 +0,0 @@ -package com.avaje.ebeaninternal.server.resource; - -import com.avaje.ebean.config.ServerConfig; -import com.avaje.ebeaninternal.server.lib.resource.DirectoryFinder; -import com.avaje.ebeaninternal.server.lib.resource.FileResourceSource; -import com.avaje.ebeaninternal.server.lib.resource.ResourceSource; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.File; - -/** - * Creates a ResourceManager for a server depending on the avaje.properties. - *

- * This can use URL based resource loading for web applications or file based - * otherwise. - *

- */ -public class ResourceManagerFactory { - - private static final Logger logger = LoggerFactory.getLogger(ResourceManagerFactory.class); - - /** - * Construct with the properties for a server. - */ - public ResourceManagerFactory() { - } - - /** - * Create the resource manager given the properties for this server. - */ - public static ResourceManager createResourceManager(ServerConfig serverConfig) { - - ResourceSource resourceSource = createResourceSource(serverConfig); - File autofetchDir = getAutofetchDir(serverConfig, resourceSource); - - return new ResourceManager(resourceSource, autofetchDir); - } - - - /** - * Return the directory that autofetch file goes into. - */ - protected static File getAutofetchDir(ServerConfig serverConfig, ResourceSource resourceSource) { - - String dir = null; - if (serverConfig.getAutofetchConfig() != null) { - dir = serverConfig.getAutofetchConfig().getLogDirectory(); - } - if (dir != null) { - return new File(dir); - } - - String realPath = resourceSource.getRealPath(); - if (realPath != null) { - return new File(realPath); - - } else { - throw new RuntimeException("No autofetch directory set?"); - } - } - - /** - * Return the resource loader for external sql files. - *

- * This can be url based (for webapps) or otherwise file based. - *

- */ - protected static ResourceSource createResourceSource(ServerConfig serverConfig) { - - // default for web application, override this for file system - String defaultDir = serverConfig.getResourceDirectory(); - - // use File System directory - return createFileSource(defaultDir); - } - - private static ResourceSource createFileSource(String fileDir) { - - if (fileDir != null) { - // explicitly stated so - File dir = new File(fileDir); - if (dir.exists()) { - logger.info("ResourceManager initialised: type[file] [" + fileDir + "]"); - return new FileResourceSource(fileDir); - } else { - String msg = "ResourceManager could not find directory [" + fileDir + "]"; - throw new IllegalArgumentException(msg); - } - } - - // try to guess the directory starting from the current working - // directory, and searching to a maximum depth of 3 subdirectories - File guessDir = DirectoryFinder.find(null, "WEB-INF", 3); - if (guessDir != null) { - // Typically this means we found the WEB-INF directory below the - // current working directory - logger.info("ResourceManager initialised: type[file] [" + guessDir.getPath() + "]"); - return new FileResourceSource(guessDir.getPath()); - } - - // default to the current working directory - File workingDir = new File("."); - return new FileResourceSource(workingDir); - } - -} diff --git a/src/main/java/com/avaje/ebeaninternal/server/resource/package.html b/src/main/java/com/avaje/ebeaninternal/server/resource/package.html deleted file mode 100644 index abed4e076..000000000 --- a/src/main/java/com/avaje/ebeaninternal/server/resource/package.html +++ /dev/null @@ -1,10 +0,0 @@ - - - - Resource Management - - -Resource Management - - - \ No newline at end of file diff --git a/src/test/java/com/avaje/ebeaninternal/server/querydefn/OrmQueryDetailTest.java b/src/test/java/com/avaje/ebeaninternal/server/querydefn/OrmQueryDetailTest.java index 4a6a18ec6..3fa127b73 100644 --- a/src/test/java/com/avaje/ebeaninternal/server/querydefn/OrmQueryDetailTest.java +++ b/src/test/java/com/avaje/ebeaninternal/server/querydefn/OrmQueryDetailTest.java @@ -10,10 +10,10 @@ public class OrmQueryDetailTest { @Test public void test_isAutoTuneEqual() { - OrmQueryDetailParser parser1 = new OrmQueryDetailParser("select order (id,name) fetch customer (name) fetch details (code)"); + OrmQueryDetailParser parser1 = new OrmQueryDetailParser("select (id,name) fetch customer (name) fetch details (code)"); OrmQueryDetail detail1 = parser1.parse(); - OrmQueryDetailParser parser2 = new OrmQueryDetailParser("select order (id,name) fetch details (code) fetch customer (name)"); + OrmQueryDetailParser parser2 = new OrmQueryDetailParser("select (id,name) fetch details (code) fetch customer (name)"); OrmQueryDetail detail2 = parser2.parse(); assertTrue(detail1.isAutoTuneEqual(detail2)); From 71dd0e57286fb35f06a2a951bb386d2aa0a8b33b Mon Sep 17 00:00:00 2001 From: Robin Bygrave Date: Thu, 10 Sep 2015 00:17:49 +1200 Subject: [PATCH 5/9] AutoTune update - rename and profiling against original query --- .../java/com/avaje/ebean/EbeanServer.java | 5 +- .../com/avaje/ebean/config/ServerConfig.java | 16 +- .../server/autofetch/ProfilingListener.java | 3 +- .../server/autofetch/model/Origin.java | 18 +- .../autofetch/service/ProfileManager.java | 16 +- .../autofetch/service/ProfileOrigin.java | 12 +- .../server/querydefn/OrmQueryDetail.java | 24 +-- .../server/querydefn/OrmQueryProperties.java | 11 +- .../ebeaninternal/api/TDSpiEbeanServer.java | 2 +- .../querydefn/OrmQueryDetailParserTest.java | 39 ++++ .../com/avaje/tests/basic/TestBatchLazy.java | 85 ++++----- .../avaje/tests/basic/lob/TestByteOnly.java | 4 +- .../query/TestAutofetchTuneWithJoin.java | 160 ++++++++-------- .../query/autotune/TestAutoTuneProfiling.java | 178 ++++++++++-------- src/test/resources/ebean.properties | 17 +- 15 files changed, 322 insertions(+), 268 deletions(-) diff --git a/src/main/java/com/avaje/ebean/EbeanServer.java b/src/main/java/com/avaje/ebean/EbeanServer.java index 3f19306c8..327d89275 100644 --- a/src/main/java/com/avaje/ebean/EbeanServer.java +++ b/src/main/java/com/avaje/ebean/EbeanServer.java @@ -106,10 +106,9 @@ public interface EbeanServer { void shutdown(boolean shutdownDataSource, boolean deregisterDriver); /** - * Return the AdminAutofetch which is used to control and configure the - * Autofetch service at runtime. + * Return AutoTune which is used to control the AutoTune service at runtime. */ - AdminAutofetch getAdminAutofetch(); + AutoTune getAutoTune(); /** * Return the name. This is used with {@link Ebean#getServer(String)} to get a diff --git a/src/main/java/com/avaje/ebean/config/ServerConfig.java b/src/main/java/com/avaje/ebean/config/ServerConfig.java index 1cdc2290c..8e13cadc4 100644 --- a/src/main/java/com/avaje/ebean/config/ServerConfig.java +++ b/src/main/java/com/avaje/ebean/config/ServerConfig.java @@ -130,7 +130,7 @@ public class ServerConfig { /** * Config controlling the autofetch behaviour. */ - private AutofetchConfig autofetchConfig = new AutofetchConfig(); + private AutoTuneConfig autoTuneConfig = new AutoTuneConfig(); /** * The JSON format used for DateTime types. Default to millis. @@ -1106,15 +1106,15 @@ public class ServerConfig { /** * Return the configuration for the Autofetch feature. */ - public AutofetchConfig getAutofetchConfig() { - return autofetchConfig; + public AutoTuneConfig getAutoTuneConfig() { + return autoTuneConfig; } /** * Set the configuration for the Autofetch feature. */ - public void setAutofetchConfig(AutofetchConfig autofetchConfig) { - this.autofetchConfig = autofetchConfig; + public void setAutoTuneConfig(AutoTuneConfig autoTuneConfig) { + this.autoTuneConfig = autoTuneConfig; } /** @@ -2003,7 +2003,7 @@ public class ServerConfig { * This is broken out for the same reason as above - preserve existing behaviour but let it be overridden. */ protected void loadAutofetchSettings(PropertiesWrapper p) { - autofetchConfig.loadSettings(p); + autoTuneConfig.loadSettings(p); } /** @@ -2017,8 +2017,8 @@ public class ServerConfig { if (namingConvention != null) { namingConvention.loadFromProperties(p); } - if (autofetchConfig == null) { - autofetchConfig = new AutofetchConfig(); + if (autoTuneConfig == null) { + autoTuneConfig = new AutoTuneConfig(); } loadAutofetchSettings(p); diff --git a/src/main/java/com/avaje/ebeaninternal/server/autofetch/ProfilingListener.java b/src/main/java/com/avaje/ebeaninternal/server/autofetch/ProfilingListener.java index ca496456f..f24300b91 100644 --- a/src/main/java/com/avaje/ebeaninternal/server/autofetch/ProfilingListener.java +++ b/src/main/java/com/avaje/ebeaninternal/server/autofetch/ProfilingListener.java @@ -2,6 +2,7 @@ package com.avaje.ebeaninternal.server.autofetch; import com.avaje.ebean.bean.NodeUsageListener; import com.avaje.ebean.bean.ObjectGraphNode; +import com.avaje.ebeaninternal.api.SpiQuery; /** * Profiling listener gets call backs for node usage and the associated query executions. @@ -21,5 +22,5 @@ public interface ProfilingListener extends NodeUsageListener { * Return true if this request should be profiled (based on the * profiling ratio and collection count for this origin). */ - boolean isProfileRequest(ObjectGraphNode origin); + boolean isProfileRequest(ObjectGraphNode origin, SpiQuery query); } diff --git a/src/main/java/com/avaje/ebeaninternal/server/autofetch/model/Origin.java b/src/main/java/com/avaje/ebeaninternal/server/autofetch/model/Origin.java index 86eebba46..eb2203c92 100644 --- a/src/main/java/com/avaje/ebeaninternal/server/autofetch/model/Origin.java +++ b/src/main/java/com/avaje/ebeaninternal/server/autofetch/model/Origin.java @@ -23,7 +23,7 @@ import javax.xml.bind.annotation.XmlType; * <attribute name="key" use="required" type="{http://www.w3.org/2001/XMLSchema}string" /> * <attribute name="beanType" type="{http://www.w3.org/2001/XMLSchema}string" /> * <attribute name="detail" type="{http://www.w3.org/2001/XMLSchema}string" /> - * <attribute name="tuneDetail" type="{http://www.w3.org/2001/XMLSchema}string" /> + * <attribute name="original" type="{http://www.w3.org/2001/XMLSchema}string" /> * </restriction> * </complexContent> * </complexType> @@ -45,8 +45,8 @@ public class Origin { protected String beanType; @XmlAttribute(name = "detail") protected String detail; - @XmlAttribute(name = "tuneDetail") - protected String tuneDetail; + @XmlAttribute(name = "original") + protected String original; /** * Gets the value of the callStack property. @@ -145,27 +145,27 @@ public class Origin { } /** - * Gets the value of the tuneDetail property. + * Gets the value of the original property. * * @return * possible object is * {@link String } * */ - public String getTuneDetail() { - return tuneDetail; + public String getOriginal() { + return original; } /** - * Sets the value of the tuneDetail property. + * Sets the value of the original property. * * @param value * allowed object is * {@link String } * */ - public void setTuneDetail(String value) { - this.tuneDetail = value; + public void setOriginal(String value) { + this.original = value; } } diff --git a/src/main/java/com/avaje/ebeaninternal/server/autofetch/service/ProfileManager.java b/src/main/java/com/avaje/ebeaninternal/server/autofetch/service/ProfileManager.java index b76e66e47..4ef3b464a 100644 --- a/src/main/java/com/avaje/ebeaninternal/server/autofetch/service/ProfileManager.java +++ b/src/main/java/com/avaje/ebeaninternal/server/autofetch/service/ProfileManager.java @@ -3,8 +3,9 @@ package com.avaje.ebeaninternal.server.autofetch.service; import com.avaje.ebean.bean.NodeUsageCollector; import com.avaje.ebean.bean.ObjectGraphNode; import com.avaje.ebean.bean.ObjectGraphOrigin; -import com.avaje.ebean.config.AutofetchConfig; +import com.avaje.ebean.config.AutoTuneConfig; import com.avaje.ebeaninternal.api.SpiEbeanServer; +import com.avaje.ebeaninternal.api.SpiQuery; import com.avaje.ebeaninternal.server.autofetch.AutoTuneCollection; import com.avaje.ebeaninternal.server.autofetch.ProfilingListener; import com.avaje.ebeaninternal.server.deploy.BeanDescriptor; @@ -40,7 +41,7 @@ public class ProfileManager implements ProfilingListener { private final SpiEbeanServer server; - public ProfileManager(AutofetchConfig config, SpiEbeanServer server) { + public ProfileManager(AutoTuneConfig config, SpiEbeanServer server) { this.server = server; this.profilingRate = config.getProfilingRate(); this.profilingBase = config.getProfilingBase(); @@ -48,10 +49,17 @@ public class ProfileManager implements ProfilingListener { } @Override - public boolean isProfileRequest(ObjectGraphNode origin) { + public boolean isProfileRequest(ObjectGraphNode origin, SpiQuery query) { ProfileOrigin profileOrigin = profileMap.get(origin.getOriginQueryPoint().getKey()); - return profileOrigin == null || profileOrigin.isProfile(); + if (profileOrigin == null) { + profileOrigin = new ProfileOrigin(origin.getOriginQueryPoint(), queryTuningAddVersion, profilingBase, profilingRate); + profileOrigin.setOriginalQuery(query.getDetail().toString()); + profileMap.put(origin.getOriginQueryPoint().getKey(), profileOrigin); + return true; + } else { + return profileOrigin.isProfile(); + } } /** diff --git a/src/main/java/com/avaje/ebeaninternal/server/autofetch/service/ProfileOrigin.java b/src/main/java/com/avaje/ebeaninternal/server/autofetch/service/ProfileOrigin.java index 411f9756f..d83c163d9 100644 --- a/src/main/java/com/avaje/ebeaninternal/server/autofetch/service/ProfileOrigin.java +++ b/src/main/java/com/avaje/ebeaninternal/server/autofetch/service/ProfileOrigin.java @@ -38,6 +38,8 @@ public class ProfileOrigin { private final AtomicLong profileCount = new AtomicLong(); + private String originalQuery; + public ProfileOrigin(ObjectGraphOrigin origin, boolean queryTuningAddVersion, int profilingBase, double profilingRate) { this.origin = origin; this.queryTuningAddVersion = queryTuningAddVersion; @@ -45,6 +47,14 @@ public class ProfileOrigin { this.profilingRate = profilingRate; } + public String getOriginalQuery() { + return originalQuery; + } + + public void setOriginalQuery(String originalQuery) { + this.originalQuery = originalQuery; + } + /** * Return true if this query should be profiled based on a percentage rate. */ @@ -74,7 +84,7 @@ public class ProfileOrigin { } OrmQueryDetail detail = buildDetail(rootDesc); - AutoTuneCollection.Entry entry = req.add(origin, detail); + AutoTuneCollection.Entry entry = req.add(origin, detail, originalQuery); Collection values = queryStatsMap.values(); for (ProfileOriginQuery queryEntry : values) { diff --git a/src/main/java/com/avaje/ebeaninternal/server/querydefn/OrmQueryDetail.java b/src/main/java/com/avaje/ebeaninternal/server/querydefn/OrmQueryDetail.java index 954c5f4fe..0f4ab5f46 100644 --- a/src/main/java/com/avaje/ebeaninternal/server/querydefn/OrmQueryDetail.java +++ b/src/main/java/com/avaje/ebeaninternal/server/querydefn/OrmQueryDetail.java @@ -81,7 +81,7 @@ public class OrmQueryDetail implements Serializable { } /** - * Return true if equal in terms of autofetch (select and joins). + * Return true if equal in terms of autoTune (select and fetch). */ public boolean isAutoTuneEqual(OrmQueryDetail otherDetail) { @@ -91,6 +91,9 @@ public class OrmQueryDetail implements Serializable { if (fetchPaths == null) { return otherDetail.fetchPaths == null; } + if (fetchPaths.size() != otherDetail.fetchPaths.size()) { + return false; + } Set> entries = fetchPaths.entrySet(); for (Map.Entry entry : entries) { OrmQueryProperties chunk = otherDetail.getChunk(entry.getKey(), false); @@ -100,7 +103,6 @@ public class OrmQueryDetail implements Serializable { } return true; - //return autofetchPlanHash() == otherDetail.autofetchPlanHash(); } private boolean isSame(OrmQueryProperties p1, OrmQueryProperties p2) { @@ -110,22 +112,6 @@ public class OrmQueryDetail implements Serializable { return p1.isSame(p2); } -// /** -// * Calculate the hash for the query plan. -// */ -// private int autofetchPlanHash() { -// -// int hc = (baseProps == null ? 1 : baseProps.autofetchPlanHash()); -// -// if (fetchPaths != null) { -// for (OrmQueryProperties p : fetchPaths.values()) { -// hc = hc * 31 + p.autofetchPlanHash(); -// } -// } -// -// return hc; -// } - public String toString() { StringBuilder sb = new StringBuilder(); if (baseProps != null) { @@ -236,7 +222,7 @@ public class OrmQueryDetail implements Serializable { boolean tuned = false; OrmQueryProperties tunedRoot = tunedDetail.getChunk(null, false); - if (tunedRoot != null && tunedRoot.hasProperties()) { + if (tunedRoot != null) { tuned = true; baseProps.setTunedProperties(tunedRoot); diff --git a/src/main/java/com/avaje/ebeaninternal/server/querydefn/OrmQueryProperties.java b/src/main/java/com/avaje/ebeaninternal/server/querydefn/OrmQueryProperties.java index 23d270ce4..21f2e566b 100644 --- a/src/main/java/com/avaje/ebeaninternal/server/querydefn/OrmQueryProperties.java +++ b/src/main/java/com/avaje/ebeaninternal/server/querydefn/OrmQueryProperties.java @@ -215,9 +215,13 @@ public class OrmQueryProperties implements Serializable { * Set the properties from a matching autofetch tuned properties. */ public void setTunedProperties(OrmQueryProperties tunedProperties) { - this.properties = tunedProperties.properties; - this.trimmedProperties = tunedProperties.trimmedProperties; - this.included = tunedProperties.included; + if (tunedProperties.hasProperties()) { + this.properties = tunedProperties.properties; + this.trimmedProperties = tunedProperties.trimmedProperties; + this.included = tunedProperties.included; + this.queryFetchBatch = Math.max(queryFetchBatch, tunedProperties.queryFetchBatch); + this.lazyFetchBatch = Math.max(lazyFetchBatch, tunedProperties.lazyFetchBatch); + } } /** @@ -262,6 +266,7 @@ public class OrmQueryProperties implements Serializable { copy.parentPath = parentPath; copy.path = path; copy.properties = properties; + copy.trimmedProperties = trimmedProperties; copy.cache = cache; copy.readOnly = readOnly; copy.queryFetchAll = queryFetchAll; diff --git a/src/test/java/com/avaje/ebeaninternal/api/TDSpiEbeanServer.java b/src/test/java/com/avaje/ebeaninternal/api/TDSpiEbeanServer.java index adbcc84d8..d6f443bc7 100644 --- a/src/test/java/com/avaje/ebeaninternal/api/TDSpiEbeanServer.java +++ b/src/test/java/com/avaje/ebeaninternal/api/TDSpiEbeanServer.java @@ -216,7 +216,7 @@ public class TDSpiEbeanServer implements SpiEbeanServer { } @Override - public AdminAutofetch getAdminAutofetch() { + public AutoTune getAutoTune() { return null; } diff --git a/src/test/java/com/avaje/ebeaninternal/server/querydefn/OrmQueryDetailParserTest.java b/src/test/java/com/avaje/ebeaninternal/server/querydefn/OrmQueryDetailParserTest.java index 5ddd475b1..0e877728c 100644 --- a/src/test/java/com/avaje/ebeaninternal/server/querydefn/OrmQueryDetailParserTest.java +++ b/src/test/java/com/avaje/ebeaninternal/server/querydefn/OrmQueryDetailParserTest.java @@ -77,4 +77,43 @@ public class OrmQueryDetailParserTest extends BaseTestCase { assertThat(chunk.getAllIncludedProperties()).contains("sku","description"); } + @Test + public void testParseWithPlusQuery() throws Exception { + + OrmQueryDetailParser p = new OrmQueryDetailParser("select (id,name) fetch customer (+query,id,name,email)"); + OrmQueryDetail detail = p.parse(); + + OrmQueryProperties root = detail.getChunk(null, false); + assertNull(root.getPath()); + assertThat(root.getAllIncludedProperties()).contains("id", "name"); + + OrmQueryProperties chunk = detail.getChunk("customer", false); + assertThat(chunk.getPath()).isEqualTo("customer"); + assertThat(chunk.getAllIncludedProperties()).contains("id", "name", "email"); + assertThat(chunk.isQueryFetch()).isTrue(); + + } + + @Test + public void testTuneApply() { + + OrmQueryDetailParser p = new OrmQueryDetailParser("select (status) fetch customer (email)"); + OrmQueryDetail detail = p.parse(); + + OrmQueryDetailParser p2 = new OrmQueryDetailParser("select (id,name) fetch customer (+query,id,name,email)"); + OrmQueryDetail tune = p2.parse(); + + + detail.tuneFetchProperties(tune); + + OrmQueryProperties root = detail.getChunk(null, false); + assertNull(root.getPath()); + assertThat(root.getAllIncludedProperties()).contains("id", "name"); + + OrmQueryProperties chunk = detail.getChunk("customer", false); + assertThat(chunk.getPath()).isEqualTo("customer"); + assertThat(chunk.getAllIncludedProperties()).contains("id", "name", "email"); + assertThat(chunk.isQueryFetch()).isTrue(); + } + } \ No newline at end of file diff --git a/src/test/java/com/avaje/tests/basic/TestBatchLazy.java b/src/test/java/com/avaje/tests/basic/TestBatchLazy.java index 607d0c4cb..8a2e3ca37 100644 --- a/src/test/java/com/avaje/tests/basic/TestBatchLazy.java +++ b/src/test/java/com/avaje/tests/basic/TestBatchLazy.java @@ -1,44 +1,41 @@ -package com.avaje.tests.basic; - -import java.util.List; - -import org.junit.Test; - -import com.avaje.ebean.AdminAutofetch; -import com.avaje.ebean.BaseTestCase; -import com.avaje.ebean.Ebean; -import com.avaje.ebean.Query; -import com.avaje.tests.model.basic.Customer; -import com.avaje.tests.model.basic.Order; -import com.avaje.tests.model.basic.OrderDetail; -import com.avaje.tests.model.basic.ResetBasicData; - -public class TestBatchLazy extends BaseTestCase { - - @Test - public void testMe() { - - ResetBasicData.reset(); - - Query query = Ebean.find(Order.class); - List list = query.findList(); - - - for (Order order : list) { - Customer customer = order.getCustomer(); - customer.getName(); - - List details = order.getDetails(); - for (OrderDetail orderDetail : details) { - orderDetail.getProduct().getSku(); - } - } - - AdminAutofetch adminAutofetch = Ebean.getServer(null).getAdminAutofetch(); - adminAutofetch.collectUsageViaGC(); - - } - - - -} +package com.avaje.tests.basic; + +import com.avaje.ebean.BaseTestCase; +import com.avaje.ebean.Ebean; +import com.avaje.ebean.Query; +import com.avaje.tests.model.basic.Customer; +import com.avaje.tests.model.basic.Order; +import com.avaje.tests.model.basic.OrderDetail; +import com.avaje.tests.model.basic.ResetBasicData; +import org.junit.Test; + +import java.util.List; + +public class TestBatchLazy extends BaseTestCase { + + @Test + public void testMe() { + + ResetBasicData.reset(); + + Query query = Ebean.find(Order.class); + List list = query.findList(); + + + for (Order order : list) { + Customer customer = order.getCustomer(); + customer.getName(); + + List details = order.getDetails(); + for (OrderDetail orderDetail : details) { + orderDetail.getProduct().getSku(); + } + } + + Ebean.getDefaultServer().getAutoTune().collectProfiling(); + + } + + + +} diff --git a/src/test/java/com/avaje/tests/basic/lob/TestByteOnly.java b/src/test/java/com/avaje/tests/basic/lob/TestByteOnly.java index 758ac7e59..1ce5a64a3 100644 --- a/src/test/java/com/avaje/tests/basic/lob/TestByteOnly.java +++ b/src/test/java/com/avaje/tests/basic/lob/TestByteOnly.java @@ -24,8 +24,8 @@ public class TestByteOnly extends BaseTestCase { Ebean.save(e2); - // Ebean.getServer(null).getAdminAutofetch().collectUsageViaGC(); - // Ebean.getServer(null).getAdminAutofetch().updateTunedQueryInfo(); + // Ebean.getServer(null).getAutoTune().collectProfiling(); + // Ebean.getServer(null).getAutoTune().updateTunedQueryInfo(); System.out.println("done"); } diff --git a/src/test/java/com/avaje/tests/query/TestAutofetchTuneWithJoin.java b/src/test/java/com/avaje/tests/query/TestAutofetchTuneWithJoin.java index d5e2cbdf9..d82fbfe6a 100644 --- a/src/test/java/com/avaje/tests/query/TestAutofetchTuneWithJoin.java +++ b/src/test/java/com/avaje/tests/query/TestAutofetchTuneWithJoin.java @@ -1,82 +1,78 @@ -package com.avaje.tests.query; - -import java.util.List; - -import org.junit.Test; - -import com.avaje.ebean.AdminAutofetch; -import com.avaje.ebean.BaseTestCase; -import com.avaje.ebean.Ebean; -import com.avaje.ebean.Query; -import com.avaje.ebean.bean.ObjectGraphNode; -import com.avaje.ebean.bean.ObjectGraphOrigin; -import com.avaje.ebeaninternal.api.SpiQuery; -import com.avaje.tests.model.basic.Address; -import com.avaje.tests.model.basic.Customer; -import com.avaje.tests.model.basic.Order; -import com.avaje.tests.model.basic.ResetBasicData; - -public class TestAutofetchTuneWithJoin extends BaseTestCase { - - @Test - public void test() { - runQuery(); - collectUsage(); - } - - private void runQuery() { - - ResetBasicData.reset(); - - Query q = Ebean.find(Order.class) - .setAutofetch(true) - //.fetch("customer") - //.fetch("customer.contacts") - .where().lt("id", 3).query(); - - List list = q.findList(); - - for (int i = 0; i < list.size(); i++) { - Order order = list.get(i); - order.getOrderDate(); - order.getShipDate(); - // order.setShipDate(new Date(System.currentTimeMillis())); - Customer customer = order.getCustomer(); - customer.getName(); - Address shippingAddress = customer.getShippingAddress(); - if (shippingAddress != null) { - shippingAddress.getLine1(); - shippingAddress.getCity(); - } - // customer.getContacts() - } - - SpiQuery sq = (SpiQuery) q; - ObjectGraphNode parentNode = sq.getParentNode(); - ObjectGraphOrigin origin = parentNode.getOriginQueryPoint(); - - System.out.println("Origin:" + origin.getKey()); - - // MetaAutoFetchStatistic metaAutoFetchStatistic = - // ((DefaultOrmQuery)q).getMetaAutoFetchStatistic(); - // if (metaAutoFetchStatistic != null) { - // List nodeUsageStats = - // metaAutoFetchStatistic.getNodeUsageStats(); - // System.out.println(nodeUsageStats); - // List queryStats = metaAutoFetchStatistic.getQueryStats(); - // System.out.println(queryStats); - // } - - if (q.isAutofetchTuned()) { - System.out.println("TUNED..."); - } - } - - private static void collectUsage() { - - AdminAutofetch adminAutofetch = Ebean.getServer(null).getAdminAutofetch(); - adminAutofetch.collectUsageViaGC(); - - } - -} +package com.avaje.tests.query; + +import com.avaje.ebean.BaseTestCase; +import com.avaje.ebean.Ebean; +import com.avaje.ebean.Query; +import com.avaje.ebean.bean.ObjectGraphNode; +import com.avaje.ebean.bean.ObjectGraphOrigin; +import com.avaje.ebeaninternal.api.SpiQuery; +import com.avaje.tests.model.basic.Address; +import com.avaje.tests.model.basic.Customer; +import com.avaje.tests.model.basic.Order; +import com.avaje.tests.model.basic.ResetBasicData; +import org.junit.Test; + +import java.util.List; + +public class TestAutofetchTuneWithJoin extends BaseTestCase { + + @Test + public void test() { + runQuery(); + collectUsage(); + } + + private void runQuery() { + + ResetBasicData.reset(); + + Query q = Ebean.find(Order.class) + .setAutofetch(true) + //.fetch("customer") + //.fetch("customer.contacts") + .where().lt("id", 3).query(); + + List list = q.findList(); + + for (int i = 0; i < list.size(); i++) { + Order order = list.get(i); + order.getOrderDate(); + order.getShipDate(); + // order.setShipDate(new Date(System.currentTimeMillis())); + Customer customer = order.getCustomer(); + customer.getName(); + Address shippingAddress = customer.getShippingAddress(); + if (shippingAddress != null) { + shippingAddress.getLine1(); + shippingAddress.getCity(); + } + // customer.getContacts() + } + + SpiQuery sq = (SpiQuery) q; + ObjectGraphNode parentNode = sq.getParentNode(); + ObjectGraphOrigin origin = parentNode.getOriginQueryPoint(); + + System.out.println("Origin:" + origin.getKey()); + + // MetaAutoFetchStatistic metaAutoFetchStatistic = + // ((DefaultOrmQuery)q).getMetaAutoFetchStatistic(); + // if (metaAutoFetchStatistic != null) { + // List nodeUsageStats = + // metaAutoFetchStatistic.getNodeUsageStats(); + // System.out.println(nodeUsageStats); + // List queryStats = metaAutoFetchStatistic.getQueryStats(); + // System.out.println(queryStats); + // } + + if (q.isAutofetchTuned()) { + System.out.println("TUNED..."); + } + } + + private static void collectUsage() { + + Ebean.getDefaultServer().getAutoTune().collectProfiling(); + } + +} diff --git a/src/test/java/com/avaje/tests/query/autotune/TestAutoTuneProfiling.java b/src/test/java/com/avaje/tests/query/autotune/TestAutoTuneProfiling.java index e270607e2..2bcccb34a 100644 --- a/src/test/java/com/avaje/tests/query/autotune/TestAutoTuneProfiling.java +++ b/src/test/java/com/avaje/tests/query/autotune/TestAutoTuneProfiling.java @@ -1,82 +1,96 @@ -package com.avaje.tests.query.autotune; - -import com.avaje.ebean.AdminAutofetch; -import com.avaje.ebean.BaseTestCase; -import com.avaje.ebean.Ebean; -import com.avaje.ebean.Query; -import com.avaje.ebean.bean.ObjectGraphNode; -import com.avaje.ebean.bean.ObjectGraphOrigin; -import com.avaje.ebeaninternal.api.SpiQuery; -import com.avaje.tests.model.basic.Address; -import com.avaje.tests.model.basic.Customer; -import com.avaje.tests.model.basic.Order; -import com.avaje.tests.model.basic.ResetBasicData; -import org.junit.Test; - -import java.util.List; - -public class TestAutoTuneProfiling extends BaseTestCase { - - @Test - public void test() { - - ResetBasicData.reset(); - - for (int i = 0; i < 1; i++) { - execute(); - } - - - collectUsage(); - } - - - private void execute() { - useOrderDate(); - useOrderDateCustomerName(); - useLots(); - } - - - private Order findById(long id) { - return Ebean.find(Order.class) - .setAutofetch(true) - .setId(id) - .findUnique(); - } - - private void useOrderDate() { - Order order = findById(3); - order.getStatus(); - order.getShipDate(); - } - - private void useOrderDateCustomerName() { - Order order = findById(3); - order.getOrderDate(); - order.getCustomer().getName(); - } - - private void useLots() { - - Order order = findById(3); - order.getOrderDate(); - order.getShipDate(); - // order.setShipDate(new Date(System.currentTimeMillis())); - Customer customer = order.getCustomer(); - customer.getName(); - Address shippingAddress = customer.getShippingAddress(); - if (shippingAddress != null) { - shippingAddress.getLine1(); - shippingAddress.getCity(); - } - } - - private static void collectUsage() { - - AdminAutofetch adminAutofetch = Ebean.getServer(null).getAdminAutofetch(); - adminAutofetch.collectUsageViaGC(); - - } - -} +package com.avaje.tests.query.autotune; + +import com.avaje.ebean.BaseTestCase; +import com.avaje.ebean.Ebean; +import com.avaje.tests.model.basic.Address; +import com.avaje.tests.model.basic.Customer; +import com.avaje.tests.model.basic.Order; +import com.avaje.tests.model.basic.OrderDetail; +import com.avaje.tests.model.basic.ResetBasicData; +import org.junit.Test; + +import java.util.List; + +public class TestAutoTuneProfiling extends BaseTestCase { + + @Test + public void test() { + + ResetBasicData.reset(); + + for (int i = 0; i < 1; i++) { + execute(); + } + + + collectUsage(); + } + + + private void execute() { + useOrderDate(); + useOrderDateCustomerName(); + useLots(); + useLotUntuned(); + } + + private Order findById(long id) { + return Ebean.find(Order.class) + .select("status, orderDate, shipDate") + .setId(id) + .findUnique(); + } + + private void useOrderDate() { + Order order = findById(3); + order.getStatus(); + order.getShipDate(); + } + + private void useOrderDateCustomerName() { + Order order = findById(3); + order.getOrderDate(); + order.getCustomer().getName(); + } + + private void useLots() { + + Order order = findById(3); + order.getOrderDate(); + order.getShipDate(); + // order.setShipDate(new Date(System.currentTimeMillis())); + Customer customer = order.getCustomer(); + customer.getName(); + Address shippingAddress = customer.getShippingAddress(); + if (shippingAddress != null) { + shippingAddress.getLine1(); + shippingAddress.getCity(); + } + } + + private void useLotUntuned() { + Order order = findById(3); + List details = order.getDetails(); + for (OrderDetail detail : details) { + detail.getProduct().getName(); + detail.getOrderQty(); + detail.getShipQty(); + detail.getUnitPrice(); + } + + Customer customer = order.getCustomer(); + customer.getName(); + Address billingAddress = customer.getBillingAddress(); + if (billingAddress != null) { + billingAddress.getCity(); + billingAddress.getLine1(); + billingAddress.getLine2(); + } + } + + private static void collectUsage() { + + Ebean.getDefaultServer().getAutoTune().collectProfiling(); + } + +} diff --git a/src/test/resources/ebean.properties b/src/test/resources/ebean.properties index 10323b913..228b08aa6 100644 --- a/src/test/resources/ebean.properties +++ b/src/test/resources/ebean.properties @@ -11,15 +11,14 @@ ebean.encryptKeyManager=com.avaje.tests.basic.encrypt.BasicEncyptKeyManager -ebean.autofetch.querytuning=true -ebean.autofetch.profiling=true -ebean.autofetch.implicitmode=default_off -#ebean.autofetch.implicitmode=default_onifempty -ebean.autofetch.profiling.min=1 -ebean.autofetch.profiling.base=10 -#ebean.autofetch.profiling.rate=0.05 -ebean.autofetch.garbageCollectionOnShutdown=true -ebean.autofetch.traceUsageCollection=true +ebean.autotune.querytuning=true +ebean.autotune.profiling=true +#ebean.autoTune.implicitmode=default_off +#ebean.autotune.implicitmode=default_onifempty +#ebean.autofetch.profiling.min=1 +#ebean.autofetch.profiling.base=10 +##ebean.autofetch.profiling.rate=0.05 +#ebean.autofetch.garbageCollectionOnShutdown=true ebean.ddl.generate=true From a2a4a15f32c489cd1b6b111b27a8eb5a24e3064a Mon Sep 17 00:00:00 2001 From: Robin Bygrave Date: Thu, 10 Sep 2015 00:18:11 +1200 Subject: [PATCH 6/9] AutoTune update - rename and profiling against original query --- ebean-autotune-profiling.xml | 32 --- ebean-autotune.xml | 48 ++-- .../java/com/avaje/ebean/AdminAutofetch.java | 23 -- src/main/java/com/avaje/ebean/AutoTune.java | 14 ++ ...tofetchConfig.java => AutoTuneConfig.java} | 80 +++---- .../{AutofetchMode.java => AutoTuneMode.java} | 2 +- .../server/autofetch/AutoTuneCollection.java | 16 +- .../server/autofetch/AutoTuneService.java | 17 +- .../autofetch/service/BaseQueryTuner.java | 16 +- .../service/DefaultAutoTuneService.java | 210 ++++++------------ .../server/core/DefaultServer.java | 8 +- .../server/deploy/BeanDescriptor.java | 10 +- src/main/resources/ebean-autotune-1.0.xsd | 2 +- .../autofetch/AutofetchEmbeddedTest.java | 8 +- 14 files changed, 181 insertions(+), 305 deletions(-) delete mode 100644 ebean-autotune-profiling.xml delete mode 100644 src/main/java/com/avaje/ebean/AdminAutofetch.java create mode 100644 src/main/java/com/avaje/ebean/AutoTune.java rename src/main/java/com/avaje/ebean/config/{AutofetchConfig.java => AutoTuneConfig.java} (60%) rename src/main/java/com/avaje/ebean/config/{AutofetchMode.java => AutoTuneMode.java} (96%) diff --git a/ebean-autotune-profiling.xml b/ebean-autotune-profiling.xml deleted file mode 100644 index 3911ea279..000000000 --- a/ebean-autotune-profiling.xml +++ /dev/null @@ -1,32 +0,0 @@ - - - - - - - com.avaje.tests.query.autotune.TestAutoTuneProfiling.findById(TestAutoTuneProfiling.java:45) - com.avaje.tests.query.autotune.TestAutoTuneProfiling.useOrderDate(TestAutoTuneProfiling.java:49) - com.avaje.tests.query.autotune.TestAutoTuneProfiling.execute(TestAutoTuneProfiling.java:35) - com.avaje.tests.query.autotune.TestAutoTuneProfiling.test(TestAutoTuneProfiling.java:26) - sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) - - - - - - - - - - com.avaje.tests.query.autotune.TestAutoTuneProfiling.findById(TestAutoTuneProfiling.java:45) - com.avaje.tests.query.autotune.TestAutoTuneProfiling.useOrderDateCustomerName(TestAutoTuneProfiling.java:55) - com.avaje.tests.query.autotune.TestAutoTuneProfiling.execute(TestAutoTuneProfiling.java:36) - com.avaje.tests.query.autotune.TestAutoTuneProfiling.test(TestAutoTuneProfiling.java:26) - sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) - - - - - diff --git a/ebean-autotune.xml b/ebean-autotune.xml index 73563dec3..53be9c98e 100644 --- a/ebean-autotune.xml +++ b/ebean-autotune.xml @@ -1,31 +1,37 @@ - - - com.avaje.tests.query.autotune.TestAutoTuneProfiling.findById(TestAutoTuneProfiling.java:45) - com.avaje.tests.query.autotune.TestAutoTuneProfiling.useLots(TestAutoTuneProfiling.java:62) - com.avaje.tests.query.autotune.TestAutoTuneProfiling.execute(TestAutoTuneProfiling.java:37) - com.avaje.tests.query.autotune.TestAutoTuneProfiling.test(TestAutoTuneProfiling.java:26) + + com.avaje.tests.query.autotune.TestAutoTuneProfiling.findById(TestAutoTuneProfiling.java:41) + com.avaje.tests.query.autotune.TestAutoTuneProfiling.useLots(TestAutoTuneProfiling.java:58) + com.avaje.tests.query.autotune.TestAutoTuneProfiling.execute(TestAutoTuneProfiling.java:33) + com.avaje.tests.query.autotune.TestAutoTuneProfiling.test(TestAutoTuneProfiling.java:22) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) - - com.avaje.tests.query.autotune.TestAutoTuneProfiling.findById(TestAutoTuneProfiling.java:45) - com.avaje.tests.query.autotune.TestAutoTuneProfiling.useOrderDate(TestAutoTuneProfiling.java:49) - com.avaje.tests.query.autotune.TestAutoTuneProfiling.execute(TestAutoTuneProfiling.java:35) - com.avaje.tests.query.autotune.TestAutoTuneProfiling.test(TestAutoTuneProfiling.java:26) + + com.avaje.tests.query.autotune.TestAutoTuneProfiling.findById(TestAutoTuneProfiling.java:41) + com.avaje.tests.query.autotune.TestAutoTuneProfiling.useOrderDateCustomerName(TestAutoTuneProfiling.java:51) + com.avaje.tests.query.autotune.TestAutoTuneProfiling.execute(TestAutoTuneProfiling.java:32) + com.avaje.tests.query.autotune.TestAutoTuneProfiling.test(TestAutoTuneProfiling.java:22) + sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) + + + + com.avaje.tests.query.autotune.TestAutoTuneProfiling.findById(TestAutoTuneProfiling.java:41) + com.avaje.tests.query.autotune.TestAutoTuneProfiling.useLotUntuned(TestAutoTuneProfiling.java:72) + com.avaje.tests.query.autotune.TestAutoTuneProfiling.execute(TestAutoTuneProfiling.java:34) + com.avaje.tests.query.autotune.TestAutoTuneProfiling.test(TestAutoTuneProfiling.java:22) + sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) + + + + com.avaje.tests.query.autotune.TestAutoTuneProfiling.findById(TestAutoTuneProfiling.java:41) + com.avaje.tests.query.autotune.TestAutoTuneProfiling.useOrderDate(TestAutoTuneProfiling.java:45) + com.avaje.tests.query.autotune.TestAutoTuneProfiling.execute(TestAutoTuneProfiling.java:31) + com.avaje.tests.query.autotune.TestAutoTuneProfiling.test(TestAutoTuneProfiling.java:22) sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) - - - - - - - - - diff --git a/src/main/java/com/avaje/ebean/AdminAutofetch.java b/src/main/java/com/avaje/ebean/AdminAutofetch.java deleted file mode 100644 index 49ab9287e..000000000 --- a/src/main/java/com/avaje/ebean/AdminAutofetch.java +++ /dev/null @@ -1,23 +0,0 @@ -package com.avaje.ebean; - -/** - * Administrative control of Autofetch during runtime. - */ -public interface AdminAutofetch { - - /** - * Fire a garbage collection (hint to the JVM). Assuming garbage collection - * fires this will gather the usage profiling information. - */ - void collectUsageViaGC(); - - /** - * This will take the current profiling information and update the "tuned - * query detail". - *

- * This is done periodically and can also be manually invoked. - *

- */ - void updateTunedQueryInfo(); - -} diff --git a/src/main/java/com/avaje/ebean/AutoTune.java b/src/main/java/com/avaje/ebean/AutoTune.java new file mode 100644 index 000000000..c5d17ac55 --- /dev/null +++ b/src/main/java/com/avaje/ebean/AutoTune.java @@ -0,0 +1,14 @@ +package com.avaje.ebean; + +/** + * Administrative control of AutoTune during runtime. + */ +public interface AutoTune { + + /** + * Fire a garbage collection (hint to the JVM). Assuming garbage collection + * fires this will gather remaining usage profiling information. + */ + void collectProfiling(); + +} diff --git a/src/main/java/com/avaje/ebean/config/AutofetchConfig.java b/src/main/java/com/avaje/ebean/config/AutoTuneConfig.java similarity index 60% rename from src/main/java/com/avaje/ebean/config/AutofetchConfig.java rename to src/main/java/com/avaje/ebean/config/AutoTuneConfig.java index 96aa821d5..e95fe2395 100644 --- a/src/main/java/com/avaje/ebean/config/AutofetchConfig.java +++ b/src/main/java/com/avaje/ebean/config/AutoTuneConfig.java @@ -1,11 +1,11 @@ package com.avaje.ebean.config; /** - * Defines the Autofetch behaviour for a EbeanServer. + * Defines the AutoTune behaviour for a EbeanServer. */ -public class AutofetchConfig { +public class AutoTuneConfig { - private AutofetchMode mode = AutofetchMode.DEFAULT_ONIFEMPTY; + private AutoTuneMode mode = AutoTuneMode.DEFAULT_ON; private boolean queryTuning; @@ -17,27 +17,25 @@ public class AutofetchConfig { private double profilingRate = 0.01; - private int profileUpdateFrequency = 60; - private int garbageCollectionWait = 100; - private boolean garbageCollectionOnShutdown; + private boolean skipCollectionOnShutdown; - public AutofetchConfig() { + public AutoTuneConfig() { } /** - * Return the mode used when autofetch has not been explicit defined on a + * Return the mode used when autoTune has not been explicit defined on a * query. */ - public AutofetchMode getMode() { + public AutoTuneMode getMode() { return mode; } /** - * Set the mode used when autofetch has not been explicit defined on a query. + * Set the mode used when autoTune has not been explicit defined on a query. */ - public void setMode(AutofetchMode mode) { + public void setMode(AutoTuneMode mode) { this.mode = mode; } @@ -74,6 +72,9 @@ public class AutofetchConfig { * If this is false then the version property will be added when profiling * detects that the bean is possibly going to be modified. *

+ *

+ * Generally this is not expected to be turned on. + *

*/ public void setQueryTuningAddVersion(boolean queryTuningAddVersion) { this.queryTuningAddVersion = queryTuningAddVersion; @@ -128,29 +129,13 @@ public class AutofetchConfig { this.profilingRate = profilingRate; } - /** - * Return the frequency in seconds to update the autofetch tuned queries from - * the profiled information. - */ - public int getProfileUpdateFrequency() { - return profileUpdateFrequency; - } - - /** - * Set the frequency in seconds to update the autofetch tuned queries from the - * profiled information. - */ - public void setProfileUpdateFrequency(int profileUpdateFrequency) { - this.profileUpdateFrequency = profileUpdateFrequency; - } - /** * Return the time in millis to wait after a system gc to collect profiling * information. *

* The profiling information is collected on object finalise. As such we * generally don't want to trigger GC (let the JVM do its thing) but on - * shutdown the autofetch manager will trigger System.gc() and then wait + * shutdown the autoTune manager will trigger System.gc() and then wait * (default 100 millis) to hopefully collect profiling information - * especially for short run unit tests. *

@@ -160,33 +145,24 @@ public class AutofetchConfig { } /** - * Set the time in millis to wait after a System.gc() to collect profiling - * information. + * Set the time in millis to wait after a System.gc() to collect profiling information. */ public void setGarbageCollectionWait(int garbageCollectionWait) { this.garbageCollectionWait = garbageCollectionWait; } - /** - * Return true if GC should be trigger on shutdown. - *

- * Autofetch profiling information is collected as part of garbage collection. - *

+ * Return true if profiling collection should be skipped on shutdown. */ - public boolean isGarbageCollectionOnShutdown() { - return garbageCollectionOnShutdown; + public boolean isSkipCollectionOnShutdown() { + return skipCollectionOnShutdown; } /** - * Set to true if you want GC to trigger on shutdown. - *

- * This would be done if you want to try and collect Autofetch profiling information - * on shutdown. - *

+ * Set to true if profiling collection should be skipped on shutdown. */ - public void setGarbageCollectionOnShutdown(boolean garbageCollectionOnShutdown) { - this.garbageCollectionOnShutdown = garbageCollectionOnShutdown; + public void setSkipCollectionOnShutdown(boolean skipCollectionOnShutdown) { + this.skipCollectionOnShutdown = skipCollectionOnShutdown; } /** @@ -194,16 +170,14 @@ public class AutofetchConfig { */ public void loadSettings(PropertiesWrapper p) { - queryTuning = p.getBoolean("autotune.querytuning", queryTuning); - queryTuningAddVersion = p.getBoolean("autotune.queryTuningAddVersion", queryTuningAddVersion); - garbageCollectionOnShutdown = p.getBoolean("autotune.garbageCollectionOnShutdown", garbageCollectionOnShutdown); + queryTuning = p.getBoolean("autoTune.queryTuning", queryTuning); + queryTuningAddVersion = p.getBoolean("autoTune.queryTuningAddVersion", queryTuningAddVersion); + skipCollectionOnShutdown = p.getBoolean("autoTune.skipCollectionOnShutdown", skipCollectionOnShutdown); - profiling = p.getBoolean("autotune.profiling", profiling); - mode = p.getEnum(AutofetchMode.class, "autotune.implicitmode", mode); + mode = p.getEnum(AutoTuneMode.class, "autoTune.mode", mode); - profilingBase = p.getInt("autotune.profiling.base", profilingBase); - - profilingRate = p.getDouble("autotune.profiling.rate", profilingRate); - profileUpdateFrequency = p.getInt("autotune.profiling.updatefrequency", profileUpdateFrequency); + profiling = p.getBoolean("autoTune.profiling", profiling); + profilingBase = p.getInt("autoTune.profilingBase", profilingBase); + profilingRate = p.getDouble("autoTune.profilingRate", profilingRate); } } diff --git a/src/main/java/com/avaje/ebean/config/AutofetchMode.java b/src/main/java/com/avaje/ebean/config/AutoTuneMode.java similarity index 96% rename from src/main/java/com/avaje/ebean/config/AutofetchMode.java rename to src/main/java/com/avaje/ebean/config/AutoTuneMode.java index 86dcf95db..b255421ff 100644 --- a/src/main/java/com/avaje/ebean/config/AutofetchMode.java +++ b/src/main/java/com/avaje/ebean/config/AutoTuneMode.java @@ -11,7 +11,7 @@ import com.avaje.ebean.Query; * query. *

*/ -public enum AutofetchMode { +public enum AutoTuneMode { /** * Don't implicitly use Autofetch. Must explicitly turn it on. diff --git a/src/main/java/com/avaje/ebeaninternal/server/autofetch/AutoTuneCollection.java b/src/main/java/com/avaje/ebeaninternal/server/autofetch/AutoTuneCollection.java index eb62059b9..98285d9d3 100644 --- a/src/main/java/com/avaje/ebeaninternal/server/autofetch/AutoTuneCollection.java +++ b/src/main/java/com/avaje/ebeaninternal/server/autofetch/AutoTuneCollection.java @@ -13,8 +13,8 @@ public class AutoTuneCollection { List entries = new ArrayList(); - public Entry add(ObjectGraphOrigin origin, OrmQueryDetail detail) { - Entry entry = new Entry(origin, detail); + public Entry add(ObjectGraphOrigin origin, OrmQueryDetail detail, String sourceQuery) { + Entry entry = new Entry(origin, detail, sourceQuery); entries.add(entry); return entry; } @@ -38,14 +38,20 @@ public class AutoTuneCollection { */ private final OrmQueryDetail detail; + /** + * The original/existing query detail. + */ + private final String originalQuery; + /** * Summary execution statistics for queries related to this origin point. */ private final List queries = new ArrayList(); - public Entry(ObjectGraphOrigin origin, OrmQueryDetail detail) { + public Entry(ObjectGraphOrigin origin, OrmQueryDetail detail, String originalQuery) { this.origin = origin; this.detail = detail; + this.originalQuery = originalQuery; } public void addQuery(EntryQuery entryQuery) { @@ -60,6 +66,10 @@ public class AutoTuneCollection { return detail; } + public String getOriginalQuery() { + return originalQuery; + } + public List getQueries() { return queries; } diff --git a/src/main/java/com/avaje/ebeaninternal/server/autofetch/AutoTuneService.java b/src/main/java/com/avaje/ebeaninternal/server/autofetch/AutoTuneService.java index 291cfe9f3..2e8258697 100644 --- a/src/main/java/com/avaje/ebeaninternal/server/autofetch/AutoTuneService.java +++ b/src/main/java/com/avaje/ebeaninternal/server/autofetch/AutoTuneService.java @@ -1,8 +1,6 @@ package com.avaje.ebeaninternal.server.autofetch; -import com.avaje.ebean.AdminAutofetch; -import com.avaje.ebean.config.ServerConfig; -import com.avaje.ebeaninternal.api.SpiEbeanServer; +import com.avaje.ebean.AutoTune; import com.avaje.ebeaninternal.api.SpiQuery; /** @@ -17,7 +15,7 @@ import com.avaje.ebeaninternal.api.SpiQuery; * etc). These are applied to the query when tuneQuery() is called. *

*/ -public interface AutoTuneService extends AdminAutofetch { +public interface AutoTuneService extends AutoTune { /** * Load the query tuning information. @@ -41,16 +39,7 @@ public interface AutoTuneService extends AdminAutofetch { * Fire a garbage collection (hint to the JVM). Assuming garbage collection * fires this will gather the usage profiling information. */ - void collectUsageViaGC(); - - /** - * This will take the current profiling information and update the "tuned - * query detail". - *

- * This is done periodically and can also be manually invoked. - *

- */ - void updateTunedQueryInfo(); + void collectProfiling(); /** * On shutdown fire garbage collection and collect statistics. Note that diff --git a/src/main/java/com/avaje/ebeaninternal/server/autofetch/service/BaseQueryTuner.java b/src/main/java/com/avaje/ebeaninternal/server/autofetch/service/BaseQueryTuner.java index 9536197d5..f3614d4f8 100644 --- a/src/main/java/com/avaje/ebeaninternal/server/autofetch/service/BaseQueryTuner.java +++ b/src/main/java/com/avaje/ebeaninternal/server/autofetch/service/BaseQueryTuner.java @@ -2,8 +2,8 @@ package com.avaje.ebeaninternal.server.autofetch.service; import com.avaje.ebean.bean.CallStack; import com.avaje.ebean.bean.ObjectGraphNode; -import com.avaje.ebean.config.AutofetchConfig; -import com.avaje.ebean.config.AutofetchMode; +import com.avaje.ebean.config.AutoTuneConfig; +import com.avaje.ebean.config.AutoTuneMode; import com.avaje.ebeaninternal.api.SpiEbeanServer; import com.avaje.ebeaninternal.api.SpiQuery; import com.avaje.ebeaninternal.server.autofetch.ProfilingListener; @@ -22,7 +22,7 @@ public class BaseQueryTuner { private boolean profiling; - private final AutofetchMode mode; + private final AutoTuneMode mode; /** * Map of the tuned query details per profile query point. @@ -34,7 +34,9 @@ public class BaseQueryTuner { private final ProfilingListener profilingListener; - public BaseQueryTuner(AutofetchConfig config, SpiEbeanServer server, ProfilingListener profilingListener) { + boolean fullProfiling = true; + + public BaseQueryTuner(AutoTuneConfig config, SpiEbeanServer server, ProfilingListener profilingListener) { this.server = server; this.profilingListener = profilingListener; this.mode = config.getMode(); @@ -58,8 +60,6 @@ public class BaseQueryTuner { return (info == null) ? null : info.getTunedDetail(); } - boolean fullProfiling = true; - /** * Auto tune the query and enable profiling. */ @@ -90,7 +90,7 @@ public class BaseQueryTuner { ObjectGraphNode origin = query.setOrigin(stack); if (profiling) { - if (profilingListener.isProfileRequest(origin)) { + if (profilingListener.isProfileRequest(origin, query)) { // collect more profiling based on profiling rate etc query.setProfilingListener(profilingListener); } @@ -108,7 +108,7 @@ public class BaseQueryTuner { // create a query point to identify the query ObjectGraphNode origin = query.setOrigin(stack); - if (profilingListener.isProfileRequest(origin)) { + if (profilingListener.isProfileRequest(origin, query)) { // collect more profiling based on profiling rate etc query.setProfilingListener(profilingListener); } diff --git a/src/main/java/com/avaje/ebeaninternal/server/autofetch/service/DefaultAutoTuneService.java b/src/main/java/com/avaje/ebeaninternal/server/autofetch/service/DefaultAutoTuneService.java index 3a22f1d57..445b59715 100644 --- a/src/main/java/com/avaje/ebeaninternal/server/autofetch/service/DefaultAutoTuneService.java +++ b/src/main/java/com/avaje/ebeaninternal/server/autofetch/service/DefaultAutoTuneService.java @@ -1,7 +1,7 @@ package com.avaje.ebeaninternal.server.autofetch.service; import com.avaje.ebean.bean.ObjectGraphOrigin; -import com.avaje.ebean.config.AutofetchConfig; +import com.avaje.ebean.config.AutoTuneConfig; import com.avaje.ebean.config.ServerConfig; import com.avaje.ebeaninternal.api.SpiEbeanServer; import com.avaje.ebeaninternal.api.SpiQuery; @@ -18,6 +18,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.File; +import java.text.SimpleDateFormat; +import java.util.Date; import java.util.List; /** @@ -29,87 +31,107 @@ public class DefaultAutoTuneService implements AutoTuneService { private final long defaultGarbageCollectionWait; - private final boolean garbageCollectionOnShutdown; + private final boolean skipCollectionOnShutdown; private final BaseQueryTuner queryTuner; private final ProfileManager profileManager; + private final boolean profiling; + public DefaultAutoTuneService(SpiEbeanServer server, ServerConfig serverConfig) { - AutofetchConfig config = serverConfig.getAutofetchConfig(); + AutoTuneConfig config = serverConfig.getAutoTuneConfig(); + this.profiling = config.isProfiling(); this.profileManager = new ProfileManager(config, server); this.queryTuner = new BaseQueryTuner(config, server, profileManager); - this.garbageCollectionOnShutdown = config.isGarbageCollectionOnShutdown(); + this.skipCollectionOnShutdown = config.isSkipCollectionOnShutdown(); this.defaultGarbageCollectionWait = (long) config.getGarbageCollectionWait(); } /** * Load the query tuning information from it's data store. */ + @Override public void startup() { File file = new File("ebean-autotune.xml"); AutoTuneXmlReader reader = new AutoTuneXmlReader(); Autotune profiling = reader.read(file); - List originList = profiling.getOrigin(); - for (Origin origin : originList) { - String key = origin.getKey(); - String detail = origin.getDetail(); - OrmQueryDetailParser parser = new OrmQueryDetailParser(detail); - OrmQueryDetail fetchDetail = parser.parse(); - TunedQueryInfo tunedQueryInfo = new TunedQueryInfo(fetchDetail); - queryTuner.load(key, tunedQueryInfo); + for (Origin origin : profiling.getOrigin()) { + queryTuner.load(origin.getKey(), createTunedQueryInfo(origin)); } } - private void saveProfiling() { + @NotNull + private TunedQueryInfo createTunedQueryInfo(Origin origin) { + OrmQueryDetail detail = new OrmQueryDetailParser(origin.getDetail()).parse(); + return new TunedQueryInfo(detail); + } + + private void saveProfiling(boolean reset) { Autotune document = new Autotune(); - AutoTuneCollection autoTuneCollection = profileManager.profilingCollection(false); + AutoTuneCollection autoTuneCollection = profileManager.profilingCollection(reset); List entries = autoTuneCollection.getEntries(); for (AutoTuneCollection.Entry entry : entries) { - ObjectGraphOrigin point = entry.getOrigin(); - OrmQueryDetail profileDetail = entry.getDetail(); - - OrmQueryDetail tuneDetail = queryTuner.get(point.getKey()); - if (tuneDetail == null) { - ProfileNew profileNew = document.getProfileNew(); - if (profileNew == null) { - profileNew = new ProfileNew(); - document.setProfileNew(profileNew); - } - profileNew.getOrigin().add( createOrigin(entry, point)); - - } else if (!tuneDetail.isAutoTuneEqual(profileDetail)) { - Origin origin1 = createOrigin(entry, point); - origin1.setTuneDetail(tuneDetail.toString()); - ProfileDiff diff = document.getProfileDiff(); - if (diff == null) { - diff = new ProfileDiff(); - document.setProfileDiff(diff); - } - diff.getOrigin().add(origin1); - } + saveProfilingEntry(document, entry); } - File file = new File("ebean-autotune-profiling.xml"); + SimpleDateFormat df = new SimpleDateFormat("yyyyMMdd-HHmmss"); + String now = df.format(new Date()); + + File file = new File("ebean-autotune-profiling"+"-"+now+".xml"); AutoTuneXmlWriter writer = new AutoTuneXmlWriter(); writer.write(document, file); } + private void saveProfilingEntry(Autotune document, AutoTuneCollection.Entry entry) { + + ObjectGraphOrigin point = entry.getOrigin(); + OrmQueryDetail profileDetail = entry.getDetail(); + + // compare with the existing query tuning entry + OrmQueryDetail tuneDetail = queryTuner.get(point.getKey()); + if (tuneDetail == null) { + // New entry + ProfileNew profileNew = document.getProfileNew(); + if (profileNew == null) { + profileNew = new ProfileNew(); + document.setProfileNew(profileNew); + } + Origin origin = createOrigin(entry, point); + origin.setOriginal(entry.getOriginalQuery()); + profileNew.getOrigin().add(origin); + + } else if (!tuneDetail.isAutoTuneEqual(profileDetail)) { + // Diff entry + Origin origin = createOrigin(entry, point); + origin.setOriginal(tuneDetail.toString()); + ProfileDiff diff = document.getProfileDiff(); + if (diff == null) { + diff = new ProfileDiff(); + document.setProfileDiff(diff); + } + diff.getOrigin().add(origin); + } + } + + /** + * Create the XML Origin bean for the given entry and ObjectGraphOrigin. + */ @NotNull private Origin createOrigin(AutoTuneCollection.Entry entry, ObjectGraphOrigin point) { - Origin origin1 = new Origin(); - origin1.setKey(point.getKey()); - origin1.setBeanType(point.getBeanType()); - origin1.setDetail(entry.getDetail().toString()); - origin1.setCallStack(point.getCallStack().description("\n")); - return origin1; + Origin origin = new Origin(); + origin.setKey(point.getKey()); + origin.setBeanType(point.getBeanType()); + origin.setDetail(entry.getDetail().toString()); + origin.setCallStack(point.getCallStack().description("\n")); + return origin; } /** @@ -120,10 +142,11 @@ public class DefaultAutoTuneService implements AutoTuneService { * collection may not otherwise occur at all. *

*/ + @Override public void shutdown() { - if (garbageCollectionOnShutdown) { - collectUsageViaGC(-1); - saveProfiling(); + if (profiling && !skipCollectionOnShutdown) { + collectProfiling(-1); + saveProfiling(false); } } @@ -143,11 +166,12 @@ public class DefaultAutoTuneService implements AutoTuneService { * defaults to 100 milliseconds. *

*/ - public void collectUsageViaGC() { - collectUsageViaGC(-1); + @Override + public void collectProfiling() { + collectProfiling(-1); } - public void collectUsageViaGC(long waitMillis) { + public void collectProfiling(long waitMillis) { System.gc(); try { if (waitMillis < 0) { @@ -157,99 +181,13 @@ public class DefaultAutoTuneService implements AutoTuneService { } catch (InterruptedException e) { logger.warn("Error while sleeping after System.gc() request.", e); } - updateTunedQueryInfo(); } - /** - * Update the tuned fetch plans from the current usage information. - */ - public void updateTunedQueryInfo() { - -// if (!profiling) { -// // we are not collecting any profiling information at -// // the moment so don't try updating the tuned query plans. -// return "Not profiling"; -// } -// -// synchronized (statisticsMonitor) { -// -// Counters counters = new Counters(); -// -// for (ProfileOrigin queryPointStatistics : statisticsMap.values()) { -// if (!queryPointStatistics.hasUsage()) { -// // no usage statistics collected yet... -// counters.incrementNoUsage(); -// } else { -// updateTunedQueryFromUsage(counters, queryPointStatistics); -// } -// } -// -// String summaryInfo = counters.toString(); -// -// if (counters.isInteresting()) { -// // only log it if its interesting -// logging.logSummary(summaryInfo); -// } -// -// return summaryInfo; -// } - } - - -// -// private void updateTunedQueryFromUsage(Counters counters, ProfileOrigin statistics) { -// -// ObjectGraphOrigin queryPoint = statistics.getOrigin(); -// String beanType = queryPoint.getBeanType(); -// -// try { -// Class beanClass = ClassUtil.forName(beanType, this.getClass()); -// BeanDescriptor beanDescriptor = server.getBeanDescriptor(beanClass); -// if (beanDescriptor != null) { -// -// // Determine the fetch plan from the latest statistics. -// // Use this to compare with current "tuned fetch plan". -// OrmQueryDetail newFetchDetail = statistics.buildTunedFetch(beanDescriptor); -// -// // get the current tuned fetch info... -// TunedQueryInfo currentFetch = tunedQueryInfoMap.get(queryPoint.getKey()); -// -// if (currentFetch == null) { -// // its a new fetch plan, add it. -// counters.incrementNew(); -// -// currentFetch = statistics.createTunedFetch(newFetchDetail); -// logging.logNew(currentFetch); -// tunedQueryInfoMap.put(queryPoint.getKey(), currentFetch); -// -// } else if (!currentFetch.isSame(newFetchDetail)) { -// // the fetch plan has changed, update it. -// counters.incrementModified(); -// -// logging.logChanged(currentFetch, newFetchDetail); -// currentFetch.setTunedDetail(newFetchDetail); -// -// } else { -// // the fetch plan has not changed... -// counters.incrementUnchanged(); -// } -// -// currentFetch.setProfileCount(statistics.getCounter()); -// } -// -// } catch (ClassNotFoundException e) { -// // expected after renaming/moving an entity bean -// String msg = e.toString() + " updating autoFetch tuned query for " + beanType -// + ". It isLikely this bean has been renamed or moved"; -// logging.logInfo(msg, null); -// statisticsMap.remove(statistics.getOrigin().getKey()); -// } -// } -// /** * Auto tune the query and enable profiling. */ + @Override public boolean tuneQuery(SpiQuery query) { return queryTuner.tuneQuery(query); } diff --git a/src/main/java/com/avaje/ebeaninternal/server/core/DefaultServer.java b/src/main/java/com/avaje/ebeaninternal/server/core/DefaultServer.java index cdab0e94f..8bfdb6ea0 100644 --- a/src/main/java/com/avaje/ebeaninternal/server/core/DefaultServer.java +++ b/src/main/java/com/avaje/ebeaninternal/server/core/DefaultServer.java @@ -330,7 +330,7 @@ public final class DefaultServer implements SpiServer, SpiEbeanServer { return ddlGenerator; } - public AdminAutofetch getAdminAutofetch() { + public AutoTune getAutoTune() { return autoTuneService; } @@ -1039,7 +1039,7 @@ public final class DefaultServer implements SpiServer, SpiEbeanServer { public SpiOrmQueryRequest createQueryRequest(BeanDescriptor desc, SpiQuery query, Transaction t) { - if (desc.isAutoFetchTunable() && !query.isSqlSelect() && !autoTuneService.tuneQuery(query)) { + if (desc.isAutoTunable() && !query.isSqlSelect() && !autoTuneService.tuneQuery(query)) { // use deployment FetchType.LAZY/EAGER annotations // to define the 'default' select clause query.setDefaultSelectClause(); @@ -1047,13 +1047,13 @@ public final class DefaultServer implements SpiServer, SpiEbeanServer { if (query.selectAllForLazyLoadProperty()) { // we need to select all properties to ensure the lazy load property - // was included (was not included by default or via autofetch). + // was included (was not included by default or via autoTune). if (logger.isDebugEnabled()) { logger.debug("Using selectAllForLazyLoadProperty"); } } - // if determine cost and no origin for Autofetch + // if determine cost and no origin for AutoTune if (query.getParentNode() == null) { query.setOrigin(createCallStack()); } diff --git a/src/main/java/com/avaje/ebeaninternal/server/deploy/BeanDescriptor.java b/src/main/java/com/avaje/ebeaninternal/server/deploy/BeanDescriptor.java index e73218217..049063b82 100644 --- a/src/main/java/com/avaje/ebeaninternal/server/deploy/BeanDescriptor.java +++ b/src/main/java/com/avaje/ebeaninternal/server/deploy/BeanDescriptor.java @@ -126,7 +126,7 @@ public class BeanDescriptor implements MetaBeanInfo, SpiBeanType { */ private final String selectLastInsertedId; - private final boolean autoFetchTunable; + private final boolean autoTunable; /** * The concurrency mode for beans of this type. @@ -366,7 +366,7 @@ public class BeanDescriptor implements MetaBeanInfo, SpiBeanType { this.baseTable = InternString.intern(deploy.getBaseTable()); this.baseTableAsOf = deploy.getBaseTableAsOf(); this.baseTableVersionsBetween = deploy.getBaseTableVersionsBetween(); - this.autoFetchTunable = EntityType.ORM.equals(entityType) && (beanFinder == null); + this.autoTunable = EntityType.ORM.equals(entityType) && (beanFinder == null); // helper object used to derive lists of properties DeployBeanPropertyLists listHelper = new DeployBeanPropertyLists(owner, this, deploy); @@ -1603,10 +1603,10 @@ public class BeanDescriptor implements MetaBeanInfo, SpiBeanType { } /** - * Return true if queries for beans of this type are autoFetch tunable. + * Return true if queries for beans of this type are auto tunable. */ - public boolean isAutoFetchTunable() { - return autoFetchTunable; + public boolean isAutoTunable() { + return autoTunable; } /** diff --git a/src/main/resources/ebean-autotune-1.0.xsd b/src/main/resources/ebean-autotune-1.0.xsd index 42a1ce081..fd4fc3ad2 100644 --- a/src/main/resources/ebean-autotune-1.0.xsd +++ b/src/main/resources/ebean-autotune-1.0.xsd @@ -23,7 +23,7 @@ - + diff --git a/src/test/java/com/avaje/tests/autofetch/AutofetchEmbeddedTest.java b/src/test/java/com/avaje/tests/autofetch/AutofetchEmbeddedTest.java index ba59ae6cb..c6de0cd29 100644 --- a/src/test/java/com/avaje/tests/autofetch/AutofetchEmbeddedTest.java +++ b/src/test/java/com/avaje/tests/autofetch/AutofetchEmbeddedTest.java @@ -37,9 +37,9 @@ public class AutofetchEmbeddedTest extends BaseTestCase { @Test public void testEmbeddedBeanQueryTuning() { -// Ebean.getServer(null).getAdminAutofetch().setProfiling(true); -// Ebean.getServer(null).getAdminAutofetch().setQueryTuning(true); -// Ebean.getServer(null).getAdminAutofetch().setProfilingBase(1); +// Ebean.getServer(null).getAutoTune().setProfiling(true); +// Ebean.getServer(null).getAutoTune().setQueryTuning(true); +// Ebean.getServer(null).getAutoTune().setProfilingBase(1); // // EMain testBean = new EMain(); // testBean.setName("test"); @@ -58,7 +58,7 @@ public class AutofetchEmbeddedTest extends BaseTestCase { // Ebean.commitTransaction(); // } finally { // Ebean.endTransaction(); -// logger.debug(Ebean.getServer(null).getAdminAutofetch().collectUsageViaGC()); +// logger.debug(Ebean.getServer(null).getAutoTune().collectProfiling()); // } // } } From 00964777572b79b958c037e6131cf633eada9be9 Mon Sep 17 00:00:00 2001 From: Robin Bygrave Date: Thu, 10 Sep 2015 09:16:39 +1200 Subject: [PATCH 7/9] AutoTune update - repackage tests and sort profiling xml entries --- .../service/DefaultAutoTuneService.java | 40 ++++++++++++++++++- .../autofetch/TunedQueryInfoTest.java | 2 +- .../TunedQueryWithNullFetchedBeanTest.java | 2 +- .../transaction/TestAutoCommitDataSource.java | 2 +- .../transaction/TestBatchPersistCascade.java | 2 +- src/test/resources/ebean.properties | 6 --- 6 files changed, 42 insertions(+), 12 deletions(-) rename src/test/java/com/avaje/{ebeaninternal/server => tests}/autofetch/TunedQueryInfoTest.java (99%) rename src/test/java/com/avaje/{ebeaninternal/server => tests}/autofetch/TunedQueryWithNullFetchedBeanTest.java (97%) rename src/test/java/com/avaje/{ebeaninternal/server => tests}/transaction/TestAutoCommitDataSource.java (97%) rename src/test/java/com/avaje/{ebeaninternal/server => tests}/transaction/TestBatchPersistCascade.java (98%) diff --git a/src/main/java/com/avaje/ebeaninternal/server/autofetch/service/DefaultAutoTuneService.java b/src/main/java/com/avaje/ebeaninternal/server/autofetch/service/DefaultAutoTuneService.java index 445b59715..d2d7242d5 100644 --- a/src/main/java/com/avaje/ebeaninternal/server/autofetch/service/DefaultAutoTuneService.java +++ b/src/main/java/com/avaje/ebeaninternal/server/autofetch/service/DefaultAutoTuneService.java @@ -19,6 +19,8 @@ import org.slf4j.LoggerFactory; import java.io.File; import java.text.SimpleDateFormat; +import java.util.Collections; +import java.util.Comparator; import java.util.Date; import java.util.List; @@ -39,6 +41,8 @@ public class DefaultAutoTuneService implements AutoTuneService { private final boolean profiling; + private final String serverName; + public DefaultAutoTuneService(SpiEbeanServer server, ServerConfig serverConfig) { AutoTuneConfig config = serverConfig.getAutoTuneConfig(); @@ -46,7 +50,7 @@ public class DefaultAutoTuneService implements AutoTuneService { this.profiling = config.isProfiling(); this.profileManager = new ProfileManager(config, server); this.queryTuner = new BaseQueryTuner(config, server, profileManager); - + this.serverName = server.getName(); this.skipCollectionOnShutdown = config.isSkipCollectionOnShutdown(); this.defaultGarbageCollectionWait = (long) config.getGarbageCollectionWait(); } @@ -78,16 +82,48 @@ public class DefaultAutoTuneService implements AutoTuneService { AutoTuneCollection autoTuneCollection = profileManager.profilingCollection(reset); List entries = autoTuneCollection.getEntries(); + for (AutoTuneCollection.Entry entry : entries) { saveProfilingEntry(document, entry); } + sortDocument(document); + SimpleDateFormat df = new SimpleDateFormat("yyyyMMdd-HHmmss"); String now = df.format(new Date()); - File file = new File("ebean-autotune-profiling"+"-"+now+".xml"); + File file = new File("ebean-profiling"+"-"+serverName+"-"+now+".xml"); AutoTuneXmlWriter writer = new AutoTuneXmlWriter(); writer.write(document, file); + + logger.debug("profiling entries:{}", entries.size()); + } + + /** + * Set the diff and new entries by bean type followed by key. + */ + private void sortDocument(Autotune document) { + + ProfileDiff profileDiff = document.getProfileDiff(); + if (profileDiff != null) { + Collections.sort(profileDiff.getOrigin(), new OriginNameKeySort()); + } + ProfileNew profileNew = document.getProfileNew(); + if (profileNew != null) { + Collections.sort(profileNew.getOrigin(), new OriginNameKeySort()); + } + } + + class OriginNameKeySort implements Comparator { + + @Override + public int compare(Origin o1, Origin o2) { + int comp = o1.getBeanType().compareTo(o2.getBeanType()); + if (comp == 0) { + comp = o1.getKey().compareTo(o2.getKey()); + } + return comp; + } } private void saveProfilingEntry(Autotune document, AutoTuneCollection.Entry entry) { diff --git a/src/test/java/com/avaje/ebeaninternal/server/autofetch/TunedQueryInfoTest.java b/src/test/java/com/avaje/tests/autofetch/TunedQueryInfoTest.java similarity index 99% rename from src/test/java/com/avaje/ebeaninternal/server/autofetch/TunedQueryInfoTest.java rename to src/test/java/com/avaje/tests/autofetch/TunedQueryInfoTest.java index a27881bc5..5c4ff601f 100644 --- a/src/test/java/com/avaje/ebeaninternal/server/autofetch/TunedQueryInfoTest.java +++ b/src/test/java/com/avaje/tests/autofetch/TunedQueryInfoTest.java @@ -1,4 +1,4 @@ -package com.avaje.ebeaninternal.server.autofetch; +package com.avaje.tests.autofetch; import java.util.List; import java.util.Set; diff --git a/src/test/java/com/avaje/ebeaninternal/server/autofetch/TunedQueryWithNullFetchedBeanTest.java b/src/test/java/com/avaje/tests/autofetch/TunedQueryWithNullFetchedBeanTest.java similarity index 97% rename from src/test/java/com/avaje/ebeaninternal/server/autofetch/TunedQueryWithNullFetchedBeanTest.java rename to src/test/java/com/avaje/tests/autofetch/TunedQueryWithNullFetchedBeanTest.java index 2e304b56b..501614941 100644 --- a/src/test/java/com/avaje/ebeaninternal/server/autofetch/TunedQueryWithNullFetchedBeanTest.java +++ b/src/test/java/com/avaje/tests/autofetch/TunedQueryWithNullFetchedBeanTest.java @@ -1,4 +1,4 @@ -package com.avaje.ebeaninternal.server.autofetch; +package com.avaje.tests.autofetch; import java.util.List; import java.util.Set; diff --git a/src/test/java/com/avaje/ebeaninternal/server/transaction/TestAutoCommitDataSource.java b/src/test/java/com/avaje/tests/transaction/TestAutoCommitDataSource.java similarity index 97% rename from src/test/java/com/avaje/ebeaninternal/server/transaction/TestAutoCommitDataSource.java rename to src/test/java/com/avaje/tests/transaction/TestAutoCommitDataSource.java index b24b37316..75ec8d753 100644 --- a/src/test/java/com/avaje/ebeaninternal/server/transaction/TestAutoCommitDataSource.java +++ b/src/test/java/com/avaje/tests/transaction/TestAutoCommitDataSource.java @@ -1,4 +1,4 @@ -package com.avaje.ebeaninternal.server.transaction; +package com.avaje.tests.transaction; import com.avaje.ebean.BaseTestCase; import com.avaje.ebean.EbeanServer; diff --git a/src/test/java/com/avaje/ebeaninternal/server/transaction/TestBatchPersistCascade.java b/src/test/java/com/avaje/tests/transaction/TestBatchPersistCascade.java similarity index 98% rename from src/test/java/com/avaje/ebeaninternal/server/transaction/TestBatchPersistCascade.java rename to src/test/java/com/avaje/tests/transaction/TestBatchPersistCascade.java index 9e0cc9608..db66fb72f 100644 --- a/src/test/java/com/avaje/ebeaninternal/server/transaction/TestBatchPersistCascade.java +++ b/src/test/java/com/avaje/tests/transaction/TestBatchPersistCascade.java @@ -1,4 +1,4 @@ -package com.avaje.ebeaninternal.server.transaction; +package com.avaje.tests.transaction; import com.avaje.ebean.BaseTestCase; import com.avaje.ebean.Ebean; diff --git a/src/test/resources/ebean.properties b/src/test/resources/ebean.properties index 228b08aa6..12018186d 100644 --- a/src/test/resources/ebean.properties +++ b/src/test/resources/ebean.properties @@ -13,12 +13,6 @@ ebean.encryptKeyManager=com.avaje.tests.basic.encrypt.BasicEncyptKeyManager ebean.autotune.querytuning=true ebean.autotune.profiling=true -#ebean.autoTune.implicitmode=default_off -#ebean.autotune.implicitmode=default_onifempty -#ebean.autofetch.profiling.min=1 -#ebean.autofetch.profiling.base=10 -##ebean.autofetch.profiling.rate=0.05 -#ebean.autofetch.garbageCollectionOnShutdown=true ebean.ddl.generate=true From 373598ab26c21812d56c7d17ec4d8b38ce3ab329 Mon Sep 17 00:00:00 2001 From: Robin Bygrave Date: Thu, 10 Sep 2015 10:02:17 +1200 Subject: [PATCH 8/9] AutoTune update - config properties for queryTuningFile and profilingFile + logging adjustment --- .../avaje/ebean/config/AutoTuneConfig.java | 35 ++++++++++ .../service/DefaultAutoTuneService.java | 66 ++++++++++++++----- src/test/resources/ebean.properties | 3 + 3 files changed, 88 insertions(+), 16 deletions(-) diff --git a/src/main/java/com/avaje/ebean/config/AutoTuneConfig.java b/src/main/java/com/avaje/ebean/config/AutoTuneConfig.java index e95fe2395..6eab25e8c 100644 --- a/src/main/java/com/avaje/ebean/config/AutoTuneConfig.java +++ b/src/main/java/com/avaje/ebean/config/AutoTuneConfig.java @@ -7,12 +7,16 @@ public class AutoTuneConfig { private AutoTuneMode mode = AutoTuneMode.DEFAULT_ON; + private String queryTuningFile = "ebean-autotune.xml"; + private boolean queryTuning; private boolean queryTuningAddVersion; private boolean profiling; + private String profilingFile = "ebean-profiling"; + private int profilingBase = 5; private double profilingRate = 0.01; @@ -24,6 +28,34 @@ public class AutoTuneConfig { public AutoTuneConfig() { } + /** + * Return the name of the file that holds the query tuning information. + */ + public String getQueryTuningFile() { + return queryTuningFile; + } + + /** + * Set the name of the file that holds the query tuning information. + */ + public void setQueryTuningFile(String queryTuningFile) { + this.queryTuningFile = queryTuningFile; + } + + /** + * Return the name of the file that profiling information is written to. + */ + public String getProfilingFile() { + return profilingFile; + } + + /** + * Set the name of the file that profiling information is written to. + */ + public void setProfilingFile(String profilingFile) { + this.profilingFile = profilingFile; + } + /** * Return the mode used when autoTune has not been explicit defined on a * query. @@ -172,6 +204,8 @@ public class AutoTuneConfig { queryTuning = p.getBoolean("autoTune.queryTuning", queryTuning); queryTuningAddVersion = p.getBoolean("autoTune.queryTuningAddVersion", queryTuningAddVersion); + queryTuningFile = p.get("autoTune.queryTuningFile", queryTuningFile); + skipCollectionOnShutdown = p.getBoolean("autoTune.skipCollectionOnShutdown", skipCollectionOnShutdown); mode = p.getEnum(AutoTuneMode.class, "autoTune.mode", mode); @@ -179,5 +213,6 @@ public class AutoTuneConfig { profiling = p.getBoolean("autoTune.profiling", profiling); profilingBase = p.getInt("autoTune.profilingBase", profilingBase); profilingRate = p.getDouble("autoTune.profilingRate", profilingRate); + profilingFile = p.get("autoTune.profilingFile", profilingFile); } } diff --git a/src/main/java/com/avaje/ebeaninternal/server/autofetch/service/DefaultAutoTuneService.java b/src/main/java/com/avaje/ebeaninternal/server/autofetch/service/DefaultAutoTuneService.java index d2d7242d5..12600ad6d 100644 --- a/src/main/java/com/avaje/ebeaninternal/server/autofetch/service/DefaultAutoTuneService.java +++ b/src/main/java/com/avaje/ebeaninternal/server/autofetch/service/DefaultAutoTuneService.java @@ -23,6 +23,7 @@ import java.util.Collections; import java.util.Comparator; import java.util.Date; import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; /** * Implementation of the AutoTuneService which is comprised of profiling and query tuning. @@ -41,16 +42,25 @@ public class DefaultAutoTuneService implements AutoTuneService { private final boolean profiling; + private final boolean queryTuning; + + private final String tuningFile; + + private final String profilingFile; + private final String serverName; public DefaultAutoTuneService(SpiEbeanServer server, ServerConfig serverConfig) { AutoTuneConfig config = serverConfig.getAutoTuneConfig(); + this.queryTuning = config.isQueryTuning(); this.profiling = config.isProfiling(); + this.tuningFile = config.getQueryTuningFile(); + this.profilingFile = config.getProfilingFile(); + this.serverName = server.getName(); this.profileManager = new ProfileManager(config, server); this.queryTuner = new BaseQueryTuner(config, server, profileManager); - this.serverName = server.getName(); this.skipCollectionOnShutdown = config.isSkipCollectionOnShutdown(); this.defaultGarbageCollectionWait = (long) config.getGarbageCollectionWait(); } @@ -61,11 +71,19 @@ public class DefaultAutoTuneService implements AutoTuneService { @Override public void startup() { - File file = new File("ebean-autotune.xml"); - AutoTuneXmlReader reader = new AutoTuneXmlReader(); - Autotune profiling = reader.read(file); - for (Origin origin : profiling.getOrigin()) { - queryTuner.load(origin.getKey(), createTunedQueryInfo(origin)); + if (queryTuning) { + File file = new File(tuningFile); + if (!file.exists()) { + logger.warn("AutoTune file {} not found - no automatic tuning will be applied", file.getAbsolutePath()); + + } else { + AutoTuneXmlReader reader = new AutoTuneXmlReader(); + Autotune profiling = reader.read(file); + logger.info("AutoTune loading {} tuning entries", profiling.getOrigin().size()); + for (Origin origin : profiling.getOrigin()) { + queryTuner.load(origin.getKey(), createTunedQueryInfo(origin)); + } + } } } @@ -83,20 +101,32 @@ public class DefaultAutoTuneService implements AutoTuneService { List entries = autoTuneCollection.getEntries(); + // count "new" and "diff" profiling entries + AtomicInteger newCounter = new AtomicInteger(); + AtomicInteger diffCounter = new AtomicInteger(); + for (AutoTuneCollection.Entry entry : entries) { - saveProfilingEntry(document, entry); + saveProfilingEntry(document, entry, newCounter, diffCounter); } - sortDocument(document); + int totalNew = newCounter.get(); + int totalDiff = diffCounter.get(); + if (totalNew == 0 && totalDiff == 0) { + logger.debug("No new or diff entries for profiling server:{}", serverName); - SimpleDateFormat df = new SimpleDateFormat("yyyyMMdd-HHmmss"); - String now = df.format(new Date()); + } else { + sortDocument(document); - File file = new File("ebean-profiling"+"-"+serverName+"-"+now+".xml"); - AutoTuneXmlWriter writer = new AutoTuneXmlWriter(); - writer.write(document, file); + SimpleDateFormat df = new SimpleDateFormat("yyyyMMdd-HHmmss"); + String now = df.format(new Date()); - logger.debug("profiling entries:{}", entries.size()); + // write the file with serverName and now suffix as we can output the profiling many times + File file = new File(profilingFile + "-" + serverName + "-" + now + ".xml"); + AutoTuneXmlWriter writer = new AutoTuneXmlWriter(); + writer.write(document, file); + + logger.debug("writing new:{} diff:{} profiling entries for server:{}", totalNew, totalDiff, serverName); + } } /** @@ -114,6 +144,9 @@ public class DefaultAutoTuneService implements AutoTuneService { } } + /** + * Comparator sort by bean type then key. + */ class OriginNameKeySort implements Comparator { @Override @@ -126,7 +159,7 @@ public class DefaultAutoTuneService implements AutoTuneService { } } - private void saveProfilingEntry(Autotune document, AutoTuneCollection.Entry entry) { + private void saveProfilingEntry(Autotune document, AutoTuneCollection.Entry entry, AtomicInteger newCount, AtomicInteger diffCount) { ObjectGraphOrigin point = entry.getOrigin(); OrmQueryDetail profileDetail = entry.getDetail(); @@ -135,6 +168,7 @@ public class DefaultAutoTuneService implements AutoTuneService { OrmQueryDetail tuneDetail = queryTuner.get(point.getKey()); if (tuneDetail == null) { // New entry + newCount.incrementAndGet(); ProfileNew profileNew = document.getProfileNew(); if (profileNew == null) { profileNew = new ProfileNew(); @@ -146,6 +180,7 @@ public class DefaultAutoTuneService implements AutoTuneService { } else if (!tuneDetail.isAutoTuneEqual(profileDetail)) { // Diff entry + diffCount.incrementAndGet(); Origin origin = createOrigin(entry, point); origin.setOriginal(tuneDetail.toString()); ProfileDiff diff = document.getProfileDiff(); @@ -219,7 +254,6 @@ public class DefaultAutoTuneService implements AutoTuneService { } } - /** * Auto tune the query and enable profiling. */ diff --git a/src/test/resources/ebean.properties b/src/test/resources/ebean.properties index 12018186d..3205cad75 100644 --- a/src/test/resources/ebean.properties +++ b/src/test/resources/ebean.properties @@ -12,7 +12,10 @@ ebean.encryptKeyManager=com.avaje.tests.basic.encrypt.BasicEncyptKeyManager ebean.autotune.querytuning=true +#ebean.autotune.queryTuningFile=${HOME}/config/tuning.xml + ebean.autotune.profiling=true +#ebean.autotune.profilingFile=/tmp/eb-prof ebean.ddl.generate=true From 74330918e16bc7a2eb5928fcc69d98dc2d6023f6 Mon Sep 17 00:00:00 2001 From: Robin Bygrave Date: Thu, 10 Sep 2015 10:04:31 +1200 Subject: [PATCH 9/9] AutoTune update - .gitignore the profiling output --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index bb406be3c..51330a2eb 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,7 @@ target/ logs/ log/ +ebean-profiling*.xml /db /mydb.db !src/test/ddl-review/*.sql