AutoTune update - rename and profiling against original query

This commit is contained in:
Robin Bygrave
2015-09-10 00:18:11 +12:00
parent 71dd0e5728
commit a2a4a15f32
14 changed files with 181 additions and 305 deletions
@@ -1,23 +0,0 @@
package com.avaje.ebean;
/**
* Administrative control of Autofetch during runtime.
*/
public interface AdminAutofetch {
/**
* Fire a garbage collection (hint to the JVM). Assuming garbage collection
* fires this will gather the usage profiling information.
*/
void collectUsageViaGC();
/**
* This will take the current profiling information and update the "tuned
* query detail".
* <p>
* This is done periodically and can also be manually invoked.
* </p>
*/
void updateTunedQueryInfo();
}
@@ -0,0 +1,14 @@
package com.avaje.ebean;
/**
* Administrative control of AutoTune during runtime.
*/
public interface AutoTune {
/**
* Fire a garbage collection (hint to the JVM). Assuming garbage collection
* fires this will gather remaining usage profiling information.
*/
void collectProfiling();
}
@@ -1,11 +1,11 @@
package com.avaje.ebean.config;
/**
* Defines the Autofetch behaviour for a EbeanServer.
* Defines the AutoTune behaviour for a EbeanServer.
*/
public class AutofetchConfig {
public class AutoTuneConfig {
private AutofetchMode mode = AutofetchMode.DEFAULT_ONIFEMPTY;
private AutoTuneMode mode = AutoTuneMode.DEFAULT_ON;
private boolean queryTuning;
@@ -17,27 +17,25 @@ public class AutofetchConfig {
private double profilingRate = 0.01;
private int profileUpdateFrequency = 60;
private int garbageCollectionWait = 100;
private boolean garbageCollectionOnShutdown;
private boolean skipCollectionOnShutdown;
public AutofetchConfig() {
public AutoTuneConfig() {
}
/**
* Return the mode used when autofetch has not been explicit defined on a
* Return the mode used when autoTune has not been explicit defined on a
* query.
*/
public AutofetchMode getMode() {
public AutoTuneMode getMode() {
return mode;
}
/**
* Set the mode used when autofetch has not been explicit defined on a query.
* Set the mode used when autoTune has not been explicit defined on a query.
*/
public void setMode(AutofetchMode mode) {
public void setMode(AutoTuneMode mode) {
this.mode = mode;
}
@@ -74,6 +72,9 @@ public class AutofetchConfig {
* If this is false then the version property will be added when profiling
* detects that the bean is possibly going to be modified.
* </p>
* <p>
* Generally this is not expected to be turned on.
* </p>
*/
public void setQueryTuningAddVersion(boolean queryTuningAddVersion) {
this.queryTuningAddVersion = queryTuningAddVersion;
@@ -128,29 +129,13 @@ public class AutofetchConfig {
this.profilingRate = profilingRate;
}
/**
* Return the frequency in seconds to update the autofetch tuned queries from
* the profiled information.
*/
public int getProfileUpdateFrequency() {
return profileUpdateFrequency;
}
/**
* Set the frequency in seconds to update the autofetch tuned queries from the
* profiled information.
*/
public void setProfileUpdateFrequency(int profileUpdateFrequency) {
this.profileUpdateFrequency = profileUpdateFrequency;
}
/**
* Return the time in millis to wait after a system gc to collect profiling
* information.
* <p>
* The profiling information is collected on object finalise. As such we
* generally don't want to trigger GC (let the JVM do its thing) but on
* shutdown the autofetch manager will trigger System.gc() and then wait
* shutdown the autoTune manager will trigger System.gc() and then wait
* (default 100 millis) to hopefully collect profiling information -
* especially for short run unit tests.
* </p>
@@ -160,33 +145,24 @@ public class AutofetchConfig {
}
/**
* Set the time in millis to wait after a System.gc() to collect profiling
* information.
* Set the time in millis to wait after a System.gc() to collect profiling information.
*/
public void setGarbageCollectionWait(int garbageCollectionWait) {
this.garbageCollectionWait = garbageCollectionWait;
}
/**
* Return true if GC should be trigger on shutdown.
* <p>
* Autofetch profiling information is collected as part of garbage collection.
* </p>
* Return true if profiling collection should be skipped on shutdown.
*/
public boolean isGarbageCollectionOnShutdown() {
return garbageCollectionOnShutdown;
public boolean isSkipCollectionOnShutdown() {
return skipCollectionOnShutdown;
}
/**
* Set to true if you want GC to trigger on shutdown.
* <p>
* This would be done if you want to try and collect Autofetch profiling information
* on shutdown.
* </p>
* Set to true if profiling collection should be skipped on shutdown.
*/
public void setGarbageCollectionOnShutdown(boolean garbageCollectionOnShutdown) {
this.garbageCollectionOnShutdown = garbageCollectionOnShutdown;
public void setSkipCollectionOnShutdown(boolean skipCollectionOnShutdown) {
this.skipCollectionOnShutdown = skipCollectionOnShutdown;
}
/**
@@ -194,16 +170,14 @@ public class AutofetchConfig {
*/
public void loadSettings(PropertiesWrapper p) {
queryTuning = p.getBoolean("autotune.querytuning", queryTuning);
queryTuningAddVersion = p.getBoolean("autotune.queryTuningAddVersion", queryTuningAddVersion);
garbageCollectionOnShutdown = p.getBoolean("autotune.garbageCollectionOnShutdown", garbageCollectionOnShutdown);
queryTuning = p.getBoolean("autoTune.queryTuning", queryTuning);
queryTuningAddVersion = p.getBoolean("autoTune.queryTuningAddVersion", queryTuningAddVersion);
skipCollectionOnShutdown = p.getBoolean("autoTune.skipCollectionOnShutdown", skipCollectionOnShutdown);
profiling = p.getBoolean("autotune.profiling", profiling);
mode = p.getEnum(AutofetchMode.class, "autotune.implicitmode", mode);
mode = p.getEnum(AutoTuneMode.class, "autoTune.mode", mode);
profilingBase = p.getInt("autotune.profiling.base", profilingBase);
profilingRate = p.getDouble("autotune.profiling.rate", profilingRate);
profileUpdateFrequency = p.getInt("autotune.profiling.updatefrequency", profileUpdateFrequency);
profiling = p.getBoolean("autoTune.profiling", profiling);
profilingBase = p.getInt("autoTune.profilingBase", profilingBase);
profilingRate = p.getDouble("autoTune.profilingRate", profilingRate);
}
}
@@ -11,7 +11,7 @@ import com.avaje.ebean.Query;
* query.
* </p>
*/
public enum AutofetchMode {
public enum AutoTuneMode {
/**
* Don't implicitly use Autofetch. Must explicitly turn it on.
@@ -13,8 +13,8 @@ public class AutoTuneCollection {
List<Entry> entries = new ArrayList<Entry>();
public Entry add(ObjectGraphOrigin origin, OrmQueryDetail detail) {
Entry entry = new Entry(origin, detail);
public Entry add(ObjectGraphOrigin origin, OrmQueryDetail detail, String sourceQuery) {
Entry entry = new Entry(origin, detail, sourceQuery);
entries.add(entry);
return entry;
}
@@ -38,14 +38,20 @@ public class AutoTuneCollection {
*/
private final OrmQueryDetail detail;
/**
* The original/existing query detail.
*/
private final String originalQuery;
/**
* Summary execution statistics for queries related to this origin point.
*/
private final List<EntryQuery> queries = new ArrayList<EntryQuery>();
public Entry(ObjectGraphOrigin origin, OrmQueryDetail detail) {
public Entry(ObjectGraphOrigin origin, OrmQueryDetail detail, String originalQuery) {
this.origin = origin;
this.detail = detail;
this.originalQuery = originalQuery;
}
public void addQuery(EntryQuery entryQuery) {
@@ -60,6 +66,10 @@ public class AutoTuneCollection {
return detail;
}
public String getOriginalQuery() {
return originalQuery;
}
public List<EntryQuery> getQueries() {
return queries;
}
@@ -1,8 +1,6 @@
package com.avaje.ebeaninternal.server.autofetch;
import com.avaje.ebean.AdminAutofetch;
import com.avaje.ebean.config.ServerConfig;
import com.avaje.ebeaninternal.api.SpiEbeanServer;
import com.avaje.ebean.AutoTune;
import com.avaje.ebeaninternal.api.SpiQuery;
/**
@@ -17,7 +15,7 @@ import com.avaje.ebeaninternal.api.SpiQuery;
* etc). These are applied to the query when tuneQuery() is called.
* </p>
*/
public interface AutoTuneService extends AdminAutofetch {
public interface AutoTuneService extends AutoTune {
/**
* Load the query tuning information.
@@ -41,16 +39,7 @@ public interface AutoTuneService extends AdminAutofetch {
* Fire a garbage collection (hint to the JVM). Assuming garbage collection
* fires this will gather the usage profiling information.
*/
void collectUsageViaGC();
/**
* This will take the current profiling information and update the "tuned
* query detail".
* <p>
* This is done periodically and can also be manually invoked.
* </p>
*/
void updateTunedQueryInfo();
void collectProfiling();
/**
* On shutdown fire garbage collection and collect statistics. Note that
@@ -2,8 +2,8 @@ package com.avaje.ebeaninternal.server.autofetch.service;
import com.avaje.ebean.bean.CallStack;
import com.avaje.ebean.bean.ObjectGraphNode;
import com.avaje.ebean.config.AutofetchConfig;
import com.avaje.ebean.config.AutofetchMode;
import com.avaje.ebean.config.AutoTuneConfig;
import com.avaje.ebean.config.AutoTuneMode;
import com.avaje.ebeaninternal.api.SpiEbeanServer;
import com.avaje.ebeaninternal.api.SpiQuery;
import com.avaje.ebeaninternal.server.autofetch.ProfilingListener;
@@ -22,7 +22,7 @@ public class BaseQueryTuner {
private boolean profiling;
private final AutofetchMode mode;
private final AutoTuneMode mode;
/**
* Map of the tuned query details per profile query point.
@@ -34,7 +34,9 @@ public class BaseQueryTuner {
private final ProfilingListener profilingListener;
public BaseQueryTuner(AutofetchConfig config, SpiEbeanServer server, ProfilingListener profilingListener) {
boolean fullProfiling = true;
public BaseQueryTuner(AutoTuneConfig config, SpiEbeanServer server, ProfilingListener profilingListener) {
this.server = server;
this.profilingListener = profilingListener;
this.mode = config.getMode();
@@ -58,8 +60,6 @@ public class BaseQueryTuner {
return (info == null) ? null : info.getTunedDetail();
}
boolean fullProfiling = true;
/**
* Auto tune the query and enable profiling.
*/
@@ -90,7 +90,7 @@ public class BaseQueryTuner {
ObjectGraphNode origin = query.setOrigin(stack);
if (profiling) {
if (profilingListener.isProfileRequest(origin)) {
if (profilingListener.isProfileRequest(origin, query)) {
// collect more profiling based on profiling rate etc
query.setProfilingListener(profilingListener);
}
@@ -108,7 +108,7 @@ public class BaseQueryTuner {
// create a query point to identify the query
ObjectGraphNode origin = query.setOrigin(stack);
if (profilingListener.isProfileRequest(origin)) {
if (profilingListener.isProfileRequest(origin, query)) {
// collect more profiling based on profiling rate etc
query.setProfilingListener(profilingListener);
}
@@ -1,7 +1,7 @@
package com.avaje.ebeaninternal.server.autofetch.service;
import com.avaje.ebean.bean.ObjectGraphOrigin;
import com.avaje.ebean.config.AutofetchConfig;
import com.avaje.ebean.config.AutoTuneConfig;
import com.avaje.ebean.config.ServerConfig;
import com.avaje.ebeaninternal.api.SpiEbeanServer;
import com.avaje.ebeaninternal.api.SpiQuery;
@@ -18,6 +18,8 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.List;
/**
@@ -29,87 +31,107 @@ public class DefaultAutoTuneService implements AutoTuneService {
private final long defaultGarbageCollectionWait;
private final boolean garbageCollectionOnShutdown;
private final boolean skipCollectionOnShutdown;
private final BaseQueryTuner queryTuner;
private final ProfileManager profileManager;
private final boolean profiling;
public DefaultAutoTuneService(SpiEbeanServer server, ServerConfig serverConfig) {
AutofetchConfig config = serverConfig.getAutofetchConfig();
AutoTuneConfig config = serverConfig.getAutoTuneConfig();
this.profiling = config.isProfiling();
this.profileManager = new ProfileManager(config, server);
this.queryTuner = new BaseQueryTuner(config, server, profileManager);
this.garbageCollectionOnShutdown = config.isGarbageCollectionOnShutdown();
this.skipCollectionOnShutdown = config.isSkipCollectionOnShutdown();
this.defaultGarbageCollectionWait = (long) config.getGarbageCollectionWait();
}
/**
* Load the query tuning information from it's data store.
*/
@Override
public void startup() {
File file = new File("ebean-autotune.xml");
AutoTuneXmlReader reader = new AutoTuneXmlReader();
Autotune profiling = reader.read(file);
List<Origin> originList = profiling.getOrigin();
for (Origin origin : originList) {
String key = origin.getKey();
String detail = origin.getDetail();
OrmQueryDetailParser parser = new OrmQueryDetailParser(detail);
OrmQueryDetail fetchDetail = parser.parse();
TunedQueryInfo tunedQueryInfo = new TunedQueryInfo(fetchDetail);
queryTuner.load(key, tunedQueryInfo);
for (Origin origin : profiling.getOrigin()) {
queryTuner.load(origin.getKey(), createTunedQueryInfo(origin));
}
}
private void saveProfiling() {
@NotNull
private TunedQueryInfo createTunedQueryInfo(Origin origin) {
OrmQueryDetail detail = new OrmQueryDetailParser(origin.getDetail()).parse();
return new TunedQueryInfo(detail);
}
private void saveProfiling(boolean reset) {
Autotune document = new Autotune();
AutoTuneCollection autoTuneCollection = profileManager.profilingCollection(false);
AutoTuneCollection autoTuneCollection = profileManager.profilingCollection(reset);
List<AutoTuneCollection.Entry> entries = autoTuneCollection.getEntries();
for (AutoTuneCollection.Entry entry : entries) {
ObjectGraphOrigin point = entry.getOrigin();
OrmQueryDetail profileDetail = entry.getDetail();
OrmQueryDetail tuneDetail = queryTuner.get(point.getKey());
if (tuneDetail == null) {
ProfileNew profileNew = document.getProfileNew();
if (profileNew == null) {
profileNew = new ProfileNew();
document.setProfileNew(profileNew);
}
profileNew.getOrigin().add( createOrigin(entry, point));
} else if (!tuneDetail.isAutoTuneEqual(profileDetail)) {
Origin origin1 = createOrigin(entry, point);
origin1.setTuneDetail(tuneDetail.toString());
ProfileDiff diff = document.getProfileDiff();
if (diff == null) {
diff = new ProfileDiff();
document.setProfileDiff(diff);
}
diff.getOrigin().add(origin1);
}
saveProfilingEntry(document, entry);
}
File file = new File("ebean-autotune-profiling.xml");
SimpleDateFormat df = new SimpleDateFormat("yyyyMMdd-HHmmss");
String now = df.format(new Date());
File file = new File("ebean-autotune-profiling"+"-"+now+".xml");
AutoTuneXmlWriter writer = new AutoTuneXmlWriter();
writer.write(document, file);
}
private void saveProfilingEntry(Autotune document, AutoTuneCollection.Entry entry) {
ObjectGraphOrigin point = entry.getOrigin();
OrmQueryDetail profileDetail = entry.getDetail();
// compare with the existing query tuning entry
OrmQueryDetail tuneDetail = queryTuner.get(point.getKey());
if (tuneDetail == null) {
// New entry
ProfileNew profileNew = document.getProfileNew();
if (profileNew == null) {
profileNew = new ProfileNew();
document.setProfileNew(profileNew);
}
Origin origin = createOrigin(entry, point);
origin.setOriginal(entry.getOriginalQuery());
profileNew.getOrigin().add(origin);
} else if (!tuneDetail.isAutoTuneEqual(profileDetail)) {
// Diff entry
Origin origin = createOrigin(entry, point);
origin.setOriginal(tuneDetail.toString());
ProfileDiff diff = document.getProfileDiff();
if (diff == null) {
diff = new ProfileDiff();
document.setProfileDiff(diff);
}
diff.getOrigin().add(origin);
}
}
/**
* Create the XML Origin bean for the given entry and ObjectGraphOrigin.
*/
@NotNull
private Origin createOrigin(AutoTuneCollection.Entry entry, ObjectGraphOrigin point) {
Origin origin1 = new Origin();
origin1.setKey(point.getKey());
origin1.setBeanType(point.getBeanType());
origin1.setDetail(entry.getDetail().toString());
origin1.setCallStack(point.getCallStack().description("\n"));
return origin1;
Origin origin = new Origin();
origin.setKey(point.getKey());
origin.setBeanType(point.getBeanType());
origin.setDetail(entry.getDetail().toString());
origin.setCallStack(point.getCallStack().description("\n"));
return origin;
}
/**
@@ -120,10 +142,11 @@ public class DefaultAutoTuneService implements AutoTuneService {
* collection may not otherwise occur at all.
* </p>
*/
@Override
public void shutdown() {
if (garbageCollectionOnShutdown) {
collectUsageViaGC(-1);
saveProfiling();
if (profiling && !skipCollectionOnShutdown) {
collectProfiling(-1);
saveProfiling(false);
}
}
@@ -143,11 +166,12 @@ public class DefaultAutoTuneService implements AutoTuneService {
* defaults to 100 milliseconds.
* </p>
*/
public void collectUsageViaGC() {
collectUsageViaGC(-1);
@Override
public void collectProfiling() {
collectProfiling(-1);
}
public void collectUsageViaGC(long waitMillis) {
public void collectProfiling(long waitMillis) {
System.gc();
try {
if (waitMillis < 0) {
@@ -157,99 +181,13 @@ public class DefaultAutoTuneService implements AutoTuneService {
} catch (InterruptedException e) {
logger.warn("Error while sleeping after System.gc() request.", e);
}
updateTunedQueryInfo();
}
/**
* Update the tuned fetch plans from the current usage information.
*/
public void updateTunedQueryInfo() {
// if (!profiling) {
// // we are not collecting any profiling information at
// // the moment so don't try updating the tuned query plans.
// return "Not profiling";
// }
//
// synchronized (statisticsMonitor) {
//
// Counters counters = new Counters();
//
// for (ProfileOrigin queryPointStatistics : statisticsMap.values()) {
// if (!queryPointStatistics.hasUsage()) {
// // no usage statistics collected yet...
// counters.incrementNoUsage();
// } else {
// updateTunedQueryFromUsage(counters, queryPointStatistics);
// }
// }
//
// String summaryInfo = counters.toString();
//
// if (counters.isInteresting()) {
// // only log it if its interesting
// logging.logSummary(summaryInfo);
// }
//
// return summaryInfo;
// }
}
//
// private void updateTunedQueryFromUsage(Counters counters, ProfileOrigin statistics) {
//
// ObjectGraphOrigin queryPoint = statistics.getOrigin();
// String beanType = queryPoint.getBeanType();
//
// try {
// Class<?> beanClass = ClassUtil.forName(beanType, this.getClass());
// BeanDescriptor<?> beanDescriptor = server.getBeanDescriptor(beanClass);
// if (beanDescriptor != null) {
//
// // Determine the fetch plan from the latest statistics.
// // Use this to compare with current "tuned fetch plan".
// OrmQueryDetail newFetchDetail = statistics.buildTunedFetch(beanDescriptor);
//
// // get the current tuned fetch info...
// TunedQueryInfo currentFetch = tunedQueryInfoMap.get(queryPoint.getKey());
//
// if (currentFetch == null) {
// // its a new fetch plan, add it.
// counters.incrementNew();
//
// currentFetch = statistics.createTunedFetch(newFetchDetail);
// logging.logNew(currentFetch);
// tunedQueryInfoMap.put(queryPoint.getKey(), currentFetch);
//
// } else if (!currentFetch.isSame(newFetchDetail)) {
// // the fetch plan has changed, update it.
// counters.incrementModified();
//
// logging.logChanged(currentFetch, newFetchDetail);
// currentFetch.setTunedDetail(newFetchDetail);
//
// } else {
// // the fetch plan has not changed...
// counters.incrementUnchanged();
// }
//
// currentFetch.setProfileCount(statistics.getCounter());
// }
//
// } catch (ClassNotFoundException e) {
// // expected after renaming/moving an entity bean
// String msg = e.toString() + " updating autoFetch tuned query for " + beanType
// + ". It isLikely this bean has been renamed or moved";
// logging.logInfo(msg, null);
// statisticsMap.remove(statistics.getOrigin().getKey());
// }
// }
//
/**
* Auto tune the query and enable profiling.
*/
@Override
public boolean tuneQuery(SpiQuery<?> query) {
return queryTuner.tuneQuery(query);
}
@@ -330,7 +330,7 @@ public final class DefaultServer implements SpiServer, SpiEbeanServer {
return ddlGenerator;
}
public AdminAutofetch getAdminAutofetch() {
public AutoTune getAutoTune() {
return autoTuneService;
}
@@ -1039,7 +1039,7 @@ public final class DefaultServer implements SpiServer, SpiEbeanServer {
public <T> SpiOrmQueryRequest<T> createQueryRequest(BeanDescriptor<T> desc, SpiQuery<T> query, Transaction t) {
if (desc.isAutoFetchTunable() && !query.isSqlSelect() && !autoTuneService.tuneQuery(query)) {
if (desc.isAutoTunable() && !query.isSqlSelect() && !autoTuneService.tuneQuery(query)) {
// use deployment FetchType.LAZY/EAGER annotations
// to define the 'default' select clause
query.setDefaultSelectClause();
@@ -1047,13 +1047,13 @@ public final class DefaultServer implements SpiServer, SpiEbeanServer {
if (query.selectAllForLazyLoadProperty()) {
// we need to select all properties to ensure the lazy load property
// was included (was not included by default or via autofetch).
// was included (was not included by default or via autoTune).
if (logger.isDebugEnabled()) {
logger.debug("Using selectAllForLazyLoadProperty");
}
}
// if determine cost and no origin for Autofetch
// if determine cost and no origin for AutoTune
if (query.getParentNode() == null) {
query.setOrigin(createCallStack());
}
@@ -126,7 +126,7 @@ public class BeanDescriptor<T> implements MetaBeanInfo, SpiBeanType<T> {
*/
private final String selectLastInsertedId;
private final boolean autoFetchTunable;
private final boolean autoTunable;
/**
* The concurrency mode for beans of this type.
@@ -366,7 +366,7 @@ public class BeanDescriptor<T> implements MetaBeanInfo, SpiBeanType<T> {
this.baseTable = InternString.intern(deploy.getBaseTable());
this.baseTableAsOf = deploy.getBaseTableAsOf();
this.baseTableVersionsBetween = deploy.getBaseTableVersionsBetween();
this.autoFetchTunable = EntityType.ORM.equals(entityType) && (beanFinder == null);
this.autoTunable = EntityType.ORM.equals(entityType) && (beanFinder == null);
// helper object used to derive lists of properties
DeployBeanPropertyLists listHelper = new DeployBeanPropertyLists(owner, this, deploy);
@@ -1603,10 +1603,10 @@ public class BeanDescriptor<T> implements MetaBeanInfo, SpiBeanType<T> {
}
/**
* Return true if queries for beans of this type are autoFetch tunable.
* Return true if queries for beans of this type are auto tunable.
*/
public boolean isAutoFetchTunable() {
return autoFetchTunable;
public boolean isAutoTunable() {
return autoTunable;
}
/**