AutoTune update - rename and profiling against original query

This commit is contained in:
Robin Bygrave
2015-09-10 00:17:49 +12:00
parent 116cc3a3fe
commit 71dd0e5728
15 changed files with 322 additions and 268 deletions
@@ -106,10 +106,9 @@ public interface EbeanServer {
void shutdown(boolean shutdownDataSource, boolean deregisterDriver);
/**
* Return the AdminAutofetch which is used to control and configure the
* Autofetch service at runtime.
* Return AutoTune which is used to control the AutoTune service at runtime.
*/
AdminAutofetch getAdminAutofetch();
AutoTune getAutoTune();
/**
* Return the name. This is used with {@link Ebean#getServer(String)} to get a
@@ -130,7 +130,7 @@ public class ServerConfig {
/**
* Config controlling the autofetch behaviour.
*/
private AutofetchConfig autofetchConfig = new AutofetchConfig();
private AutoTuneConfig autoTuneConfig = new AutoTuneConfig();
/**
* The JSON format used for DateTime types. Default to millis.
@@ -1106,15 +1106,15 @@ public class ServerConfig {
/**
* Return the configuration for the Autofetch feature.
*/
public AutofetchConfig getAutofetchConfig() {
return autofetchConfig;
public AutoTuneConfig getAutoTuneConfig() {
return autoTuneConfig;
}
/**
* Set the configuration for the Autofetch feature.
*/
public void setAutofetchConfig(AutofetchConfig autofetchConfig) {
this.autofetchConfig = autofetchConfig;
public void setAutoTuneConfig(AutoTuneConfig autoTuneConfig) {
this.autoTuneConfig = autoTuneConfig;
}
/**
@@ -2003,7 +2003,7 @@ public class ServerConfig {
* This is broken out for the same reason as above - preserve existing behaviour but let it be overridden.
*/
protected void loadAutofetchSettings(PropertiesWrapper p) {
autofetchConfig.loadSettings(p);
autoTuneConfig.loadSettings(p);
}
/**
@@ -2017,8 +2017,8 @@ public class ServerConfig {
if (namingConvention != null) {
namingConvention.loadFromProperties(p);
}
if (autofetchConfig == null) {
autofetchConfig = new AutofetchConfig();
if (autoTuneConfig == null) {
autoTuneConfig = new AutoTuneConfig();
}
loadAutofetchSettings(p);
@@ -2,6 +2,7 @@ package com.avaje.ebeaninternal.server.autofetch;
import com.avaje.ebean.bean.NodeUsageListener;
import com.avaje.ebean.bean.ObjectGraphNode;
import com.avaje.ebeaninternal.api.SpiQuery;
/**
* Profiling listener gets call backs for node usage and the associated query executions.
@@ -21,5 +22,5 @@ public interface ProfilingListener extends NodeUsageListener {
* Return true if this request should be profiled (based on the
* profiling ratio and collection count for this origin).
*/
boolean isProfileRequest(ObjectGraphNode origin);
boolean isProfileRequest(ObjectGraphNode origin, SpiQuery<?> query);
}
@@ -23,7 +23,7 @@ import javax.xml.bind.annotation.XmlType;
* &lt;attribute name="key" use="required" type="{http://www.w3.org/2001/XMLSchema}string" />
* &lt;attribute name="beanType" type="{http://www.w3.org/2001/XMLSchema}string" />
* &lt;attribute name="detail" type="{http://www.w3.org/2001/XMLSchema}string" />
* &lt;attribute name="tuneDetail" type="{http://www.w3.org/2001/XMLSchema}string" />
* &lt;attribute name="original" type="{http://www.w3.org/2001/XMLSchema}string" />
* &lt;/restriction>
* &lt;/complexContent>
* &lt;/complexType>
@@ -45,8 +45,8 @@ public class Origin {
protected String beanType;
@XmlAttribute(name = "detail")
protected String detail;
@XmlAttribute(name = "tuneDetail")
protected String tuneDetail;
@XmlAttribute(name = "original")
protected String original;
/**
* Gets the value of the callStack property.
@@ -145,27 +145,27 @@ public class Origin {
}
/**
* Gets the value of the tuneDetail property.
* Gets the value of the original property.
*
* @return
* possible object is
* {@link String }
*
*/
public String getTuneDetail() {
return tuneDetail;
public String getOriginal() {
return original;
}
/**
* Sets the value of the tuneDetail property.
* Sets the value of the original property.
*
* @param value
* allowed object is
* {@link String }
*
*/
public void setTuneDetail(String value) {
this.tuneDetail = value;
public void setOriginal(String value) {
this.original = value;
}
}
@@ -3,8 +3,9 @@ package com.avaje.ebeaninternal.server.autofetch.service;
import com.avaje.ebean.bean.NodeUsageCollector;
import com.avaje.ebean.bean.ObjectGraphNode;
import com.avaje.ebean.bean.ObjectGraphOrigin;
import com.avaje.ebean.config.AutofetchConfig;
import com.avaje.ebean.config.AutoTuneConfig;
import com.avaje.ebeaninternal.api.SpiEbeanServer;
import com.avaje.ebeaninternal.api.SpiQuery;
import com.avaje.ebeaninternal.server.autofetch.AutoTuneCollection;
import com.avaje.ebeaninternal.server.autofetch.ProfilingListener;
import com.avaje.ebeaninternal.server.deploy.BeanDescriptor;
@@ -40,7 +41,7 @@ public class ProfileManager implements ProfilingListener {
private final SpiEbeanServer server;
public ProfileManager(AutofetchConfig config, SpiEbeanServer server) {
public ProfileManager(AutoTuneConfig config, SpiEbeanServer server) {
this.server = server;
this.profilingRate = config.getProfilingRate();
this.profilingBase = config.getProfilingBase();
@@ -48,10 +49,17 @@ public class ProfileManager implements ProfilingListener {
}
@Override
public boolean isProfileRequest(ObjectGraphNode origin) {
public boolean isProfileRequest(ObjectGraphNode origin, SpiQuery<?> query) {
ProfileOrigin profileOrigin = profileMap.get(origin.getOriginQueryPoint().getKey());
return profileOrigin == null || profileOrigin.isProfile();
if (profileOrigin == null) {
profileOrigin = new ProfileOrigin(origin.getOriginQueryPoint(), queryTuningAddVersion, profilingBase, profilingRate);
profileOrigin.setOriginalQuery(query.getDetail().toString());
profileMap.put(origin.getOriginQueryPoint().getKey(), profileOrigin);
return true;
} else {
return profileOrigin.isProfile();
}
}
/**
@@ -38,6 +38,8 @@ public class ProfileOrigin {
private final AtomicLong profileCount = new AtomicLong();
private String originalQuery;
public ProfileOrigin(ObjectGraphOrigin origin, boolean queryTuningAddVersion, int profilingBase, double profilingRate) {
this.origin = origin;
this.queryTuningAddVersion = queryTuningAddVersion;
@@ -45,6 +47,14 @@ public class ProfileOrigin {
this.profilingRate = profilingRate;
}
public String getOriginalQuery() {
return originalQuery;
}
public void setOriginalQuery(String originalQuery) {
this.originalQuery = originalQuery;
}
/**
* Return true if this query should be profiled based on a percentage rate.
*/
@@ -74,7 +84,7 @@ public class ProfileOrigin {
}
OrmQueryDetail detail = buildDetail(rootDesc);
AutoTuneCollection.Entry entry = req.add(origin, detail);
AutoTuneCollection.Entry entry = req.add(origin, detail, originalQuery);
Collection<ProfileOriginQuery> values = queryStatsMap.values();
for (ProfileOriginQuery queryEntry : values) {
@@ -81,7 +81,7 @@ public class OrmQueryDetail implements Serializable {
}
/**
* Return true if equal in terms of autofetch (select and joins).
* Return true if equal in terms of autoTune (select and fetch).
*/
public boolean isAutoTuneEqual(OrmQueryDetail otherDetail) {
@@ -91,6 +91,9 @@ public class OrmQueryDetail implements Serializable {
if (fetchPaths == null) {
return otherDetail.fetchPaths == null;
}
if (fetchPaths.size() != otherDetail.fetchPaths.size()) {
return false;
}
Set<Map.Entry<String, OrmQueryProperties>> entries = fetchPaths.entrySet();
for (Map.Entry<String, OrmQueryProperties> entry : entries) {
OrmQueryProperties chunk = otherDetail.getChunk(entry.getKey(), false);
@@ -100,7 +103,6 @@ public class OrmQueryDetail implements Serializable {
}
return true;
//return autofetchPlanHash() == otherDetail.autofetchPlanHash();
}
private boolean isSame(OrmQueryProperties p1, OrmQueryProperties p2) {
@@ -110,22 +112,6 @@ public class OrmQueryDetail implements Serializable {
return p1.isSame(p2);
}
// /**
// * Calculate the hash for the query plan.
// */
// private int autofetchPlanHash() {
//
// int hc = (baseProps == null ? 1 : baseProps.autofetchPlanHash());
//
// if (fetchPaths != null) {
// for (OrmQueryProperties p : fetchPaths.values()) {
// hc = hc * 31 + p.autofetchPlanHash();
// }
// }
//
// return hc;
// }
public String toString() {
StringBuilder sb = new StringBuilder();
if (baseProps != null) {
@@ -236,7 +222,7 @@ public class OrmQueryDetail implements Serializable {
boolean tuned = false;
OrmQueryProperties tunedRoot = tunedDetail.getChunk(null, false);
if (tunedRoot != null && tunedRoot.hasProperties()) {
if (tunedRoot != null) {
tuned = true;
baseProps.setTunedProperties(tunedRoot);
@@ -215,9 +215,13 @@ public class OrmQueryProperties implements Serializable {
* Set the properties from a matching autofetch tuned properties.
*/
public void setTunedProperties(OrmQueryProperties tunedProperties) {
this.properties = tunedProperties.properties;
this.trimmedProperties = tunedProperties.trimmedProperties;
this.included = tunedProperties.included;
if (tunedProperties.hasProperties()) {
this.properties = tunedProperties.properties;
this.trimmedProperties = tunedProperties.trimmedProperties;
this.included = tunedProperties.included;
this.queryFetchBatch = Math.max(queryFetchBatch, tunedProperties.queryFetchBatch);
this.lazyFetchBatch = Math.max(lazyFetchBatch, tunedProperties.lazyFetchBatch);
}
}
/**
@@ -262,6 +266,7 @@ public class OrmQueryProperties implements Serializable {
copy.parentPath = parentPath;
copy.path = path;
copy.properties = properties;
copy.trimmedProperties = trimmedProperties;
copy.cache = cache;
copy.readOnly = readOnly;
copy.queryFetchAll = queryFetchAll;
@@ -216,7 +216,7 @@ public class TDSpiEbeanServer implements SpiEbeanServer {
}
@Override
public AdminAutofetch getAdminAutofetch() {
public AutoTune getAutoTune() {
return null;
}
@@ -77,4 +77,43 @@ public class OrmQueryDetailParserTest extends BaseTestCase {
assertThat(chunk.getAllIncludedProperties()).contains("sku","description");
}
@Test
public void testParseWithPlusQuery() throws Exception {
OrmQueryDetailParser p = new OrmQueryDetailParser("select (id,name) fetch customer (+query,id,name,email)");
OrmQueryDetail detail = p.parse();
OrmQueryProperties root = detail.getChunk(null, false);
assertNull(root.getPath());
assertThat(root.getAllIncludedProperties()).contains("id", "name");
OrmQueryProperties chunk = detail.getChunk("customer", false);
assertThat(chunk.getPath()).isEqualTo("customer");
assertThat(chunk.getAllIncludedProperties()).contains("id", "name", "email");
assertThat(chunk.isQueryFetch()).isTrue();
}
@Test
public void testTuneApply() {
OrmQueryDetailParser p = new OrmQueryDetailParser("select (status) fetch customer (email)");
OrmQueryDetail detail = p.parse();
OrmQueryDetailParser p2 = new OrmQueryDetailParser("select (id,name) fetch customer (+query,id,name,email)");
OrmQueryDetail tune = p2.parse();
detail.tuneFetchProperties(tune);
OrmQueryProperties root = detail.getChunk(null, false);
assertNull(root.getPath());
assertThat(root.getAllIncludedProperties()).contains("id", "name");
OrmQueryProperties chunk = detail.getChunk("customer", false);
assertThat(chunk.getPath()).isEqualTo("customer");
assertThat(chunk.getAllIncludedProperties()).contains("id", "name", "email");
assertThat(chunk.isQueryFetch()).isTrue();
}
}
@@ -1,44 +1,41 @@
package com.avaje.tests.basic;
import java.util.List;
import org.junit.Test;
import com.avaje.ebean.AdminAutofetch;
import com.avaje.ebean.BaseTestCase;
import com.avaje.ebean.Ebean;
import com.avaje.ebean.Query;
import com.avaje.tests.model.basic.Customer;
import com.avaje.tests.model.basic.Order;
import com.avaje.tests.model.basic.OrderDetail;
import com.avaje.tests.model.basic.ResetBasicData;
public class TestBatchLazy extends BaseTestCase {
@Test
public void testMe() {
ResetBasicData.reset();
Query<Order> query = Ebean.find(Order.class);
List<Order> list = query.findList();
for (Order order : list) {
Customer customer = order.getCustomer();
customer.getName();
List<OrderDetail> details = order.getDetails();
for (OrderDetail orderDetail : details) {
orderDetail.getProduct().getSku();
}
}
AdminAutofetch adminAutofetch = Ebean.getServer(null).getAdminAutofetch();
adminAutofetch.collectUsageViaGC();
}
}
package com.avaje.tests.basic;
import com.avaje.ebean.BaseTestCase;
import com.avaje.ebean.Ebean;
import com.avaje.ebean.Query;
import com.avaje.tests.model.basic.Customer;
import com.avaje.tests.model.basic.Order;
import com.avaje.tests.model.basic.OrderDetail;
import com.avaje.tests.model.basic.ResetBasicData;
import org.junit.Test;
import java.util.List;
public class TestBatchLazy extends BaseTestCase {
@Test
public void testMe() {
ResetBasicData.reset();
Query<Order> query = Ebean.find(Order.class);
List<Order> list = query.findList();
for (Order order : list) {
Customer customer = order.getCustomer();
customer.getName();
List<OrderDetail> details = order.getDetails();
for (OrderDetail orderDetail : details) {
orderDetail.getProduct().getSku();
}
}
Ebean.getDefaultServer().getAutoTune().collectProfiling();
}
}
@@ -24,8 +24,8 @@ public class TestByteOnly extends BaseTestCase {
Ebean.save(e2);
// Ebean.getServer(null).getAdminAutofetch().collectUsageViaGC();
// Ebean.getServer(null).getAdminAutofetch().updateTunedQueryInfo();
// Ebean.getServer(null).getAutoTune().collectProfiling();
// Ebean.getServer(null).getAutoTune().updateTunedQueryInfo();
System.out.println("done");
}
@@ -1,82 +1,78 @@
package com.avaje.tests.query;
import java.util.List;
import org.junit.Test;
import com.avaje.ebean.AdminAutofetch;
import com.avaje.ebean.BaseTestCase;
import com.avaje.ebean.Ebean;
import com.avaje.ebean.Query;
import com.avaje.ebean.bean.ObjectGraphNode;
import com.avaje.ebean.bean.ObjectGraphOrigin;
import com.avaje.ebeaninternal.api.SpiQuery;
import com.avaje.tests.model.basic.Address;
import com.avaje.tests.model.basic.Customer;
import com.avaje.tests.model.basic.Order;
import com.avaje.tests.model.basic.ResetBasicData;
public class TestAutofetchTuneWithJoin extends BaseTestCase {
@Test
public void test() {
runQuery();
collectUsage();
}
private void runQuery() {
ResetBasicData.reset();
Query<Order> q = Ebean.find(Order.class)
.setAutofetch(true)
//.fetch("customer")
//.fetch("customer.contacts")
.where().lt("id", 3).query();
List<Order> list = q.findList();
for (int i = 0; i < list.size(); i++) {
Order order = list.get(i);
order.getOrderDate();
order.getShipDate();
// order.setShipDate(new Date(System.currentTimeMillis()));
Customer customer = order.getCustomer();
customer.getName();
Address shippingAddress = customer.getShippingAddress();
if (shippingAddress != null) {
shippingAddress.getLine1();
shippingAddress.getCity();
}
// customer.getContacts()
}
SpiQuery<?> sq = (SpiQuery<?>) q;
ObjectGraphNode parentNode = sq.getParentNode();
ObjectGraphOrigin origin = parentNode.getOriginQueryPoint();
System.out.println("Origin:" + origin.getKey());
// MetaAutoFetchStatistic metaAutoFetchStatistic =
// ((DefaultOrmQuery<?>)q).getMetaAutoFetchStatistic();
// if (metaAutoFetchStatistic != null) {
// List<NodeUsageStats> nodeUsageStats =
// metaAutoFetchStatistic.getNodeUsageStats();
// System.out.println(nodeUsageStats);
// List<QueryStats> queryStats = metaAutoFetchStatistic.getQueryStats();
// System.out.println(queryStats);
// }
if (q.isAutofetchTuned()) {
System.out.println("TUNED...");
}
}
private static void collectUsage() {
AdminAutofetch adminAutofetch = Ebean.getServer(null).getAdminAutofetch();
adminAutofetch.collectUsageViaGC();
}
}
package com.avaje.tests.query;
import com.avaje.ebean.BaseTestCase;
import com.avaje.ebean.Ebean;
import com.avaje.ebean.Query;
import com.avaje.ebean.bean.ObjectGraphNode;
import com.avaje.ebean.bean.ObjectGraphOrigin;
import com.avaje.ebeaninternal.api.SpiQuery;
import com.avaje.tests.model.basic.Address;
import com.avaje.tests.model.basic.Customer;
import com.avaje.tests.model.basic.Order;
import com.avaje.tests.model.basic.ResetBasicData;
import org.junit.Test;
import java.util.List;
public class TestAutofetchTuneWithJoin extends BaseTestCase {
@Test
public void test() {
runQuery();
collectUsage();
}
private void runQuery() {
ResetBasicData.reset();
Query<Order> q = Ebean.find(Order.class)
.setAutofetch(true)
//.fetch("customer")
//.fetch("customer.contacts")
.where().lt("id", 3).query();
List<Order> list = q.findList();
for (int i = 0; i < list.size(); i++) {
Order order = list.get(i);
order.getOrderDate();
order.getShipDate();
// order.setShipDate(new Date(System.currentTimeMillis()));
Customer customer = order.getCustomer();
customer.getName();
Address shippingAddress = customer.getShippingAddress();
if (shippingAddress != null) {
shippingAddress.getLine1();
shippingAddress.getCity();
}
// customer.getContacts()
}
SpiQuery<?> sq = (SpiQuery<?>) q;
ObjectGraphNode parentNode = sq.getParentNode();
ObjectGraphOrigin origin = parentNode.getOriginQueryPoint();
System.out.println("Origin:" + origin.getKey());
// MetaAutoFetchStatistic metaAutoFetchStatistic =
// ((DefaultOrmQuery<?>)q).getMetaAutoFetchStatistic();
// if (metaAutoFetchStatistic != null) {
// List<NodeUsageStats> nodeUsageStats =
// metaAutoFetchStatistic.getNodeUsageStats();
// System.out.println(nodeUsageStats);
// List<QueryStats> queryStats = metaAutoFetchStatistic.getQueryStats();
// System.out.println(queryStats);
// }
if (q.isAutofetchTuned()) {
System.out.println("TUNED...");
}
}
private static void collectUsage() {
Ebean.getDefaultServer().getAutoTune().collectProfiling();
}
}
@@ -1,82 +1,96 @@
package com.avaje.tests.query.autotune;
import com.avaje.ebean.AdminAutofetch;
import com.avaje.ebean.BaseTestCase;
import com.avaje.ebean.Ebean;
import com.avaje.ebean.Query;
import com.avaje.ebean.bean.ObjectGraphNode;
import com.avaje.ebean.bean.ObjectGraphOrigin;
import com.avaje.ebeaninternal.api.SpiQuery;
import com.avaje.tests.model.basic.Address;
import com.avaje.tests.model.basic.Customer;
import com.avaje.tests.model.basic.Order;
import com.avaje.tests.model.basic.ResetBasicData;
import org.junit.Test;
import java.util.List;
public class TestAutoTuneProfiling extends BaseTestCase {
@Test
public void test() {
ResetBasicData.reset();
for (int i = 0; i < 1; i++) {
execute();
}
collectUsage();
}
private void execute() {
useOrderDate();
useOrderDateCustomerName();
useLots();
}
private Order findById(long id) {
return Ebean.find(Order.class)
.setAutofetch(true)
.setId(id)
.findUnique();
}
private void useOrderDate() {
Order order = findById(3);
order.getStatus();
order.getShipDate();
}
private void useOrderDateCustomerName() {
Order order = findById(3);
order.getOrderDate();
order.getCustomer().getName();
}
private void useLots() {
Order order = findById(3);
order.getOrderDate();
order.getShipDate();
// order.setShipDate(new Date(System.currentTimeMillis()));
Customer customer = order.getCustomer();
customer.getName();
Address shippingAddress = customer.getShippingAddress();
if (shippingAddress != null) {
shippingAddress.getLine1();
shippingAddress.getCity();
}
}
private static void collectUsage() {
AdminAutofetch adminAutofetch = Ebean.getServer(null).getAdminAutofetch();
adminAutofetch.collectUsageViaGC();
}
}
package com.avaje.tests.query.autotune;
import com.avaje.ebean.BaseTestCase;
import com.avaje.ebean.Ebean;
import com.avaje.tests.model.basic.Address;
import com.avaje.tests.model.basic.Customer;
import com.avaje.tests.model.basic.Order;
import com.avaje.tests.model.basic.OrderDetail;
import com.avaje.tests.model.basic.ResetBasicData;
import org.junit.Test;
import java.util.List;
public class TestAutoTuneProfiling extends BaseTestCase {
@Test
public void test() {
ResetBasicData.reset();
for (int i = 0; i < 1; i++) {
execute();
}
collectUsage();
}
private void execute() {
useOrderDate();
useOrderDateCustomerName();
useLots();
useLotUntuned();
}
private Order findById(long id) {
return Ebean.find(Order.class)
.select("status, orderDate, shipDate")
.setId(id)
.findUnique();
}
private void useOrderDate() {
Order order = findById(3);
order.getStatus();
order.getShipDate();
}
private void useOrderDateCustomerName() {
Order order = findById(3);
order.getOrderDate();
order.getCustomer().getName();
}
private void useLots() {
Order order = findById(3);
order.getOrderDate();
order.getShipDate();
// order.setShipDate(new Date(System.currentTimeMillis()));
Customer customer = order.getCustomer();
customer.getName();
Address shippingAddress = customer.getShippingAddress();
if (shippingAddress != null) {
shippingAddress.getLine1();
shippingAddress.getCity();
}
}
private void useLotUntuned() {
Order order = findById(3);
List<OrderDetail> details = order.getDetails();
for (OrderDetail detail : details) {
detail.getProduct().getName();
detail.getOrderQty();
detail.getShipQty();
detail.getUnitPrice();
}
Customer customer = order.getCustomer();
customer.getName();
Address billingAddress = customer.getBillingAddress();
if (billingAddress != null) {
billingAddress.getCity();
billingAddress.getLine1();
billingAddress.getLine2();
}
}
private static void collectUsage() {
Ebean.getDefaultServer().getAutoTune().collectProfiling();
}
}
+8 -9
View File
@@ -11,15 +11,14 @@
ebean.encryptKeyManager=com.avaje.tests.basic.encrypt.BasicEncyptKeyManager
ebean.autofetch.querytuning=true
ebean.autofetch.profiling=true
ebean.autofetch.implicitmode=default_off
#ebean.autofetch.implicitmode=default_onifempty
ebean.autofetch.profiling.min=1
ebean.autofetch.profiling.base=10
#ebean.autofetch.profiling.rate=0.05
ebean.autofetch.garbageCollectionOnShutdown=true
ebean.autofetch.traceUsageCollection=true
ebean.autotune.querytuning=true
ebean.autotune.profiling=true
#ebean.autoTune.implicitmode=default_off
#ebean.autotune.implicitmode=default_onifempty
#ebean.autofetch.profiling.min=1
#ebean.autofetch.profiling.base=10
##ebean.autofetch.profiling.rate=0.05
#ebean.autofetch.garbageCollectionOnShutdown=true
ebean.ddl.generate=true