From eb292e017e660346a404cc0abfdfa91c7fd2edee Mon Sep 17 00:00:00 2001 From: rob bygrave Date: Wed, 20 Nov 2019 20:15:32 +1300 Subject: [PATCH] #1868 - Change the Metric "sqlHash" into a hash of the SQL + name + location --- .../java/io/ebean/meta/MetaInfoManager.java | 5 + .../io/ebean/meta/MetaOrmQueryMetric.java | 5 - .../java/io/ebean/meta/MetaQueryMetric.java | 5 + .../java/io/ebean/meta/MetaQueryPlan.java | 4 +- .../java/io/ebean/meta/MetaTimedMetric.java | 2 +- src/main/java/io/ebean/meta/MetricData.java | 98 +++++++++++++++++++ .../server/core/DefaultMetaInfoManager.java | 6 ++ .../server/core/DumpMetrics.java | 3 +- .../server/core/DumpMetricsData.java | 95 ++++++++++++++++++ .../server/core/DumpMetricsJson.java | 2 +- .../server/profile/DQueryPlanMeta.java | 15 +++ .../server/profile/DQueryPlanMetric.java | 5 + .../server/query/CQueryBindCapture.java | 2 +- .../server/query/CQueryPlan.java | 19 ++-- .../server/query/CQueryPlanStats.java | 4 +- .../server/query/DQueryPlanOutput.java | 12 +-- .../io/ebeaninternal/server/util/Md5.java | 1 - .../tests/model/array/TestDbArray_basic.java | 2 +- .../query/finder/TestCustomerFinder.java | 2 +- 19 files changed, 256 insertions(+), 31 deletions(-) create mode 100644 src/main/java/io/ebean/meta/MetricData.java create mode 100644 src/main/java/io/ebeaninternal/server/core/DumpMetricsData.java diff --git a/src/main/java/io/ebean/meta/MetaInfoManager.java b/src/main/java/io/ebean/meta/MetaInfoManager.java index 8b38e9786..285f25135 100644 --- a/src/main/java/io/ebean/meta/MetaInfoManager.java +++ b/src/main/java/io/ebean/meta/MetaInfoManager.java @@ -28,6 +28,11 @@ public interface MetaInfoManager { */ ServerMetricsAsJson collectMetricsAsJson(); + /** + * Return the metrics as a list of MetricData. + */ + List collectMetricsAsData(); + /** * Collect query plans. */ diff --git a/src/main/java/io/ebean/meta/MetaOrmQueryMetric.java b/src/main/java/io/ebean/meta/MetaOrmQueryMetric.java index 9bce29409..6a817b037 100644 --- a/src/main/java/io/ebean/meta/MetaOrmQueryMetric.java +++ b/src/main/java/io/ebean/meta/MetaOrmQueryMetric.java @@ -12,11 +12,6 @@ public interface MetaOrmQueryMetric extends MetaQueryMetric { */ boolean isAutoTuned(); - /** - * Return the hash of the sql. - */ - String getSqlHash(); - /** * Return the time of the last query executed using this plan. */ diff --git a/src/main/java/io/ebean/meta/MetaQueryMetric.java b/src/main/java/io/ebean/meta/MetaQueryMetric.java index c59142dc6..514b700dc 100644 --- a/src/main/java/io/ebean/meta/MetaQueryMetric.java +++ b/src/main/java/io/ebean/meta/MetaQueryMetric.java @@ -20,4 +20,9 @@ public interface MetaQueryMetric extends MetaTimedMetric { */ String getSql(); + /** + * Return the hash of the plan. + */ + String getHash(); + } diff --git a/src/main/java/io/ebean/meta/MetaQueryPlan.java b/src/main/java/io/ebean/meta/MetaQueryPlan.java index e4c92c937..fb2b6d095 100644 --- a/src/main/java/io/ebean/meta/MetaQueryPlan.java +++ b/src/main/java/io/ebean/meta/MetaQueryPlan.java @@ -28,9 +28,9 @@ public interface MetaQueryPlan { String getSql(); /** - * Return the hash of the sql. + * Return the hash of the plan. */ - String getSqlHash(); + String getHash(); /** * Return a description of the bind values. diff --git a/src/main/java/io/ebean/meta/MetaTimedMetric.java b/src/main/java/io/ebean/meta/MetaTimedMetric.java index eebdb69d2..0a9bef84e 100644 --- a/src/main/java/io/ebean/meta/MetaTimedMetric.java +++ b/src/main/java/io/ebean/meta/MetaTimedMetric.java @@ -38,7 +38,7 @@ public interface MetaTimedMetric extends MetaMetric { /** * Return the total beans or rows processed or loaded. - * + *

* This will be 0 if the metric isn't a query plan (like transaction execution statistics). */ long getBeanCount(); diff --git a/src/main/java/io/ebean/meta/MetricData.java b/src/main/java/io/ebean/meta/MetricData.java new file mode 100644 index 000000000..ab67aae33 --- /dev/null +++ b/src/main/java/io/ebean/meta/MetricData.java @@ -0,0 +1,98 @@ +package io.ebean.meta; + +/** + * An individual metric. + */ +public class MetricData { + + private String name; + private String type; + private String hash; + private String loc; + private String sql; + + private Long count; + private Long mean; + private Long max; + private Long total; + + public MetricData(String name, String type) { + this.name = name; + this.type = type; + } + + public MetricData() { + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getType() { + return type; + } + + public void setType(String type) { + this.type = type; + } + + public String getHash() { + return hash; + } + + public void setHash(String hash) { + this.hash = hash; + } + + public String getLoc() { + return loc; + } + + public void setLoc(String loc) { + this.loc = loc; + } + + public String getSql() { + return sql; + } + + public void setSql(String sql) { + this.sql = sql; + } + + public Long getCount() { + return count; + } + + public void setCount(Long count) { + this.count = count; + } + + public Long getMean() { + return mean; + } + + public void setMean(Long mean) { + this.mean = mean; + } + + public Long getMax() { + return max; + } + + public void setMax(Long max) { + this.max = max; + } + + public Long getTotal() { + return total; + } + + public void setTotal(Long total) { + this.total = total; + } +} diff --git a/src/main/java/io/ebeaninternal/server/core/DefaultMetaInfoManager.java b/src/main/java/io/ebeaninternal/server/core/DefaultMetaInfoManager.java index cc7f101e0..cee385241 100644 --- a/src/main/java/io/ebeaninternal/server/core/DefaultMetaInfoManager.java +++ b/src/main/java/io/ebeaninternal/server/core/DefaultMetaInfoManager.java @@ -9,6 +9,7 @@ import io.ebean.meta.MetaOrmQueryNode; import io.ebean.meta.MetaQueryMetric; import io.ebean.meta.MetaQueryPlan; import io.ebean.meta.MetaTimedMetric; +import io.ebean.meta.MetricData; import io.ebean.meta.MetricVisitor; import io.ebean.meta.QueryPlanRequest; import io.ebean.meta.ServerMetrics; @@ -48,6 +49,11 @@ public class DefaultMetaInfoManager implements MetaInfoManager { return new DumpMetricsJson(server); } + @Override + public List collectMetricsAsData() { + return new DumpMetricsData(server).data(); + } + @Override public BasicMetricVisitor visitBasic() { BasicMetricVisitor basic = new BasicMetricVisitor(); diff --git a/src/main/java/io/ebeaninternal/server/core/DumpMetrics.java b/src/main/java/io/ebeaninternal/server/core/DumpMetrics.java index b035e20f5..55dc9e6ca 100644 --- a/src/main/java/io/ebeaninternal/server/core/DumpMetrics.java +++ b/src/main/java/io/ebeaninternal/server/core/DumpMetrics.java @@ -1,6 +1,5 @@ package io.ebeaninternal.server.core; -import io.ebean.ProfileLocation; import io.ebean.meta.MetaCountMetric; import io.ebean.meta.MetaOrmQueryMetric; import io.ebean.meta.MetaQueryMetric; @@ -131,7 +130,7 @@ class DumpMetrics { appendQueryName(metric, sb); appendCounters(metric, sb); if (dumpHash) { - sb.append("\n hash:").append(metric.getSqlHash()); + sb.append("\n hash:").append(metric.getHash()); } appendProfileAndSql(metric, sb); out(sb.toString()); diff --git a/src/main/java/io/ebeaninternal/server/core/DumpMetricsData.java b/src/main/java/io/ebeaninternal/server/core/DumpMetricsData.java new file mode 100644 index 000000000..bd4d3ecae --- /dev/null +++ b/src/main/java/io/ebeaninternal/server/core/DumpMetricsData.java @@ -0,0 +1,95 @@ +package io.ebeaninternal.server.core; + +import io.ebean.Database; +import io.ebean.meta.MetaCountMetric; +import io.ebean.meta.MetaMetric; +import io.ebean.meta.MetaOrmQueryMetric; +import io.ebean.meta.MetaQueryMetric; +import io.ebean.meta.MetaTimedMetric; +import io.ebean.meta.MetricData; +import io.ebean.meta.ServerMetrics; + +import java.util.ArrayList; +import java.util.List; + +/** + * Dump the metrics into a list of MetricData. + */ +class DumpMetricsData { + + private final Database database; + + private final List list = new ArrayList<>(); + + DumpMetricsData(Database database) { + this.database = database; + } + + List data() { + collect(database.getMetaInfoManager().collectMetrics()); + return list; + } + + private void collect(ServerMetrics serverMetrics) { + + final List timedMetrics = serverMetrics.getTimedMetrics(); + final List countMetrics = serverMetrics.getCountMetrics(); + final List ormQueryMetrics = serverMetrics.getOrmQueryMetrics(); + final List dtoQueryMetrics = serverMetrics.getDtoQueryMetrics(); + + for (MetaTimedMetric metric : timedMetrics) { + add(metric); + } + for (MetaCountMetric metric : countMetrics) { + addCount(metric); + } + for (MetaOrmQueryMetric metric : ormQueryMetrics) { + addQuery(metric); + } + for (MetaQueryMetric metric : dtoQueryMetrics) { + addDtoQuery(metric); + } + } + + private MetricData create(MetaMetric metric) { + MetricData data = new MetricData(metric.getName(), metric.getMetricType().name()); + list.add(data); + return data; + } + + private void add(MetaTimedMetric metric) { + final MetricData data = create(metric); + appendCounters(data, metric); + data.setLoc(metric.getLocation()); + } + + private void addCount(MetaCountMetric metric) { + final MetricData data = create(metric); + data.setCount(metric.getCount()); + } + + private void addQuery(MetaOrmQueryMetric metric) { + final MetricData data = create(metric); + appendCounters(data, metric); + appendLocationAndSql(data, metric); + data.setHash(metric.getHash()); + } + + private void addDtoQuery(MetaQueryMetric metric) { + final MetricData data = create(metric); + appendCounters(data, metric); + appendLocationAndSql(data, metric); + } + + private void appendLocationAndSql(MetricData data, MetaQueryMetric metric) { + data.setLoc(metric.getLocation()); + data.setSql(metric.getSql()); + } + + private void appendCounters(MetricData data, MetaTimedMetric timedMetric) { + data.setCount(timedMetric.getCount()); + data.setTotal(timedMetric.getTotal()); + data.setMean(timedMetric.getMean()); + data.setMax(timedMetric.getMax()); + } +} diff --git a/src/main/java/io/ebeaninternal/server/core/DumpMetricsJson.java b/src/main/java/io/ebeaninternal/server/core/DumpMetricsJson.java index 90bad3c95..d0ed03c12 100644 --- a/src/main/java/io/ebeaninternal/server/core/DumpMetricsJson.java +++ b/src/main/java/io/ebeaninternal/server/core/DumpMetricsJson.java @@ -192,7 +192,7 @@ class DumpMetricsJson implements ServerMetricsAsJson { appendCounters(metric); if (withHash) { key("hash"); - val(metric.getSqlHash()); + val(metric.getHash()); } appendLocationAndSql(metric); metricEnd(); diff --git a/src/main/java/io/ebeaninternal/server/profile/DQueryPlanMeta.java b/src/main/java/io/ebeaninternal/server/profile/DQueryPlanMeta.java index 9f52dd026..16d3e69cc 100644 --- a/src/main/java/io/ebeaninternal/server/profile/DQueryPlanMeta.java +++ b/src/main/java/io/ebeaninternal/server/profile/DQueryPlanMeta.java @@ -1,6 +1,7 @@ package io.ebeaninternal.server.profile; import io.ebean.ProfileLocation; +import io.ebeaninternal.server.util.Md5; class DQueryPlanMeta { @@ -9,6 +10,7 @@ class DQueryPlanMeta { private final ProfileLocation profileLocation; private final String name; private final String sql; + private final String hash; DQueryPlanMeta(Class type, String label, ProfileLocation profileLocation, String sql) { this.type = type; @@ -20,12 +22,25 @@ class DQueryPlanMeta { name += "_" + label; } this.name = name; + this.hash = initHash(); + } + + private String initHash() { + StringBuilder sb = new StringBuilder(sql).append("|").append(name); + if (profileLocation != null) { + sb.append("|").append(profileLocation.location()); + } + return Md5.hash(sb.toString()); } public Class getType() { return type; } + public String getHash() { + return hash; + } + public String getName() { return name; } diff --git a/src/main/java/io/ebeaninternal/server/profile/DQueryPlanMetric.java b/src/main/java/io/ebeaninternal/server/profile/DQueryPlanMetric.java index 0adbd10cb..7ba93fe33 100644 --- a/src/main/java/io/ebeaninternal/server/profile/DQueryPlanMetric.java +++ b/src/main/java/io/ebeaninternal/server/profile/DQueryPlanMetric.java @@ -55,6 +55,11 @@ class DQueryPlanMetric implements QueryPlanMetric { return meta.getType(); } + @Override + public String getHash() { + return meta.getHash(); + } + @Override public String getLabel() { return meta.getLabel(); diff --git a/src/main/java/io/ebeaninternal/server/query/CQueryBindCapture.java b/src/main/java/io/ebeaninternal/server/query/CQueryBindCapture.java index dceb4fc52..4ba444012 100644 --- a/src/main/java/io/ebeaninternal/server/query/CQueryBindCapture.java +++ b/src/main/java/io/ebeaninternal/server/query/CQueryBindCapture.java @@ -60,7 +60,7 @@ class CQueryBindCapture { DQueryPlanOutput queryPlan = planLogger.logQueryPlan(request.getConnection(), cQueryPlan, last); if (queryPlan != null) { - queryPlan.with(queryTimeMicros, captureCount, cQueryPlan.getSqlHash()); + queryPlan.with(queryTimeMicros, captureCount, cQueryPlan.getHash()); request.process(queryPlan); } } diff --git a/src/main/java/io/ebeaninternal/server/query/CQueryPlan.java b/src/main/java/io/ebeaninternal/server/query/CQueryPlan.java index 63f0cfd52..25e10c9ca 100644 --- a/src/main/java/io/ebeaninternal/server/query/CQueryPlan.java +++ b/src/main/java/io/ebeaninternal/server/query/CQueryPlan.java @@ -73,7 +73,7 @@ public class CQueryPlan { private final boolean rowNumberIncluded; private final String sql; - private final String sqlHash; + private final String hash; private final String logWhereSql; @@ -118,7 +118,6 @@ public class CQueryPlan { this.autoTuned = query.isAutoTuned(); this.asOfTableCount = query.getAsOfTableCount(); this.sql = sqlRes.getSql(); - this.sqlHash = md5Hash(sql); this.rowNumberIncluded = sqlRes.isIncludesRowNumberColumn(); this.sqlTree = sqlTree; this.rawSql = rawSql; @@ -127,6 +126,7 @@ public class CQueryPlan { this.stats = new CQueryPlanStats(this, server.isCollectQueryOrigins()); this.dependentTables = sqlTree.dependentTables(); this.bindCapture = initBindCapture(server.getServerConfig(), query); + this.hash = md5Hash(); } /** @@ -146,7 +146,6 @@ public class CQueryPlan { this.autoTuned = false; this.asOfTableCount = 0; this.sql = sql; - this.sqlHash = md5Hash(sql); this.sqlTree = sqlTree; this.rawSql = false; this.rowNumberIncluded = rowNumberIncluded; @@ -155,6 +154,7 @@ public class CQueryPlan { this.stats = new CQueryPlanStats(this, server.isCollectQueryOrigins()); this.dependentTables = sqlTree.dependentTables(); this.bindCapture = initBindCapture(server.getServerConfig(), query); + this.hash = md5Hash(); } private String deriveName(String label, SpiQuery.Type type) { @@ -260,23 +260,26 @@ public class CQueryPlan { private String calcAuditQueryKey() { // rawSql needs to include the MD5 hash of the sql - return rawSql ? planKey.getPartialKey() + "_" + sqlHash : planKey.getPartialKey(); + return rawSql ? planKey.getPartialKey() + "_" + hash : planKey.getPartialKey(); } /** * Return the MD5 hash of the sql. */ - private String md5Hash(String sql) { + private String md5Hash() { + StringBuilder sb = new StringBuilder(sql) + .append("|").append(name) + .append("|").append(location); try { - return Md5.hash(sql); + return Md5.hash(sb.toString()); } catch (Exception e) { logger.error("Failed to MD5 hash the query", e); return "error"; } } - String getSqlHash() { - return sqlHash; + String getHash() { + return hash; } public String getSql() { diff --git a/src/main/java/io/ebeaninternal/server/query/CQueryPlanStats.java b/src/main/java/io/ebeaninternal/server/query/CQueryPlanStats.java index 8bf9b06fc..8528b8612 100644 --- a/src/main/java/io/ebeaninternal/server/query/CQueryPlanStats.java +++ b/src/main/java/io/ebeaninternal/server/query/CQueryPlanStats.java @@ -233,8 +233,8 @@ public final class CQueryPlanStats { } @Override - public String getSqlHash() { - return queryPlan.getSqlHash(); + public String getHash() { + return queryPlan.getHash(); } @Override diff --git a/src/main/java/io/ebeaninternal/server/query/DQueryPlanOutput.java b/src/main/java/io/ebeaninternal/server/query/DQueryPlanOutput.java index fc4aa0b0a..f8d734b70 100644 --- a/src/main/java/io/ebeaninternal/server/query/DQueryPlanOutput.java +++ b/src/main/java/io/ebeaninternal/server/query/DQueryPlanOutput.java @@ -16,7 +16,7 @@ class DQueryPlanOutput implements MetaQueryPlan { private final String bind; private final String plan; - private String sqlHash; + private String hash; private long queryTimeMicros; private long captureCount; @@ -30,8 +30,8 @@ class DQueryPlanOutput implements MetaQueryPlan { } @Override - public String getSqlHash() { - return sqlHash; + public String getHash() { + return hash; } /** @@ -98,15 +98,15 @@ class DQueryPlanOutput implements MetaQueryPlan { @Override public String toString() { - return " BeanType:" + ((beanType == null) ? "" : beanType.getSimpleName()) + " planHash:" + sqlHash + " label:" + label + " queryTimeMicros:" + queryTimeMicros + " captureCount:" + captureCount + "\n SQL:" + sql + "\nBIND:" + bind + "\nPLAN:" + plan; + return " BeanType:" + ((beanType == null) ? "" : beanType.getSimpleName()) + " planHash:" + hash + " label:" + label + " queryTimeMicros:" + queryTimeMicros + " captureCount:" + captureCount + "\n SQL:" + sql + "\nBIND:" + bind + "\nPLAN:" + plan; } /** * Additionally set the query execution time and the number of bind captures. */ - void with(long queryTimeMicros, long captureCount, String sqlHash) { + void with(long queryTimeMicros, long captureCount, String hash) { this.queryTimeMicros = queryTimeMicros; this.captureCount = captureCount; - this.sqlHash = sqlHash; + this.hash = hash; } } diff --git a/src/main/java/io/ebeaninternal/server/util/Md5.java b/src/main/java/io/ebeaninternal/server/util/Md5.java index ff8983e1e..1c848f55d 100644 --- a/src/main/java/io/ebeaninternal/server/util/Md5.java +++ b/src/main/java/io/ebeaninternal/server/util/Md5.java @@ -9,7 +9,6 @@ public final class Md5 { * Return the MD5 hash of the underlying sql. */ public static String hash(String content) { - try { MessageDigest md = MessageDigest.getInstance("MD5"); return digestToHex(md.digest(content.getBytes(StandardCharsets.UTF_8))); diff --git a/src/test/java/org/tests/model/array/TestDbArray_basic.java b/src/test/java/org/tests/model/array/TestDbArray_basic.java index cadbd6561..e92527c80 100644 --- a/src/test/java/org/tests/model/array/TestDbArray_basic.java +++ b/src/test/java/org/tests/model/array/TestDbArray_basic.java @@ -32,7 +32,7 @@ public class TestDbArray_basic extends BaseTestCase { public void insert() throws SQLException { bean.setName("some stuff"); - assertThat(bean.getStatuses()).isNotNull(); + assertThat(bean.getStatuses()).as("DbArray is auto initialised").isNotNull(); List phNumbers = bean.getPhoneNumbers(); phNumbers.add("4321"); diff --git a/src/test/java/org/tests/query/finder/TestCustomerFinder.java b/src/test/java/org/tests/query/finder/TestCustomerFinder.java index 4aa9fcc3b..e9a82abbc 100644 --- a/src/test/java/org/tests/query/finder/TestCustomerFinder.java +++ b/src/test/java/org/tests/query/finder/TestCustomerFinder.java @@ -182,7 +182,7 @@ public class TestCustomerFinder extends BaseTestCase { assertThat(metricsJson).contains("\"name\":\"txn.main\", \"type\":\"TXN\""); assertThat(metricsJson).contains("\"name\":\"Customer.findList\""); assertThat(metricsJson).contains("\"loc\":\"CustomerFinder.byNameStatus(CustomerFinder.java:44)\""); - assertThat(metricsJson).contains("\"hash\":\"4d648ce0542aedfb042ad68746342730\""); + assertThat(metricsJson).contains("\"hash\":\"f089744556e1677b57c7c6ef73f65da8\""); assertThat(metricsJson).contains("\"sql\":\"select t0.id, t0.status,"); }