Merge pull request #2778 from ebean-orm/feature/metric-naming-convention

Support a naming convention function that can be applied to metrics names for reporting
This commit is contained in:
Rob Bygrave
2022-08-09 11:10:51 +12:00
committed by GitHub
19 changed files with 314 additions and 50 deletions
@@ -16,6 +16,7 @@ import io.ebean.event.changelog.ChangeLogPrepare;
import io.ebean.event.changelog.ChangeLogRegister;
import io.ebean.event.readaudit.ReadAuditLogger;
import io.ebean.event.readaudit.ReadAuditPrepare;
import io.ebean.meta.MetricNamingMatch;
import io.ebean.util.StringHelper;
import javax.persistence.EnumType;
@@ -24,6 +25,7 @@ import java.time.Clock;
import java.time.ZonedDateTime;
import java.time.format.DateTimeFormatter;
import java.util.*;
import java.util.function.Function;
/**
* The configuration used for creating a Database.
@@ -546,6 +548,8 @@ public class DatabaseConfig {
private String dumpMetricsOptions;
private Function<String, String> metricNaming = MetricNamingMatch.INSTANCE;
/**
* Construct a Database Configuration for programmatically creating an Database.
*/
@@ -3396,6 +3400,20 @@ public class DatabaseConfig {
this.loadModuleInfo = loadModuleInfo;
}
/**
* Return the naming convention to apply to metrics names.
*/
public Function<String, String> getMetricNaming() {
return metricNaming;
}
/**
* Set the naming convention to apply to metrics names.
*/
public void setMetricNaming(Function<String, String> metricNaming) {
this.metricNaming = metricNaming;
}
public enum UuidVersion {
VERSION4,
VERSION1,
@@ -2,6 +2,7 @@ package io.ebean.meta;
import java.util.ArrayList;
import java.util.List;
import java.util.function.Function;
/**
* A simple MetricVisitor that can collect the desired metrics into lists.
@@ -9,27 +10,29 @@ import java.util.List;
public class BasicMetricVisitor extends AbstractMetricVisitor implements ServerMetrics {
private final String name;
private final Function<String,String> naming;
private final List<MetaTimedMetric> timed = new ArrayList<>();
private final List<MetaQueryMetric> query = new ArrayList<>();
private final List<MetaCountMetric> count = new ArrayList<>();
public BasicMetricVisitor() {
this("db");
this("db", MetricNamingMatch.INSTANCE);
}
/**
* Construct to reset and collect everything.
*/
public BasicMetricVisitor(String name) {
this(name, true, true, true, true);
public BasicMetricVisitor(String name, Function<String,String> naming) {
this(name, naming, true, true, true, true);
}
/**
* Construct specifying reset and what to collect.
*/
public BasicMetricVisitor(String name, boolean reset, boolean collectTransactionMetrics, boolean collectQueryMetrics, boolean collectL2Metrics) {
public BasicMetricVisitor(String name, Function<String,String> naming, boolean reset, boolean collectTransactionMetrics, boolean collectQueryMetrics, boolean collectL2Metrics) {
super(reset, collectTransactionMetrics, collectQueryMetrics, collectL2Metrics);
this.name = name;
this.naming = naming;
}
@Override
@@ -37,6 +40,11 @@ public class BasicMetricVisitor extends AbstractMetricVisitor implements ServerM
return name;
}
@Override
public Function<String, String> namingConvention() {
return naming;
}
@Override
public ServerMetricsAsJson asJson() {
return new MetricsAsJson(this);
@@ -0,0 +1,16 @@
package io.ebean.meta;
import java.util.function.Function;
/**
* Metric naming convention that is exact match.
*/
public final class MetricNamingMatch implements Function<String, String> {
public static final Function<String, String> INSTANCE = new MetricNamingMatch();
@Override
public String apply(String name) {
return name;
}
}
@@ -1,10 +1,17 @@
package io.ebean.meta;
import java.util.function.Function;
/**
* Defines visitor to read and report the transaction and query metrics.
*/
public interface MetricVisitor {
/**
* Return the naming convention that should be applied to the reported metric names.
*/
Function<String, String> namingConvention();
/**
* Return true if the metrics should be reset.
*/
@@ -1,20 +1,9 @@
package io.ebeaninternal.server.core;
import io.ebean.meta.AbstractMetricVisitor;
import io.ebean.meta.BasicMetricVisitor;
import io.ebean.meta.MetaCountMetric;
import io.ebean.meta.MetaInfoManager;
import io.ebean.meta.MetaQueryMetric;
import io.ebean.meta.MetaQueryPlan;
import io.ebean.meta.MetaTimedMetric;
import io.ebean.meta.MetricData;
import io.ebean.meta.MetricVisitor;
import io.ebean.meta.QueryPlanInit;
import io.ebean.meta.QueryPlanRequest;
import io.ebean.meta.ServerMetrics;
import io.ebean.meta.ServerMetricsAsJson;
import io.ebean.meta.*;
import java.util.List;
import java.util.function.Function;
/**
* DefaultServer based implementation of MetaInfoManager.
@@ -22,9 +11,11 @@ import java.util.List;
final class DefaultMetaInfoManager implements MetaInfoManager {
private final DefaultServer server;
private final Function<String, String> naming;
DefaultMetaInfoManager(DefaultServer server) {
DefaultMetaInfoManager(DefaultServer server, Function<String, String> naming) {
this.server = server;
this.naming = naming;
}
@Override
@@ -49,7 +40,7 @@ final class DefaultMetaInfoManager implements MetaInfoManager {
@Override
public BasicMetricVisitor visitBasic() {
BasicMetricVisitor basic = new BasicMetricVisitor(server.name());
BasicMetricVisitor basic = new BasicMetricVisitor(server.name(), naming);
visitMetrics(basic);
return basic;
}
@@ -68,6 +59,11 @@ final class DefaultMetaInfoManager implements MetaInfoManager {
super(true, true, true, true);
}
@Override
public Function<String, String> namingConvention() {
return MetricNamingMatch.INSTANCE;
}
@Override
public void visitTimed(MetaTimedMetric metric) {
// ignore
@@ -213,7 +213,7 @@ public final class DefaultServer implements SpiServer, SpiEbeanServer {
this.transactionManager = config.createTransactionManager(this, docStoreComponents.updateProcessor());
this.documentStore = docStoreComponents.documentStore();
this.queryPlanManager = config.initQueryPlanManager(transactionManager);
this.metaInfoManager = new DefaultMetaInfoManager(this);
this.metaInfoManager = new DefaultMetaInfoManager(this, this.config.getMetricNaming());
this.serverPlugins = config.getPlugins();
this.ddlGenerator = config.initDdlGenerator(this);
this.scriptRunner = new DScriptRunner(this);
@@ -1482,7 +1482,7 @@ public class BeanDescriptor<T> implements BeanType<T>, STreeType, SpiBeanType {
iudMetrics.visit(visitor);
for (CQueryPlan queryPlan : queryPlanCache.values()) {
if (!queryPlan.isEmptyStats()) {
visitor.visitQuery(queryPlan.getSnapshot(visitor.reset()));
visitor.visitQuery(queryPlan.visit(visitor));
}
}
}
@@ -13,6 +13,7 @@ final class DCountMetric implements CountMetric {
private final String name;
private final LongAdder count = new LongAdder();
private String reportName;
DCountMetric(String name) {
this.name = name;
@@ -50,10 +51,17 @@ final class DCountMetric implements CountMetric {
public void visit(MetricVisitor visitor) {
long val = visitor.reset() ? count.sumThenReset() : count.sum();
if (val > 0) {
final String name = reportName != null ? reportName : reportName(visitor);
visitor.visitCount(new DCountMetricStats(name, val));
}
}
String reportName(MetricVisitor visitor) {
final String tmp = visitor.namingConvention().apply(name);
this.reportName = tmp;
return tmp;
}
private static class DCountMetricStats implements CountMetricStats {
private final String name;
@@ -11,6 +11,7 @@ final class DQueryPlanMetric implements QueryPlanMetric {
private final DQueryPlanMeta meta;
private final DTimedMetric metric;
private boolean collected;
private String reportName;
DQueryPlanMetric(DQueryPlanMeta meta, DTimedMetric metric) {
this.meta = meta;
@@ -21,11 +22,18 @@ final class DQueryPlanMetric implements QueryPlanMetric {
public void visit(MetricVisitor visitor) {
TimedMetricStats stats = metric.collect(visitor.reset());
if (stats != null) {
visitor.visitQuery(new Stats(meta, stats, collected));
String name = reportName != null ? reportName : reportName(visitor);
visitor.visitQuery(new Stats(name, meta, stats, collected));
collected = true;
}
}
String reportName(MetricVisitor visitor) {
final String tmp = visitor.namingConvention().apply(meta.getName());
this.reportName = tmp;
return tmp;
}
@Override
public TimedMetric metric() {
return metric;
@@ -33,11 +41,13 @@ final class DQueryPlanMetric implements QueryPlanMetric {
private static class Stats implements MetaQueryMetric {
private final String name;
private final DQueryPlanMeta meta;
private final TimedMetricStats stats;
private final boolean collected;
private Stats(DQueryPlanMeta meta, TimedMetricStats stats, boolean collected) {
private Stats(String name, DQueryPlanMeta meta, TimedMetricStats stats, boolean collected) {
this.name = name;
this.meta = meta;
this.stats = stats;
this.collected = collected;
@@ -75,7 +85,7 @@ final class DQueryPlanMetric implements QueryPlanMetric {
@Override
public String name() {
return meta.getName();
return name;
}
@Override
@@ -19,6 +19,7 @@ final class DTimedMetric implements TimedMetric {
private final LongAdder total = new LongAdder();
private final LongAccumulator max = new LongAccumulator(Math::max, Long.MIN_VALUE);
private boolean collected;
private String reportName;
DTimedMetric(String name) {
this.name = name;
@@ -40,9 +41,6 @@ final class DTimedMetric implements TimedMetric {
add((System.nanoTime() - startNanos) / 1000L);
}
/**
* Add a value. Usually the value is Time or Bytes etc.
*/
@Override
public void add(long value) {
count.increment();
@@ -55,9 +53,6 @@ final class DTimedMetric implements TimedMetric {
return count.sum() == 0;
}
/**
* Reset all the internal counters and start time.
*/
@Override
public void reset() {
max.reset();
@@ -67,30 +62,42 @@ final class DTimedMetric implements TimedMetric {
@Override
public void visit(MetricVisitor visitor) {
DTimeMetricStats metric = collect(visitor.reset());
if (metric != null) {
visitor.visitTimed(metric);
final long countSum = visitor.reset() ? count.sumThenReset() : count.sum();
if (countSum > 0) {
final String name = reportName != null ? reportName : reportName(visitor);
visitor.visitTimed(stats(visitor.reset(), name, countSum));
}
}
@Override
public DTimeMetricStats collect(boolean reset) {
return (count.sum() == 0) ? null : getStatistics(reset);
final long countSum = reset ? count.sumThenReset() : count.sum();
if (countSum == 0) {
return null;
} else {
return stats(reset, name, countSum);
}
}
/**
* Return the current statistics resetting the internal values if reset is true.
*/
private DTimeMetricStats getStatistics(boolean reset) {
private DTimeMetricStats stats(boolean reset, String name, long countSum) {
try {
if (reset) {
return new DTimeMetricStats(name, collected, count.sumThenReset(), total.sumThenReset(), max.getThenReset());
return new DTimeMetricStats(name, collected, countSum, total.sumThenReset(), max.getThenReset());
} else {
return new DTimeMetricStats(name, collected, count.sum(), total.sum(), max.get());
return new DTimeMetricStats(name, collected, countSum, total.sum(), max.get());
}
} finally {
collected = true;
}
}
String reportName(MetricVisitor visitor) {
final String tmp = visitor.namingConvention().apply(name);
this.reportName = tmp;
return tmp;
}
}
@@ -13,6 +13,7 @@ final class DTimedProfileLocation extends DProfileLocation implements TimedProfi
private final TimedMetric timedMetric;
private final boolean overrideMetricName;
private String fullName;
private String reportName;
DTimedProfileLocation(int lineNumber, String label, TimedMetric timedMetric) {
super(lineNumber);
@@ -47,11 +48,17 @@ final class DTimedProfileLocation extends DProfileLocation implements TimedProfi
public void visit(MetricVisitor visitor) {
TimedMetricStats collect = timedMetric.collect(visitor.reset());
if (collect != null) {
if (overrideMetricName) {
collect.setName(fullName);
}
final String name = reportName != null ? reportName : reportName(visitor, collect.name());
collect.setName(name);
collect.setLocation(location());
visitor.visitTimed(collect);
}
}
private String reportName(MetricVisitor visitor, String name) {
final String defaultName = overrideMetricName ? fullName : name;
final String tmp = visitor.namingConvention().apply(defaultName);
this.reportName = tmp;
return tmp;
}
}
@@ -4,6 +4,7 @@ import io.ebean.ProfileLocation;
import io.ebean.config.dbplatform.SqlLimitResponse;
import io.ebean.core.type.DataReader;
import io.ebean.core.type.ScalarDataReader;
import io.ebean.meta.MetricVisitor;
import io.ebean.metric.MetricFactory;
import io.ebean.metric.TimedMetric;
import io.ebeaninternal.api.*;
@@ -279,8 +280,8 @@ public class CQueryPlan implements SpiQueryPlan {
/**
* Return a copy of the current query statistics.
*/
public final Snapshot getSnapshot(boolean reset) {
return stats.getSnapshot(reset);
public final Snapshot visit(MetricVisitor visitor) {
return stats.visit(visitor);
}
/**
@@ -1,6 +1,7 @@
package io.ebeaninternal.server.query;
import io.ebean.meta.MetaQueryMetric;
import io.ebean.meta.MetricVisitor;
import io.ebean.metric.TimedMetric;
import io.ebean.metric.TimedMetricStats;
@@ -13,6 +14,7 @@ public final class CQueryPlanStats {
private final TimedMetric timedMetric;
private boolean collected;
private long lastQueryTime;
private String reportName;
/**
* Construct for a given query plan.
@@ -55,23 +57,32 @@ public final class CQueryPlanStats {
/**
* Return a Snapshot of the query execution statistics potentially resetting the internal counters.
*/
Snapshot getSnapshot(boolean reset) {
TimedMetricStats collect = timedMetric.collect(reset);
Snapshot snapshot = new Snapshot(collected, queryPlan, collect);
Snapshot visit(MetricVisitor visitor) {
TimedMetricStats collect = timedMetric.collect(visitor.reset());
String name = reportName != null ? reportName : reportName(visitor);
Snapshot snapshot = new Snapshot(name, collected, queryPlan, collect);
collected = true;
return snapshot;
}
String reportName(MetricVisitor visitor) {
final String tmp = visitor.namingConvention().apply(queryPlan.getName());
this.reportName = tmp;
return tmp;
}
/**
* A snapshot of the current statistics for a query plan.
*/
static class Snapshot implements MetaQueryMetric {
private final String name;
private final boolean collected;
private final CQueryPlan queryPlan;
private final TimedMetricStats metrics;
Snapshot(boolean collected, CQueryPlan queryPlan, TimedMetricStats metrics) {
Snapshot(String name, boolean collected, CQueryPlan queryPlan, TimedMetricStats metrics) {
this.name = name;
this.collected = collected;
this.queryPlan = queryPlan;
this.metrics = metrics;
@@ -94,7 +105,7 @@ public final class CQueryPlanStats {
@Override
public String name() {
return queryPlan.getName();
return name;
}
@Override
@@ -1,12 +1,55 @@
package io.ebeaninternal.server.profile;
import io.ebean.meta.BasicMetricVisitor;
import io.ebean.meta.MetaTimedMetric;
import io.ebean.metric.MetricFactory;
import org.junit.jupiter.api.Test;
import java.util.List;
import java.util.function.Function;
import static org.assertj.core.api.Assertions.assertThat;
class BasicProfileLocationTest {
Function<String,String> naming = (String name) -> "prefix[" + name.replace('.', '-') + "]";
@Test
void metricNameFromOverride() {
DTimedProfileLocation loc = new DTimedProfileLocation(12, "", MetricFactory.get().createTimedMetric("a.b.c"));
loc.initWith("foo.label");
loc.add(42);
BasicMetricVisitor visitor = new BasicMetricVisitor("v", naming);
loc.visit(visitor);
List<MetaTimedMetric> result = visitor.timedMetrics();
assertThat(result).hasSize(1);
assertThat(result.get(0).name()).isEqualTo("prefix[txn-named-foo-label]");
assertThat(result.get(0).total()).isEqualTo(42);
loc.add(21);
BasicMetricVisitor visitor2 = new BasicMetricVisitor("v", naming);
loc.visit(visitor2);
List<MetaTimedMetric> result2 = visitor2.timedMetrics();
assertThat(result2).hasSize(1);
assertThat(result2.get(0).name()).isEqualTo("prefix[txn-named-foo-label]");
assertThat(result2.get(0).total()).isEqualTo(21);
}
@Test
void metricNameFromTimed() {
DTimedProfileLocation loc = new DTimedProfileLocation(12, "foo", MetricFactory.get().createTimedMetric("a.b.c"));
loc.add(42);
BasicMetricVisitor visitor = new BasicMetricVisitor("v", naming);
loc.visit(visitor);
List<MetaTimedMetric> result = visitor.timedMetrics();
assertThat(result).hasSize(1);
assertThat(result.get(0).name()).isEqualTo("prefix[a-b-c]");
}
@Test
void obtain() {
DProfileLocation loc = new DTimedProfileLocation(12, "foo", MetricFactory.get().createTimedMetric("junk"));
@@ -0,0 +1,43 @@
package io.ebeaninternal.server.profile;
import io.ebean.meta.BasicMetricVisitor;
import io.ebean.meta.MetaCountMetric;
import org.junit.jupiter.api.Test;
import java.util.List;
import java.util.function.Function;
import static org.assertj.core.api.Assertions.assertThat;
class DCountMetricTest {
Function<String, String> naming = (String name) -> "prefix[" + name.replace('.', '-') + "]";
@Test
void visit() {
DCountMetric counter = new DCountMetric("org.hello");
counter.add(7);
{
BasicMetricVisitor visitor = new BasicMetricVisitor("v", naming);
counter.visit(visitor);
List<MetaCountMetric> result = visitor.countMetrics();
assertThat(result).hasSize(1);
assertThat(result.get(0).name()).isEqualTo("prefix[org-hello]");
assertThat(result.get(0).count()).isEqualTo(7);
}
{
// second collection
counter.add(4);
counter.add(8);
BasicMetricVisitor visitor2 = new BasicMetricVisitor("v", naming);
counter.visit(visitor2);
List<MetaCountMetric> result2 = visitor2.countMetrics();
assertThat(result2).hasSize(1);
assertThat(result2.get(0).name()).isEqualTo("prefix[org-hello]");
assertThat(result2.get(0).count()).isEqualTo(12);
}
}
}
@@ -0,0 +1,47 @@
package io.ebeaninternal.server.profile;
import io.ebean.meta.BasicMetricVisitor;
import io.ebean.meta.MetaQueryMetric;
import org.junit.jupiter.api.Test;
import java.util.List;
import java.util.function.Function;
import static org.assertj.core.api.Assertions.assertThat;
class DQueryPlanMetricTest {
Function<String, String> naming = (String name) -> "prefix[" + name.replace('.', '-') + "]";
@Test
void visit() {
DQueryPlanMeta meta = new DQueryPlanMeta(Object.class, "lab", null, "sql");
DTimedMetric metric = new DTimedMetric("org.timed.plan");
DQueryPlanMetric planMetric = new DQueryPlanMetric(meta, metric);
metric.add(560);
metric.add(260);
{
BasicMetricVisitor visitor = new BasicMetricVisitor("v", naming);
planMetric.visit(visitor);
List<MetaQueryMetric> result = visitor.queryMetrics();
assertThat(result).hasSize(1);
assertThat(result.get(0).name()).isEqualTo("prefix[dto-Object_lab]");
assertThat(result.get(0).count()).isEqualTo(2);
assertThat(result.get(0).total()).isEqualTo(820);
}
metric.add(410);
{
BasicMetricVisitor visitor = new BasicMetricVisitor("v", naming);
planMetric.visit(visitor);
List<MetaQueryMetric> result = visitor.queryMetrics();
assertThat(result).hasSize(1);
assertThat(result.get(0).name()).isEqualTo("prefix[dto-Object_lab]");
assertThat(result.get(0).count()).isEqualTo(1);
assertThat(result.get(0).total()).isEqualTo(410);
}
}
}
@@ -1,7 +1,12 @@
package io.ebeaninternal.server.profile;
import io.ebean.meta.BasicMetricVisitor;
import io.ebean.meta.MetaTimedMetric;
import org.junit.jupiter.api.Test;
import java.util.List;
import java.util.function.Function;
import static org.assertj.core.api.Assertions.assertThat;
public class DTimedMetricTest {
@@ -52,4 +57,37 @@ public class DTimedMetricTest {
assertThat(stats.total()).isGreaterThan(10000);
assertThat(stats.max()).isEqualTo(stats.total() / 2);
}
Function<String, String> naming = (String name) -> "prefix[" + name.replace('.', '-') + "]";
@Test
void visit() {
DTimedMetric metric = new DTimedMetric("org.timed");
metric.add(560);
metric.add(500);
{
BasicMetricVisitor visitor = new BasicMetricVisitor("v", naming);
metric.visit(visitor);
List<MetaTimedMetric> result = visitor.timedMetrics();
assertThat(result).hasSize(1);
assertThat(result.get(0).name()).isEqualTo("prefix[org-timed]");
assertThat(result.get(0).count()).isEqualTo(2);
assertThat(result.get(0).total()).isEqualTo(1060);
}
metric.add(160);
metric.add(100);
metric.add(150);
{
BasicMetricVisitor visitor = new BasicMetricVisitor("v", naming);
metric.visit(visitor);
List<MetaTimedMetric> result = visitor.timedMetrics();
assertThat(result).hasSize(1);
assertThat(result.get(0).name()).isEqualTo("prefix[org-timed]");
assertThat(result.get(0).count()).isEqualTo(3);
assertThat(result.get(0).total()).isEqualTo(410);
}
}
}
@@ -14,6 +14,7 @@ import org.tests.model.basic.ResetBasicData;
import java.util.List;
import java.util.Optional;
import java.util.function.Function;
import java.util.stream.Collectors;
import java.util.stream.Stream;
@@ -209,8 +210,10 @@ public class DtoQuery2Test extends BaseTestCase {
log.info("Found " + custs);
}
Function<String, String> lowerUnderscore = (String key) -> "prefix:" + key.replace('.','_').toLowerCase();
// collect without reset
BasicMetricVisitor basic = new BasicMetricVisitor("db", false, true, true, true);
BasicMetricVisitor basic = new BasicMetricVisitor("db", lowerUnderscore, false, true, true, true);
server().metaInfo().visitMetrics(basic);
List<MetaQueryMetric> stats = basic.queryMetrics();
@@ -219,7 +222,7 @@ public class DtoQuery2Test extends BaseTestCase {
MetaQueryMetric queryMetric = stats.get(0);
assertThat(queryMetric.label()).isEqualTo("basic");
assertThat(queryMetric.count()).isEqualTo(3);
assertThat(queryMetric.name()).isEqualTo("dto.DCust_basic");
assertThat(queryMetric.name()).isEqualTo("prefix:dto_dcust_basic");
server().findDto(DCust.class, "select c4.id, c4.name from o_customer c4 where lower(c4.name) = :name")
@@ -1,5 +1,6 @@
package io.ebean.xtest.base;
import io.ebean.meta.MetricNamingMatch;
import io.ebean.xtest.BaseTestCase;
import io.ebean.DB;
import io.ebean.DtoQuery;
@@ -286,7 +287,7 @@ class DtoQueryTest extends BaseTestCase {
}
// collect without reset
BasicMetricVisitor basic = new BasicMetricVisitor("db", false, true, true, true);
BasicMetricVisitor basic = new BasicMetricVisitor("db", MetricNamingMatch.INSTANCE, false, true, true, true);
server().metaInfo().visitMetrics(basic);
List<MetaQueryMetric> stats = basic.queryMetrics();