From 3526f71f0a622ca5c5f1b2985e3042f32d392c31 Mon Sep 17 00:00:00 2001 From: Rob Bygrave Date: Wed, 2 Nov 2016 22:10:22 +1300 Subject: [PATCH] #99 - ENH (419) : Simplify aggregation queries --- .travis/print_surefire_reports | 0 .travis/setup_database | 0 .../java/com/avaje/ebean/DRawSqlParser.java | 9 +- .../com/avaje/ebean/SimpleTextParser.java | 167 ------------------ .../avaje/ebean/annotation/Aggregation.java | 50 ++++++ .../model/visitor/VisitAllUsing.java | 2 +- .../ebeaninternal/api/ManyWhereJoins.java | 15 +- .../server/deploy/BeanDescriptorManager.java | 8 +- .../server/deploy/BeanFkeyProperty.java | 5 + .../server/deploy/BeanProperty.java | 28 ++- .../server/deploy/DbSqlContext.java | 4 + .../server/deploy/DetermineAggPath.java | 96 ++++++++++ .../deploy/meta/DeployBeanProperty.java | 42 ++++- .../server/deploy/parse/AnnotationFields.java | 5 + .../server/el/ElPropertyChain.java | 5 + .../server/el/ElPropertyDeploy.java | 5 + .../server/expression/AbstractExpression.java | 3 + .../server/query/CQueryBuilder.java | 10 ++ .../server/query/DefaultDbSqlContext.java | 7 +- .../ebeaninternal/server/query/SqlTree.java | 12 +- .../server/query/SqlTreeBuilder.java | 27 ++- .../server/query/SqlTreeNode.java | 10 ++ .../server/query/SqlTreeNodeBean.java | 26 +++ .../server/query/SqlTreeNodeExtraJoin.java | 10 ++ .../query/SqlTreeNodeManyWhereJoin.java | 10 ++ .../server/querydefn/DefaultOrmQuery.java | 5 +- .../type/CtCompoundPropertyElAdapter.java | 5 + .../server/deploy/DetermineAggPathTest.java | 34 ++++ .../basic/TestOrderTotalAmountFormula.java | 10 +- .../avaje/tests/model/tevent/TEventMany.java | 46 +++-- .../avaje/tests/model/tevent/TEventOne.java | 50 ++++-- .../tevent/TestAssocOneNullTraverse.java | 15 +- .../aggregation/TestAggregationCount.java | 55 ++++++ 33 files changed, 559 insertions(+), 217 deletions(-) mode change 100755 => 100644 .travis/print_surefire_reports mode change 100755 => 100644 .travis/setup_database delete mode 100644 src/main/java/com/avaje/ebean/SimpleTextParser.java create mode 100644 src/main/java/com/avaje/ebean/annotation/Aggregation.java create mode 100644 src/main/java/com/avaje/ebeaninternal/server/deploy/DetermineAggPath.java create mode 100644 src/test/java/com/avaje/ebeaninternal/server/deploy/DetermineAggPathTest.java create mode 100644 src/test/java/com/avaje/tests/query/aggregation/TestAggregationCount.java diff --git a/.travis/print_surefire_reports b/.travis/print_surefire_reports old mode 100755 new mode 100644 diff --git a/.travis/setup_database b/.travis/setup_database old mode 100755 new mode 100644 diff --git a/src/main/java/com/avaje/ebean/DRawSqlParser.java b/src/main/java/com/avaje/ebean/DRawSqlParser.java index 33622f356..1f3b9a560 100644 --- a/src/main/java/com/avaje/ebean/DRawSqlParser.java +++ b/src/main/java/com/avaje/ebean/DRawSqlParser.java @@ -1,6 +1,7 @@ package com.avaje.ebean; import com.avaje.ebean.RawSql.Sql; +import com.avaje.ebeaninternal.server.querydefn.SimpleTextParser; /** * Parses sql-select queries to try and determine the location where WHERE and @@ -8,13 +9,13 @@ import com.avaje.ebean.RawSql.Sql; */ class DRawSqlParser { - public static final String $_AND_HAVING = "${andHaving}"; + private static final String $_AND_HAVING = "${andHaving}"; - public static final String $_HAVING = "${having}"; + private static final String $_HAVING = "${having}"; - public static final String $_AND_WHERE = "${andWhere}"; + private static final String $_AND_WHERE = "${andWhere}"; - public static final String $_WHERE = "${where}"; + private static final String $_WHERE = "${where}"; private final SimpleTextParser textParser; diff --git a/src/main/java/com/avaje/ebean/SimpleTextParser.java b/src/main/java/com/avaje/ebean/SimpleTextParser.java deleted file mode 100644 index fbeea27e3..000000000 --- a/src/main/java/com/avaje/ebean/SimpleTextParser.java +++ /dev/null @@ -1,167 +0,0 @@ -package com.avaje.ebean; - -class SimpleTextParser { - - private final String oql; - private final char[] chars; - private final int eof; - - private int pos; - private String word; - private String lowerWord; - - SimpleTextParser(String oql) { - this.oql = oql; - this.chars = oql.toCharArray(); - this.eof = oql.length(); - } - - public int getPos() { - return pos; - } - - public String getWord() { - return word; - } - - public String peekNextWord() { - int origPos = pos; - String nw = nextWordInternal(); - pos = origPos; - return nw; - } - - /** - * Match the current and the next word. - */ - public boolean isMatch(String lowerMatch, String nextWordMatch) { - - if (isMatch(lowerMatch)) { - String nw = peekNextWord(); - if (nw != null) { - nw = nw.toLowerCase(); - return nw.equals(nextWordMatch); - } - } - return false; - } - - public boolean isFinished() { - return word == null; - } - - public int findWordLower(String lowerMatch, int afterPos) { - this.pos = afterPos; - return findWordLower(lowerMatch); - } - - public int findWordLower(String lowerMatch) { - do { - if (nextWord() == null) { - return -1; - } - if (lowerMatch.equals(lowerWord)) { - return pos - lowerWord.length(); - } - } while (true); - } - - /** - * Match the current word. - */ - public boolean isMatch(String lowerMatch) { - return lowerMatch.equals(lowerWord); - } - - public String nextWord() { - word = nextWordInternal(); - if (word != null) { - lowerWord = word.toLowerCase(); - } - return word; - } - - private String nextWordInternal() { - trimLeadingWhitespace(); - if (pos >= eof) { - return null; - } - int start = pos; - if (chars[pos] == '(') { - moveToClose(); - } else { - moveToEndOfWord(); - } - return oql.substring(start, pos); - } - - private void moveToClose() { - - pos++; - int openParenthesisCount = 0; - - for (; pos < eof; pos++) { - char c = chars[pos]; - if (c == '(') { - // count nested parenthesis - openParenthesisCount++; - - } else if (c == ')') { - if (openParenthesisCount > 0) { - // still in nested parenthesis - --openParenthesisCount; - } else { - // we have found the end - pos++; - return; - } - } - } - } - - private void moveToEndOfWord() { - char c = chars[pos]; - boolean isOperator = isOperator(c); - for (; pos < eof; pos++) { - c = chars[pos]; - if (isWordTerminator(c, isOperator)) { - return; - } - } - } - - private boolean isWordTerminator(char c, boolean isOperator) { - if (Character.isWhitespace(c)) { - return true; - } - if (isOperator(c)) { - return !isOperator; - } - return c == '(' || isOperator; - } - - private boolean isOperator(char c) { - switch (c) { - case '<': - return true; - case '>': - return true; - case '=': - return true; - case '!': - return true; - - default: - return false; - } - } - - private void trimLeadingWhitespace() { - for (; pos < eof; pos++) { - char c = chars[pos]; - if (!Character.isWhitespace(c)) { - break; - } - } - } -} \ No newline at end of file diff --git a/src/main/java/com/avaje/ebean/annotation/Aggregation.java b/src/main/java/com/avaje/ebean/annotation/Aggregation.java new file mode 100644 index 000000000..7fb6589cc --- /dev/null +++ b/src/main/java/com/avaje/ebean/annotation/Aggregation.java @@ -0,0 +1,50 @@ +package com.avaje.ebean.annotation; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** + * Specify a property to be an aggregation formula. + *

+ * The aggregation formula should be a sum, count, avg, min or max. + * By default aggregation properties are treated as transient and not + * included in a query. To populate the aggregation property it must be + * explicitly included in the select(). + *

+ * + *

Example:

+ * 
{@code
+ *
+ * @Aggregation("count(details)")
+ * Long totalCount;
+ *
+ * @Aggregation("sum(details.quantity*details.unitPrice)")
+ * Long totalAmount;
+ *
+ * }
+ * + *

Example query

+ * 
{@code
+ *
+ *  List list = Ebean.find(TEventOne.class)
+ *       .select("name, totalCount, totalUnits, totalAmount")
+ *       .where()
+ *         .startsWith("logs.description", "a")
+ *       .having()
+ *         .ge("count", 1)
+ *       .orderBy().asc("name")
+ *       .findList();
+ *
+ * }
+ */ +@Retention(RetentionPolicy.RUNTIME) +@Target(ElementType.FIELD) +public @interface Aggregation { + + /** + * Aggregation formula using sum, count, avg, min, max. + */ + String value(); +} diff --git a/src/main/java/com/avaje/ebean/dbmigration/model/visitor/VisitAllUsing.java b/src/main/java/com/avaje/ebean/dbmigration/model/visitor/VisitAllUsing.java index f8f415db1..b25add8e5 100644 --- a/src/main/java/com/avaje/ebean/dbmigration/model/visitor/VisitAllUsing.java +++ b/src/main/java/com/avaje/ebean/dbmigration/model/visitor/VisitAllUsing.java @@ -66,7 +66,7 @@ public class VisitAllUsing { BeanProperty[] propertiesNonTransient = desc.propertiesNonTransient(); for (int i = 0; i < propertiesNonTransient.length; i++) { BeanProperty p = propertiesNonTransient[i]; - if (!p.isFormula() && !p.isSecondaryTable()) { + if (p.isDDLColumn()) { visit(propertyVisitor, p); } } diff --git a/src/main/java/com/avaje/ebeaninternal/api/ManyWhereJoins.java b/src/main/java/com/avaje/ebeaninternal/api/ManyWhereJoins.java index 0caab1800..c98837a6d 100644 --- a/src/main/java/com/avaje/ebeaninternal/api/ManyWhereJoins.java +++ b/src/main/java/com/avaje/ebeaninternal/api/ManyWhereJoins.java @@ -25,6 +25,8 @@ public class ManyWhereJoins implements Serializable { private boolean formulaWithJoin; + private boolean aggregation; + /** * 'Mode' indicating that joins added while this is true are required to be outer joins. */ @@ -93,10 +95,10 @@ public class ManyWhereJoins implements Serializable { } /** - * Return true if there are no extra many where joins. + * Return true if this is an aggregation query or if there are no extra many where joins. */ - public boolean isEmpty() { - return joins.isEmpty(); + public boolean requireSqlDistinct() { + return !aggregation && !joins.isEmpty(); } /** @@ -146,4 +148,11 @@ public class ManyWhereJoins implements Serializable { return formulaProperties.toString(); } + /** + * Mark this as part of an aggregation query (so using group by clause). + */ + public void setAggregation() { + aggregation = true; + } + } diff --git a/src/main/java/com/avaje/ebeaninternal/server/deploy/BeanDescriptorManager.java b/src/main/java/com/avaje/ebeaninternal/server/deploy/BeanDescriptorManager.java index 37556b202..ab4ef1c81 100644 --- a/src/main/java/com/avaje/ebeaninternal/server/deploy/BeanDescriptorManager.java +++ b/src/main/java/com/avaje/ebeaninternal/server/deploy/BeanDescriptorManager.java @@ -38,7 +38,6 @@ import com.avaje.ebeaninternal.server.deploy.meta.DeployBeanPropertyAssocMany; import com.avaje.ebeaninternal.server.deploy.meta.DeployBeanPropertyAssocOne; import com.avaje.ebeaninternal.server.deploy.meta.DeployBeanTable; import com.avaje.ebeaninternal.server.deploy.meta.DeployTableJoin; -import com.avaje.ebeaninternal.server.deploy.parse.AnnotationBase; import com.avaje.ebeaninternal.server.deploy.parse.DeployBeanInfo; import com.avaje.ebeaninternal.server.deploy.parse.DeployCreateProperties; import com.avaje.ebeaninternal.server.deploy.parse.DeployInherit; @@ -381,13 +380,13 @@ public class BeanDescriptorManager implements BeanDescriptorMap { try { entityClass = Class.forName(entityClassName, false, classLoader); } catch (Exception e) { - logger.error("Could not load entity bean class "+entityClassName+" for ebean.xml entry"); + logger.error("Could not load entity bean class " + entityClassName + " for ebean.xml entry"); return; } DeployBeanInfo info = deployInfoMap.get(entityClass); if (info == null) { - logger.error("No entity bean for ebean.xml entry "+entityClassName); + logger.error("No entity bean for ebean.xml entry " + entityClassName); } else { for (XmRawSql sql : entityDeploy.getRawSql()) { @@ -1351,6 +1350,9 @@ public class BeanDescriptorManager implements BeanDescriptorMap { prop.setPropertyIndex(propertyIndex); prop.setGetter(beanReflect.getGetter(propertyIndex)); prop.setSetter(beanReflect.getSetter(propertyIndex)); + if (prop.isAggregation()) { + prop.setAggregationPrefix(DetermineAggPath.manyPath(prop.getAggregation(), desc)); + } } } } diff --git a/src/main/java/com/avaje/ebeaninternal/server/deploy/BeanFkeyProperty.java b/src/main/java/com/avaje/ebeaninternal/server/deploy/BeanFkeyProperty.java index f25c0e9c5..b9e438b69 100644 --- a/src/main/java/com/avaje/ebeaninternal/server/deploy/BeanFkeyProperty.java +++ b/src/main/java/com/avaje/ebeaninternal/server/deploy/BeanFkeyProperty.java @@ -35,6 +35,11 @@ public final class BeanFkeyProperty implements ElPropertyValue { return "prefix:" + prefix + " name:" + name + " dbColumn:" + dbColumn + " ph:" + placeHolder; } + @Override + public boolean isAggregation() { + return false; + } + public int getDeployOrder() { return deployOrder; } diff --git a/src/main/java/com/avaje/ebeaninternal/server/deploy/BeanProperty.java b/src/main/java/com/avaje/ebeaninternal/server/deploy/BeanProperty.java index 98458338e..ac1619718 100644 --- a/src/main/java/com/avaje/ebeaninternal/server/deploy/BeanProperty.java +++ b/src/main/java/com/avaje/ebeaninternal/server/deploy/BeanProperty.java @@ -158,6 +158,7 @@ public class BeanProperty implements ElPropertyValue, Property { */ final String dbColumn; + final String elPrefix; final String elPlaceHolder; final String elPlaceHolderEncrypted; @@ -171,6 +172,8 @@ public class BeanProperty implements ElPropertyValue, Property { */ final String sqlFormulaJoin; + final String aggregation; + final boolean formula; /** @@ -311,6 +314,7 @@ public class BeanProperty implements ElPropertyValue, Property { this.dbColumn = tableAliasIntern(descriptor, deploy.getDbColumn(), false, null); this.dbComment = deploy.getDbComment(); + this.aggregation = deploy.getAggregation(); this.sqlFormulaJoin = InternString.intern(deploy.getSqlFormulaJoin()); this.sqlFormulaSelect = InternString.intern(deploy.getSqlFormulaSelect()); this.formula = sqlFormulaSelect != null; @@ -324,6 +328,7 @@ public class BeanProperty implements ElPropertyValue, Property { this.elPlaceHolder = tableAliasIntern(descriptor, deploy.getElPlaceHolder(), false, null); this.elPlaceHolderEncrypted = tableAliasIntern(descriptor, deploy.getElPlaceHolder(), dbEncrypted, dbColumn); + this.elPrefix = deploy.getElPrefix(); this.softDelete = deploy.isSoftDelete(); if (softDelete) { @@ -370,6 +375,7 @@ public class BeanProperty implements ElPropertyValue, Property { this.sqlFormulaJoin = null; this.sqlFormulaSelect = null; this.formula = false; + this.aggregation = null; this.excludedFromHistory = source.excludedFromHistory; this.draft = source.draft; @@ -421,6 +427,7 @@ public class BeanProperty implements ElPropertyValue, Property { this.field = source.getField(); this.docOptions = source.docOptions; + this.elPrefix = override.replace(source.elPrefix, source.dbColumn); this.elPlaceHolder = override.replace(source.elPlaceHolder, source.dbColumn); this.elPlaceHolderEncrypted = override.replace(source.elPlaceHolderEncrypted, source.dbColumn); @@ -474,6 +481,13 @@ public class BeanProperty implements ElPropertyValue, Property { return true; } + /** + * Return true if this property should have a DB Column created in DDL. + */ + public boolean isDDLColumn() { + return !formula && !secondaryTable && (aggregation == null); + } + /** * Return true if this property is based on a formula. */ @@ -537,8 +551,16 @@ public class BeanProperty implements ElPropertyValue, Property { return secondaryTableJoinPrefix; } + public boolean isAggregation() { + return aggregation != null; + } + public void appendSelect(DbSqlContext ctx, boolean subQuery) { - if (formula) { + + if (aggregation != null) { + ctx.appendRawColumn(aggregation); + + } else if (formula) { ctx.appendFormulaSelect(sqlFormulaSelect); } else if (!isTransient && !ignoreDraftOnlyProperty(ctx.isDraftQuery())) { @@ -843,7 +865,7 @@ public class BeanProperty implements ElPropertyValue, Property { } public boolean containsMany() { - return false; + return aggregation != null; } @Override @@ -893,7 +915,7 @@ public class BeanProperty implements ElPropertyValue, Property { } public String getElPrefix() { - return secondaryTableJoinPrefix; + return elPrefix; } /** diff --git a/src/main/java/com/avaje/ebeaninternal/server/deploy/DbSqlContext.java b/src/main/java/com/avaje/ebeaninternal/server/deploy/DbSqlContext.java index 1f6714c06..392c63b13 100644 --- a/src/main/java/com/avaje/ebeaninternal/server/deploy/DbSqlContext.java +++ b/src/main/java/com/avaje/ebeaninternal/server/deploy/DbSqlContext.java @@ -125,4 +125,8 @@ public interface DbSqlContext { */ boolean isDraftQuery(); + /** + * Start group by clause. + */ + void startGroupBy(); } diff --git a/src/main/java/com/avaje/ebeaninternal/server/deploy/DetermineAggPath.java b/src/main/java/com/avaje/ebeaninternal/server/deploy/DetermineAggPath.java new file mode 100644 index 000000000..1b24a7217 --- /dev/null +++ b/src/main/java/com/avaje/ebeaninternal/server/deploy/DetermineAggPath.java @@ -0,0 +1,96 @@ +package com.avaje.ebeaninternal.server.deploy; + +import com.avaje.ebeaninternal.server.deploy.meta.DeployBeanDescriptor; +import com.avaje.ebeaninternal.server.deploy.meta.DeployBeanProperty; +import com.avaje.ebeaninternal.server.deploy.meta.DeployBeanPropertyAssocMany; +import com.avaje.ebeaninternal.server.deploy.meta.DeployBeanPropertyAssocOne; + +class DetermineAggPath { + + /** + * Return the many path for the given aggregation formula. + */ + static String manyPath(String aggregation, DeployBeanDescriptor desc) { + DetermineAggPath.Path path = paths(aggregation); + return path.getManyPath(0, desc); + } + + static Path paths(String aggregation) { + String aggPath = path(aggregation); + return new Path(aggPath.split("\\."), aggregation); + } + + /** + * Parse and return the full path for the aggregation. + */ + static String path(String aggregation) { + + // aggregations always have a form of sum(), avg(), max(), count() etc + // so find the first open bracket + int start = aggregation.indexOf('('); + if (start == -1) { + throw new IllegalArgumentException("Aggregation formula ["+aggregation+"] is expected to have a '(' ?"); + } + for (int i = start + 1; i< aggregation.length(); i++) { + char ch = aggregation.charAt(i); + if (!isNamePart(ch)) { + return aggregation.substring(start + 1, i); + } + } + + throw new IllegalArgumentException("Could not find path in aggregation formula ["+aggregation+"]"); + } + + private static boolean isNamePart(char ch) { + return ch == '.' || Character.isJavaIdentifierPart(ch); + } + + + /** + * Helper class holding aggregation path segments. + */ + static class Path { + + final String aggregation; + final String[] paths; + + Path(String[] paths, String aggregation) { + this.paths = paths; + this.aggregation = aggregation; + } + + int length() { + return paths.length; + } + + String path(int pos) { + if (pos == 0) { + return paths[0]; + } else { + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < pos; i++) { + if (i > 0) { + sb.append("."); + } + sb.append(paths[i]); + } + return sb.toString(); + } + } + + String getManyPath(int pos, DeployBeanDescriptor desc) { + + String path = paths[pos]; + DeployBeanProperty details = desc.getBeanProperty(path); + if (details instanceof DeployBeanPropertyAssocMany) { + return path(pos); + + } else if (details instanceof DeployBeanPropertyAssocOne) { + DeployBeanPropertyAssocOne one = (DeployBeanPropertyAssocOne)details; + DeployBeanDescriptor targetDesc = one.getTargetDeploy(); + return getManyPath(pos + 1, targetDesc); + } + throw new IllegalArgumentException("Can not find path to many in aggregation formula ["+aggregation+"]"); + } + } +} diff --git a/src/main/java/com/avaje/ebeaninternal/server/deploy/meta/DeployBeanProperty.java b/src/main/java/com/avaje/ebeaninternal/server/deploy/meta/DeployBeanProperty.java index 126f864d0..a0aa3f45e 100644 --- a/src/main/java/com/avaje/ebeaninternal/server/deploy/meta/DeployBeanProperty.java +++ b/src/main/java/com/avaje/ebeaninternal/server/deploy/meta/DeployBeanProperty.java @@ -160,6 +160,9 @@ public class DeployBeanProperty { */ private String dbColumn; + private String aggregationPrefix; + private String aggregation; + private String sqlFormulaSelect; private String sqlFormulaJoin; @@ -585,8 +588,42 @@ public class DeployBeanProperty { this.dbUpdateable = false; } + + public boolean isAggregation() { + return aggregation != null; + } + + public String getAggregation() { + return aggregation; + } + + public void setAggregation(String aggregation) { + this.aggregation = aggregation; + this.dbRead = true; + this.dbInsertable = false; + this.dbUpdateable = false; + } + + /** + * Set the path to the aggregation. + */ + public void setAggregationPrefix(String aggregationPrefix) { + this.aggregationPrefix = aggregationPrefix; + this.aggregation = aggregation.replace(aggregationPrefix, "u1"); + } + + public String getElPrefix() { + if (aggregation != null) { + return aggregationPrefix; + } else { + return secondaryTableJoinPrefix; + } + } + public String getElPlaceHolder() { - if (sqlFormulaSelect != null) { + if (aggregation != null) { + return aggregation; + } else if (sqlFormulaSelect != null) { return sqlFormulaSelect; } else { if (secondaryTableJoinPrefix != null) { @@ -604,6 +641,9 @@ public class DeployBeanProperty { if (sqlFormulaSelect != null) { return sqlFormulaSelect; } + if (aggregation != null) { + return aggregation; + } return dbColumn; } diff --git a/src/main/java/com/avaje/ebeaninternal/server/deploy/parse/AnnotationFields.java b/src/main/java/com/avaje/ebeaninternal/server/deploy/parse/AnnotationFields.java index 022044ec8..ac5172534 100644 --- a/src/main/java/com/avaje/ebeaninternal/server/deploy/parse/AnnotationFields.java +++ b/src/main/java/com/avaje/ebeaninternal/server/deploy/parse/AnnotationFields.java @@ -212,6 +212,11 @@ public class AnnotationFields extends AnnotationParser { prop.setSqlFormula(formula.select(), formula.join()); } + Aggregation aggregation = get(prop, Aggregation.class); + if (aggregation != null) { + prop.setAggregation(aggregation.value()); + } + Version version = get(prop, Version.class); if (version != null) { // explicitly specify a version column diff --git a/src/main/java/com/avaje/ebeaninternal/server/el/ElPropertyChain.java b/src/main/java/com/avaje/ebeaninternal/server/el/ElPropertyChain.java index 26a8bd57f..dcb9a0c19 100644 --- a/src/main/java/com/avaje/ebeaninternal/server/el/ElPropertyChain.java +++ b/src/main/java/com/avaje/ebeaninternal/server/el/ElPropertyChain.java @@ -83,6 +83,11 @@ public class ElPropertyChain implements ElPropertyValue { return "expr:" + expression + " chain:" + Arrays.toString(chain); } + @Override + public boolean isAggregation() { + return false; + } + private String getElPlaceHolder(String prefix, ElPropertyValue lastElPropertyValue, boolean encrypted) { if (prefix == null) { return lastElPropertyValue.getElPlaceholder(encrypted); diff --git a/src/main/java/com/avaje/ebeaninternal/server/el/ElPropertyDeploy.java b/src/main/java/com/avaje/ebeaninternal/server/el/ElPropertyDeploy.java index 4bf5caea4..2b3b08009 100644 --- a/src/main/java/com/avaje/ebeaninternal/server/el/ElPropertyDeploy.java +++ b/src/main/java/com/avaje/ebeaninternal/server/el/ElPropertyDeploy.java @@ -69,4 +69,9 @@ public interface ElPropertyDeploy { * Return the underlying bean property. */ BeanProperty getBeanProperty(); + + /** + * Return true if this is an aggregation property. + */ + boolean isAggregation(); } diff --git a/src/main/java/com/avaje/ebeaninternal/server/expression/AbstractExpression.java b/src/main/java/com/avaje/ebeaninternal/server/expression/AbstractExpression.java index 5b9569496..6200d5746 100644 --- a/src/main/java/com/avaje/ebeaninternal/server/expression/AbstractExpression.java +++ b/src/main/java/com/avaje/ebeaninternal/server/expression/AbstractExpression.java @@ -72,6 +72,9 @@ public abstract class AbstractExpression implements SpiExpression { if (elProp.containsMany()) { // for findRowCount we join to a many property manyWhereJoin.add(elProp); + if (elProp.isAggregation()) { + manyWhereJoin.setAggregation(); + } } } } diff --git a/src/main/java/com/avaje/ebeaninternal/server/query/CQueryBuilder.java b/src/main/java/com/avaje/ebeaninternal/server/query/CQueryBuilder.java index ce8efe1fa..4121f8dee 100644 --- a/src/main/java/com/avaje/ebeaninternal/server/query/CQueryBuilder.java +++ b/src/main/java/com/avaje/ebeaninternal/server/query/CQueryBuilder.java @@ -531,6 +531,16 @@ public class CQueryBuilder { } } + String groupBy = select.getGroupBy(); + if (groupBy != null) { + sb.append(" group by ").append(groupBy); + } + + String dbHaving = predicates.getDbHaving(); + if (!isEmpty(dbHaving)) { + sb.append(" having ").append(dbHaving); + } + if (dbOrderBy != null) { sb.append(" order by ").append(dbOrderBy); } diff --git a/src/main/java/com/avaje/ebeaninternal/server/query/DefaultDbSqlContext.java b/src/main/java/com/avaje/ebeaninternal/server/query/DefaultDbSqlContext.java index 9133f4366..ddba62116 100644 --- a/src/main/java/com/avaje/ebeaninternal/server/query/DefaultDbSqlContext.java +++ b/src/main/java/com/avaje/ebeaninternal/server/query/DefaultDbSqlContext.java @@ -26,7 +26,7 @@ public class DefaultDbSqlContext implements DbSqlContext { private final ArrayStack prefixStack = new ArrayStack(); - private final boolean useColumnAlias; + private boolean useColumnAlias; private int columnIndex; @@ -66,6 +66,11 @@ public class DefaultDbSqlContext implements DbSqlContext { this.historyQuery = (historySupport != null); } + @Override + public void startGroupBy() { + this.useColumnAlias = false; + } + public void addEncryptedProp(BeanProperty p) { if (encryptedProps == null) { encryptedProps = new ArrayList(); diff --git a/src/main/java/com/avaje/ebeaninternal/server/query/SqlTree.java b/src/main/java/com/avaje/ebeaninternal/server/query/SqlTree.java index 8e7a3906e..523414f2a 100644 --- a/src/main/java/com/avaje/ebeaninternal/server/query/SqlTree.java +++ b/src/main/java/com/avaje/ebeaninternal/server/query/SqlTree.java @@ -31,6 +31,8 @@ public class SqlTree { private final String fromSql; + private final String groupBy; + /** * Encrypted Properties require additional binding. */ @@ -46,13 +48,14 @@ public class SqlTree { /** * Create the SqlSelectClause. */ - public SqlTree(String summary, SqlTreeNode rootNode, String selectSql, String fromSql, String inheritanceWhereSql, + public SqlTree(String summary, SqlTreeNode rootNode, String selectSql, String fromSql, String groupBy, String inheritanceWhereSql, BeanProperty[] encryptedProps, BeanPropertyAssocMany manyProperty, Set includes, boolean includeJoins) { this.summary = summary; this.rootNode = rootNode; this.selectSql = selectSql; this.fromSql = fromSql; + this.groupBy = groupBy; this.inheritanceWhereSql = inheritanceWhereSql; this.encryptedProps = encryptedProps; this.manyProperty = manyProperty; @@ -108,6 +111,13 @@ public class SqlTree { return fromSql; } + /** + * Return the groupBy clause. + */ + public String getGroupBy() { + return groupBy; + } + /** * Return the where clause for inheritance. */ diff --git a/src/main/java/com/avaje/ebeaninternal/server/query/SqlTreeBuilder.java b/src/main/java/com/avaje/ebeaninternal/server/query/SqlTreeBuilder.java index 835fd97ed..3029a2690 100644 --- a/src/main/java/com/avaje/ebeaninternal/server/query/SqlTreeBuilder.java +++ b/src/main/java/com/avaje/ebeaninternal/server/query/SqlTreeBuilder.java @@ -129,17 +129,19 @@ public class SqlTreeBuilder { String selectSql = null; String fromSql = null; String inheritanceWhereSql = null; + String groupBy = null; BeanProperty[] encryptedProps = null; if (!rawSql) { selectSql = buildSelectClause(); fromSql = buildFromClause(); inheritanceWhereSql = buildWhereClause(); + groupBy = buildGroupByClause(); encryptedProps = ctx.getEncryptedProps(); } boolean includeJoins = (alias == null) ? false : alias.isIncludeJoins(); - return new SqlTree(summary.toString(), rootNode, selectSql, fromSql, inheritanceWhereSql, encryptedProps, + return new SqlTree(summary.toString(), rootNode, selectSql, fromSql, groupBy, inheritanceWhereSql, encryptedProps, manyProperty, queryDetail.getFetchPaths(), includeJoins); } @@ -149,15 +151,28 @@ public class SqlTreeBuilder { return "Not Used"; } rootNode.appendSelect(ctx, subQuery); + return trimComma(ctx.getContent()); + } - String selectSql = ctx.getContent(); + private String buildGroupByClause() { - // trim off the first comma - if (selectSql.length() >= SqlTreeNode.COMMA.length()) { - selectSql = selectSql.substring(SqlTreeNode.COMMA.length()); + if (rawSql || !rootNode.isAggregation()) { + return null; } + ctx.startGroupBy(); + rootNode.appendGroupBy(ctx, subQuery); + String groupBy = ctx.getContent(); + return trimComma(groupBy); + } - return selectSql; + /** + * Trim the first comma. + */ + private String trimComma(String groupBy) { + if (groupBy.length() >= SqlTreeNode.COMMA.length()) { + groupBy = groupBy.substring(SqlTreeNode.COMMA.length()); + } + return groupBy; } private String buildWhereClause() { diff --git a/src/main/java/com/avaje/ebeaninternal/server/query/SqlTreeNode.java b/src/main/java/com/avaje/ebeaninternal/server/query/SqlTreeNode.java index 0313c351c..d18936676 100644 --- a/src/main/java/com/avaje/ebeaninternal/server/query/SqlTreeNode.java +++ b/src/main/java/com/avaje/ebeaninternal/server/query/SqlTreeNode.java @@ -19,12 +19,22 @@ public interface SqlTreeNode { */ void buildRawSqlSelectChain(List selectChain); + /** + * Return true if this node includes an aggregation. + */ + boolean isAggregation(); + /** * Append the required column information to the SELECT part of the sql * statement. */ void appendSelect(DbSqlContext ctx, boolean subQuery); + /** + * Append the group by clause. + */ + void appendGroupBy(DbSqlContext ctx, boolean subQuery); + /** * Append to the FROM part of the sql. */ diff --git a/src/main/java/com/avaje/ebeaninternal/server/query/SqlTreeNodeBean.java b/src/main/java/com/avaje/ebeaninternal/server/query/SqlTreeNodeBean.java index 0bc652c12..c244f13ab 100644 --- a/src/main/java/com/avaje/ebeaninternal/server/query/SqlTreeNodeBean.java +++ b/src/main/java/com/avaje/ebeaninternal/server/query/SqlTreeNodeBean.java @@ -83,6 +83,8 @@ public class SqlTreeNodeBean implements SqlTreeNode { */ private boolean intersectionAsOfTableAlias; + private boolean aggregation; + /** * Construct for leaf node. */ @@ -381,6 +383,23 @@ public class SqlTreeNodeBean implements SqlTreeNode { } } + @Override + public void appendGroupBy(DbSqlContext ctx, boolean subQuery) { + + ctx.pushJoin(prefix); + ctx.pushTableAlias(prefix); + if (readId) { + appendSelectId(ctx, idBinder.getBeanProperty()); + } + for (int i = 0; i < properties.length; i++) { + if (!properties[i].isAggregation()) { + properties[i].appendSelect(ctx, subQuery); + } + } + ctx.popTableAlias(); + ctx.popJoin(); + } + /** * Append the property columns to the buffer. */ @@ -417,6 +436,10 @@ public class SqlTreeNodeBean implements SqlTreeNode { ctx.popJoin(); } + public boolean isAggregation() { + return aggregation; + } + /** * Append the properties to the buffer. */ @@ -424,6 +447,9 @@ public class SqlTreeNodeBean implements SqlTreeNode { for (int i = 0; i < props.length; i++) { props[i].appendSelect(ctx, subQuery); + if (props[i].isAggregation()) { + aggregation = true; + } } } diff --git a/src/main/java/com/avaje/ebeaninternal/server/query/SqlTreeNodeExtraJoin.java b/src/main/java/com/avaje/ebeaninternal/server/query/SqlTreeNodeExtraJoin.java index a3154e89c..45f30b8bc 100644 --- a/src/main/java/com/avaje/ebeaninternal/server/query/SqlTreeNodeExtraJoin.java +++ b/src/main/java/com/avaje/ebeaninternal/server/query/SqlTreeNodeExtraJoin.java @@ -56,6 +56,16 @@ public class SqlTreeNodeExtraJoin implements SqlTreeNode { // nothing to do here } + @Override + public boolean isAggregation() { + return false; + } + + @Override + public void appendGroupBy(DbSqlContext ctx, boolean subQuery) { + // nothing to do here + } + @Override public BeanProperty getSingleProperty() { throw new IllegalStateException("No expected"); diff --git a/src/main/java/com/avaje/ebeaninternal/server/query/SqlTreeNodeManyWhereJoin.java b/src/main/java/com/avaje/ebeaninternal/server/query/SqlTreeNodeManyWhereJoin.java index b3c390761..b0d7b2944 100644 --- a/src/main/java/com/avaje/ebeaninternal/server/query/SqlTreeNodeManyWhereJoin.java +++ b/src/main/java/com/avaje/ebeaninternal/server/query/SqlTreeNodeManyWhereJoin.java @@ -55,6 +55,16 @@ public class SqlTreeNodeManyWhereJoin implements SqlTreeNode { // do nothing here ... } + @Override + public boolean isAggregation() { + return false; + } + + @Override + public void appendGroupBy(DbSqlContext ctx, boolean subQuery) { + // do nothing here + } + /** * Append to the FROM clause for this node. */ diff --git a/src/main/java/com/avaje/ebeaninternal/server/querydefn/DefaultOrmQuery.java b/src/main/java/com/avaje/ebeaninternal/server/querydefn/DefaultOrmQuery.java index fa84c0039..d1237712d 100644 --- a/src/main/java/com/avaje/ebeaninternal/server/querydefn/DefaultOrmQuery.java +++ b/src/main/java/com/avaje/ebeaninternal/server/querydefn/DefaultOrmQuery.java @@ -370,7 +370,10 @@ public class DefaultOrmQuery implements SpiQuery { if (whereExpressions != null) { whereExpressions.containsMany(beanDescriptor, manyWhereJoins); } - if (!manyWhereJoins.isEmpty()) { + if (havingExpressions != null) { + havingExpressions.containsMany(beanDescriptor, manyWhereJoins); + } + if (manyWhereJoins.requireSqlDistinct()) { setSqlDistinct(true); } } diff --git a/src/main/java/com/avaje/ebeaninternal/server/type/CtCompoundPropertyElAdapter.java b/src/main/java/com/avaje/ebeaninternal/server/type/CtCompoundPropertyElAdapter.java index bd11ecaf0..452075a84 100644 --- a/src/main/java/com/avaje/ebeaninternal/server/type/CtCompoundPropertyElAdapter.java +++ b/src/main/java/com/avaje/ebeaninternal/server/type/CtCompoundPropertyElAdapter.java @@ -29,6 +29,11 @@ public class CtCompoundPropertyElAdapter implements ElPropertyValue { this.deployOrder = deployOrder; } + @Override + public boolean isAggregation() { + return false; + } + @Override public Object convert(Object value) { return value; diff --git a/src/test/java/com/avaje/ebeaninternal/server/deploy/DetermineAggPathTest.java b/src/test/java/com/avaje/ebeaninternal/server/deploy/DetermineAggPathTest.java new file mode 100644 index 000000000..bfe31d2a5 --- /dev/null +++ b/src/test/java/com/avaje/ebeaninternal/server/deploy/DetermineAggPathTest.java @@ -0,0 +1,34 @@ +package com.avaje.ebeaninternal.server.deploy; + +import org.junit.Test; + +import static org.assertj.core.api.Assertions.assertThat; + + +public class DetermineAggPathTest { + + @Test + public void path() throws Exception { + + assertThat(DetermineAggPath.path("count(details)")).isEqualTo("details"); + assertThat(DetermineAggPath.path("count(details )")).isEqualTo("details"); + + assertThat(DetermineAggPath.path("count(person.contacts)")).isEqualTo("person.contacts"); + assertThat(DetermineAggPath.path("sum(details.quantity*details.unitPrice)")).isEqualTo("details.quantity"); + } + + @Test + public void paths_simple() throws Exception { + + DetermineAggPath.Path paths = DetermineAggPath.paths("count(details)"); + assertThat(paths.paths).isEqualTo(new String[]{"details"}); + } + + @Test + public void paths_nested() throws Exception { + + DetermineAggPath.Path paths = DetermineAggPath.paths("count(person.contacts)"); + assertThat(paths.paths).isEqualTo(new String[]{"person", "contacts"}); + } + +} \ No newline at end of file diff --git a/src/test/java/com/avaje/tests/basic/TestOrderTotalAmountFormula.java b/src/test/java/com/avaje/tests/basic/TestOrderTotalAmountFormula.java index bf985b22e..f167e28a0 100644 --- a/src/test/java/com/avaje/tests/basic/TestOrderTotalAmountFormula.java +++ b/src/test/java/com/avaje/tests/basic/TestOrderTotalAmountFormula.java @@ -17,10 +17,12 @@ public class TestOrderTotalAmountFormula extends BaseTestCase { ResetBasicData.reset(); - List l0 = Ebean.find(Customer.class).select("id, name") - .fetch("orders", "status, totalAmount").where().eq("orders.details.product.name", "Desk") - .like("contacts.firstName", "Ji%") - + List l0 = Ebean.find(Customer.class) + .select("id, name") + .fetch("orders", "status, totalAmount") + .where() + .eq("orders.details.product.name", "Desk") + .like("contacts.firstName", "Ji%") .findList(); for (Customer c0 : l0) { diff --git a/src/test/java/com/avaje/tests/model/tevent/TEventMany.java b/src/test/java/com/avaje/tests/model/tevent/TEventMany.java index 9986d167e..88679715d 100644 --- a/src/test/java/com/avaje/tests/model/tevent/TEventMany.java +++ b/src/test/java/com/avaje/tests/model/tevent/TEventMany.java @@ -11,14 +11,24 @@ public class TEventMany { @Id Long id; - String many; + String description; @ManyToOne - TEventOne one; + TEventOne event; + + int units; + + double amount; @Version Long version; + public TEventMany(String description, int units, double amount) { + this.description = description; + this.units = units; + this.amount = amount; + } + public Long getId() { return id; } @@ -27,20 +37,36 @@ public class TEventMany { this.id = id; } - public String getMany() { - return many; + public String getDescription() { + return description; } - public void setMany(String many) { - this.many = many; + public void setDescription(String description) { + this.description = description; } - public TEventOne getOne() { - return one; + public TEventOne getEvent() { + return event; } - public void setOne(TEventOne one) { - this.one = one; + public void setEvent(TEventOne event) { + this.event = event; + } + + public int getUnits() { + return units; + } + + public void setUnits(int units) { + this.units = units; + } + + public double getAmount() { + return amount; + } + + public void setAmount(double amount) { + this.amount = amount; } public Long getVersion() { diff --git a/src/test/java/com/avaje/tests/model/tevent/TEventOne.java b/src/test/java/com/avaje/tests/model/tevent/TEventOne.java index 617b18a56..81c3dc3d9 100644 --- a/src/test/java/com/avaje/tests/model/tevent/TEventOne.java +++ b/src/test/java/com/avaje/tests/model/tevent/TEventOne.java @@ -1,5 +1,8 @@ package com.avaje.tests.model.tevent; +import com.avaje.ebean.annotation.Aggregation; + +import javax.persistence.CascadeType; import javax.persistence.Entity; import javax.persistence.Id; import javax.persistence.OneToMany; @@ -13,7 +16,7 @@ public class TEventOne { @Id Long id; - String one; + String name; @Version Long version; @@ -21,8 +24,21 @@ public class TEventOne { @OneToOne TEvent event; - @OneToMany(mappedBy = "one") - List many; + @Aggregation("count(logs.*)") + Long count; + + @Aggregation("sum(logs.units)") + Double totalUnits; + + @Aggregation("sum(logs.units * logs.amount)") + Double totalAmount; + + @OneToMany(mappedBy = "event", cascade = CascadeType.ALL) + List logs; + + public TEventOne(String name) { + this.name = name; + } public Long getId() { return id; @@ -32,12 +48,24 @@ public class TEventOne { this.id = id; } - public String getOne() { - return one; + public Long getCount() { + return count; } - public void setOne(String one) { - this.one = one; + public Double getTotalUnits() { + return totalUnits; + } + + public Double getTotalAmount() { + return totalAmount; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; } public Long getVersion() { @@ -56,11 +84,11 @@ public class TEventOne { this.event = event; } - public List getMany() { - return many; + public List getLogs() { + return logs; } - public void setMany(List many) { - this.many = many; + public void setLogs(List logs) { + this.logs = logs; } } diff --git a/src/test/java/com/avaje/tests/model/tevent/TestAssocOneNullTraverse.java b/src/test/java/com/avaje/tests/model/tevent/TestAssocOneNullTraverse.java index dcb79f4cc..b03df5547 100644 --- a/src/test/java/com/avaje/tests/model/tevent/TestAssocOneNullTraverse.java +++ b/src/test/java/com/avaje/tests/model/tevent/TestAssocOneNullTraverse.java @@ -13,7 +13,20 @@ public class TestAssocOneNullTraverse extends BaseTestCase { Ebean.save(event); Ebean.find(TEvent.class) - .fetch("one.many") + .fetch("one.logs") .findList(); } + +// @Test +// public void testSelectAggregation() { +// +// Query query = Ebean.find(TEvent.class) +// .select("id, name") +// .fetch("one", "count"); +// +// query.findList(); +// +// String sql = query.getGeneratedSql(); +// assertThat(sql).contains("asd"); +// } } diff --git a/src/test/java/com/avaje/tests/query/aggregation/TestAggregationCount.java b/src/test/java/com/avaje/tests/query/aggregation/TestAggregationCount.java new file mode 100644 index 000000000..f381bd20e --- /dev/null +++ b/src/test/java/com/avaje/tests/query/aggregation/TestAggregationCount.java @@ -0,0 +1,55 @@ +package com.avaje.tests.query.aggregation; + +import com.avaje.ebean.BaseTestCase; +import com.avaje.ebean.Ebean; +import com.avaje.ebean.Query; +import com.avaje.tests.model.tevent.TEventMany; +import com.avaje.tests.model.tevent.TEventOne; +import org.junit.Test; + +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; + +public class TestAggregationCount extends BaseTestCase { + + @Test + public void test() { + + TEventOne one = new TEventOne("first"); + one.getLogs().add(new TEventMany("all", 1, 10)); + one.getLogs().add(new TEventMany("be", 2, 12.2)); + one.getLogs().add(new TEventMany("add", 3, 13)); + Ebean.save(one); + + TEventOne two = new TEventOne("second"); + two.getLogs().add(new TEventMany("at", 10, 10)); + two.getLogs().add(new TEventMany("add", 30, 13)); + two.getLogs().add(new TEventMany("alf", 30, 13)); + Ebean.save(two); + + + Query query = Ebean.find(TEventOne.class) + .select("name, count, totalUnits, totalAmount") + .where() + .startsWith("logs.description", "a") + .having() + .ge("count", 1) + .orderBy().asc("name"); + + List list = query.findList(); + for (TEventOne eventOne : list) { + System.out.println(eventOne.getId() + " " + eventOne.getName() + " count:" + eventOne.getCount() + " units:" + eventOne.getTotalUnits() + " amount:" + eventOne.getTotalAmount()); + } + + assertThat(list).isNotEmpty(); + + String sql = query.getGeneratedSql(); + assertThat(sql).contains("select t0.id c0, t0.name c1, count(u1.*) c2, sum(u1.units) c3, sum(u1.units * u1.amount) c4 from tevent_one t0"); + assertThat(sql).contains("from tevent_one t0 join tevent_many u1 on u1.event_id = t0.id "); + assertThat(sql).contains("where u1.description like ? "); + assertThat(sql).contains(" group by t0.id, t0.name having count(u1.*) >= ? order by t0.name"); + + } + +}