#1214 - ENH: Add inPairs() expression - support L2 cache hits for complex natural key with findList() and in pairs expression

This commit is contained in:
Rob Bygrave
2017-11-19 16:47:32 +13:00
parent f0ddf6c7e6
commit ef91c643ed
22 changed files with 765 additions and 87 deletions
@@ -246,6 +246,11 @@ public interface ExpressionFactory {
*/
Expression icontains(String propertyName, String value);
/**
* In expression using pairs of value objects.
*/
Expression inPairs(Pairs pairs);
/**
* In - property has a value in the array of values.
*/
@@ -811,6 +811,11 @@ public interface ExpressionList<T> {
*/
ExpressionList<T> icontains(String propertyName, String value);
/**
* In expression using pairs of value objects.
*/
ExpressionList<T> inPairs(Pairs pairs);
/**
* In - using a subQuery.
*/
+220
View File
@@ -0,0 +1,220 @@
package io.ebean;
import java.util.ArrayList;
import java.util.List;
/**
* Holds a list of value object pairs.
* <p>
* This feature is to enable use of L2 cache with complex natural keys with findList() queries in cases where the
* IN clause is not a single property but instead a pair of properties.
* </p>
* <p>
* These queries can have predicates that can be translated into a list of complex natural keys such that the L2
* cache can be hit with these keys to obtain some or all of the beans from L2 cache rather than the DB.
* </p>
* <pre>{@code
*
* // where a bean is annotated with a complex
* // natural key made of several properties
* @Cache(naturalKey = {"store","code","sku"})
*
*
* Pairs pairs = new Pairs("sku", "code");
* pairs.add("sj2", 1000);
* pairs.add("sj2", 1001);
* pairs.add("pf3", 1000);
*
* List<OCachedNatKeyBean3> list = Ebean.find(OCachedNatKeyBean3.class)
* .where()
* .eq("store", "def")
* .inPairs(pairs) // IN clause with 'pairs' of values
* .orderBy("sku desc")
*
* // query expressions cover the natural key properties
* // so we can choose to hit the L2 bean cache if we want
* .setUseCache(true)
* .findList();
*
* }</pre>
* <h3>Important implementation Note</h3>
* <p>
* When binding many pairs of values we want to be able to utilise a DB index (as this type of query usually means the
* pairs are a unique key/index or part of a unique key/index and highly selective). Currently we know we can do this
* on any DB that supports expression/formula based indexes.
* using a DB string concatenation formula
* </p>
* <p>
* This means, the implementation converts the list of pairs into a list of strings via concatenation and we use a
* DB concatenation formula to match. We see SQL like:
* </p>
* <pre>{@code sql
*
* ...
* where t0.store = ? and (t0.sku||'-'||t0.code) in (?, ? )
*
* // bind values like: "sj2-1000", "pf3-1000"
*
* }</pre>
* <p>
* We often create a DB expression index to match the DB concat formula like:
* </p>
* <pre>{@code sql
*
* create index ix_name on table_name ((sku || '-' || code));
*
* }</pre>
*/
public class Pairs {
private final String property0;
private final String property1;
private final List<Entry> entries = new ArrayList<>();
/**
* Character between the values when combined via DB varchar concatenation.
*/
private String concatSeparator = "-";
/**
* Optional suffix added to DB varchar concatenation formula.
*/
private String concatSuffix;
/**
* Create with 2 property names.
*
* @param property0 The property of the first value
* @param property1 The property of the second value
*/
public Pairs(String property0, String property1) {
this.property0 = property0;
this.property1 = property1;
}
/**
* Add a pair of value objects.
* <p>
* Both values are expected to be immutable with equals and hashCode implementations.
* </p>
*
* @param a Value of the first property
* @param b Value of the second property
*/
public Pairs add(Object a, Object b) {
entries.add(new Entry(a, b));
return this;
}
/**
* Return the first property name.
*/
public String getProperty0() {
return property0;
}
/**
* Return the second property name.
*/
public String getProperty1() {
return property1;
}
/**
* Return all the value pairs.
*/
public List<Entry> getEntries() {
return entries;
}
/**
* Return the separator character used with DB varchar concatenation to combine the 2 values.
*/
public String getConcatSeparator() {
return concatSeparator;
}
/**
* Set the separator character used with DB varchar concatenation to combine the 2 values.
*/
public Pairs setConcatSeparator(String concatSeparator) {
this.concatSeparator = concatSeparator;
return this;
}
/**
* Return a suffix used with DB varchar concatenation to combine the 2 values.
*/
public String getConcatSuffix() {
return concatSuffix;
}
/**
* Add a suffix used with DB varchar concatenation to combine the 2 values.
*/
public Pairs setConcatSuffix(String concatSuffix) {
this.concatSuffix = concatSuffix;
return this;
}
public String toString() {
return "p0:" + property0 + " p1:" + property1 + " entries:" + entries;
}
/**
* A pair of 2 value objects.
* <p>
* Used to support inPairs() expression.
*/
public static class Entry {
private final Object a;
private final Object b;
/**
* Create with values for property0 and property1 respectively.
*
* @param a Value of the first property
* @param b Value of the second property
*/
public Entry(Object a, Object b) {
this.a = a;
this.b = b;
}
public String toString() {
return "{" + a + "," + b + "}";
}
/**
* Return the value for the first property.
*/
public Object getA() {
return a;
}
/**
* Return the value for the second property.
*/
public Object getB() {
return b;
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
Entry that = (Entry) o;
return a.equals(that.a) && b.equals(that.b);
}
@Override
public int hashCode() {
int result = a.hashCode();
result = 92821 * result + b.hashCode();
return result;
}
}
}
@@ -61,6 +61,8 @@ public class DatabasePlatform {
*/
protected String closeQuote = "\"";
protected String concatOperator = "||";
/**
* When set to true all db column names and table names use quoted identifiers.
*/
@@ -390,8 +392,6 @@ public class DatabasePlatform {
/**
* Return the close quote for quoted identifiers.
*
* @return the close quote
*/
public String getCloseQuote() {
return closeQuote;
@@ -399,17 +399,20 @@ public class DatabasePlatform {
/**
* Return the open quote for quoted identifiers.
*
* @return the open quote
*/
public String getOpenQuote() {
return openQuote;
}
/**
* Return the DB concat operator.
*/
public String getConcatOperator() {
return concatOperator;
}
/**
* Return the JDBC type used to store booleans.
*
* @return the boolean db type
*/
public int getBooleanDbType() {
return booleanDbType;
@@ -1,6 +1,8 @@
package io.ebeaninternal.api;
import io.ebean.Pairs;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -18,16 +20,15 @@ public class NaturalKeyEntry {
* Used when query query just has a series of EQ expressions (no IN clause).
*/
public NaturalKeyEntry(String[] naturalKey, List<NaturalKeyEq> eqList) {
this(naturalKey, eqList, null, null);
load(eqList);
this.key = calculateKey(naturalKey);
}
/**
* Create when query uses an IN clause.
*/
public NaturalKeyEntry(String[] naturalKey, List<NaturalKeyEq> eqList, String inProperty, Object inValue) {
for (NaturalKeyEq eq : eqList) {
map.put(eq.property, eq.value);
}
load(eqList);
if (inProperty != null) {
map.put(inProperty, inValue);
this.inValue = inValue;
@@ -35,6 +36,23 @@ public class NaturalKeyEntry {
this.key = calculateKey(naturalKey);
}
/**
* Create when query uses an IN PAIRS clause.
*/
public NaturalKeyEntry(String[] naturalKey, List<NaturalKeyEq> eqList,
String inMapProperty0, String inMapProperty1, Pairs.Entry pair) {
load(eqList);
map.put(inMapProperty0, pair.getA());
map.put(inMapProperty1, pair.getB());
this.inValue = pair;
this.key = calculateKey(naturalKey);
}
private void load(List<NaturalKeyEq> eqList) {
for (NaturalKeyEq eq : eqList) {
map.put(eq.property, eq.value);
}
}
private Object calculateKey(String[] naturalKey) {
@@ -51,7 +69,7 @@ public class NaturalKeyEntry {
}
/**
* Return the natural cache key.
* Return the natural cache key (String concatenation of values).
*/
public Object key() {
return key;
@@ -1,5 +1,7 @@
package io.ebeaninternal.api;
import io.ebean.Pairs;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashSet;
@@ -13,10 +15,20 @@ public class NaturalKeyQueryData<T> {
private final String[] naturalKey;
private Collection<?> inValues;
/**
* Only one of IN or IN PAIRS is allowed.
*/
private boolean hasIn;
// IN Pairs clause - only one allowed
private String inProperty0, inProperty1;
private List<Pairs.Entry> inPairs;
// IN clause - only one allowed
private Collection<?> inValues;
private String inProperty;
// normal EQ expressions
private List<NaturalKeyEq> eqList;
private NaturalKeySet set;
@@ -37,15 +49,34 @@ public class NaturalKeyQueryData<T> {
return false;
}
/**
* Match for In Pairs expression. We only allow one IN clause.
*/
public boolean matchInPairs(Pairs pairs) {
if (hasIn) {
// only 1 IN allowed (to project naturalIds)
return false;
}
if (matchProperty(pairs.getProperty0()) && matchProperty(pairs.getProperty1())) {
this.hasIn = true;
this.inProperty0 = pairs.getProperty0();
this.inProperty1 = pairs.getProperty1();
this.inPairs = pairs.getEntries();
return true;
}
return false;
}
/**
* Match for IN expression. We only allow one IN clause.
*/
public boolean matchIn(String propName, Collection<?> sourceValues) {
if (inProperty != null) {
if (hasIn) {
// only 1 IN allowed (to project naturalIds)
return false;
}
if (matchProperty(propName)) {
this.hasIn = true;
this.inProperty = propName;
this.inValues = sourceValues;
return true;
@@ -77,16 +108,22 @@ public class NaturalKeyQueryData<T> {
}
this.set = new NaturalKeySet();
if (inValues == null) {
// only one - a findOne()
set.add(new NaturalKeyEntry(naturalKey, eqList));
} else {
if (inValues != null) {
// a findList() with an IN clause so we project
// for every IN value a natural key combination
for (Object inValue : inValues) {
set.add(new NaturalKeyEntry(naturalKey, eqList, inProperty, inValue));
}
} else if (inPairs != null) {
// a findList() with an IN Map clause so we project
// for every IN value a natural key combination
for (Pairs.Entry entry : inPairs) {
set.add(new NaturalKeyEntry(naturalKey, eqList, inProperty0, inProperty1, entry));
}
} else {
// only one - a findOne()
set.add(new NaturalKeyEntry(naturalKey, eqList));
}
return set;
@@ -110,6 +147,12 @@ public class NaturalKeyQueryData<T> {
if (inProperty != null) {
exprProps.add(inProperty);
}
if (inProperty0 != null) {
exprProps.add(inProperty0);
}
if (inProperty1 != null) {
exprProps.add(inProperty1);
}
if (eqList != null) {
for (NaturalKeyEq eq : eqList) {
exprProps.add(eq.property);
@@ -133,6 +176,7 @@ public class NaturalKeyQueryData<T> {
private boolean expressionCount() {
int defined = (inValues == null) ? 0 : 1;
defined += (inPairs == null) ? 0 : 2;
defined += (eqList == null) ? 0 : eqList.size();
return defined == naturalKey.length;
}
@@ -161,6 +205,11 @@ public class NaturalKeyQueryData<T> {
Object naturalKey = hit.getKey();
Object inValue = set.getInValue(naturalKey);
inValues.remove(inValue);
} else if (inPairs != null) {
Object naturalKey = hit.getKey();
Pairs.Entry inValue = (Pairs.Entry)set.getInValue(naturalKey);
inPairs.remove(inValue);
}
beans.add(hit.getBean());
}
@@ -0,0 +1,19 @@
package io.ebeaninternal.server.core;
/**
* Not supported JSON or ARRAY expression handler.
*/
abstract class BaseDbExpression implements DbExpressionHandler {
private final String concatOperator;
BaseDbExpression(String concatOperator) {
this.concatOperator = concatOperator;
}
@Override
public String getConcatOperator() {
return concatOperator;
}
}
@@ -6,7 +6,11 @@ import io.ebeaninternal.server.expression.Op;
/**
* Not supported JSON or ARRAY expression handler.
*/
public class NotSupportedDbExpression implements DbExpressionHandler {
public class BasicDbExpression extends BaseDbExpression {
BasicDbExpression(String concatOperator) {
super(concatOperator);
}
@Override
public void json(SpiExpressionRequest request, String propName, String path, Op operator, Object value) {
@@ -8,6 +8,11 @@ import io.ebeaninternal.server.expression.Op;
*/
public interface DbExpressionHandler {
/**
* Return the DB concat operator (Usually SQL standard "||").
*/
String getConcatOperator();
/**
* Write the db platform specific json expression.
*/
@@ -91,6 +91,8 @@ public class InternalConfiguration {
private final BootupClasses bootupClasses;
private final DatabasePlatform databasePlatform;
private final DeployInherit deployInherit;
private final TypeManager typeManager;
@@ -138,8 +140,8 @@ public class InternalConfiguration {
this.serverConfig = serverConfig;
this.bootupClasses = bootupClasses;
DatabasePlatform databasePlatform = serverConfig.getDatabasePlatform();
this.expressionFactory = initExpressionFactory(serverConfig, databasePlatform);
this.databasePlatform = serverConfig.getDatabasePlatform();
this.expressionFactory = initExpressionFactory(serverConfig);
this.typeManager = new DefaultTypeManager(serverConfig, bootupClasses);
this.multiValueBind = createMultiValueBind(databasePlatform.getPlatform());
@@ -161,7 +163,7 @@ public class InternalConfiguration {
/**
* Create and return the ExpressionFactory based on configuration and database platform.
*/
private ExpressionFactory initExpressionFactory(ServerConfig serverConfig, DatabasePlatform databasePlatform) {
private ExpressionFactory initExpressionFactory(ServerConfig serverConfig) {
boolean nativeIlike = serverConfig.isExpressionNativeIlike() && databasePlatform.isSupportsNativeIlike();
return new DefaultExpressionFactory(serverConfig.isExpressionEqualsWithNullAsNoop(), nativeIlike);
@@ -260,16 +262,17 @@ public class InternalConfiguration {
* Return the JSON expression handler for the given database platform.
*/
private DbExpressionHandler getDbExpressionHandler(DatabasePlatform databasePlatform) {
final Platform platform = databasePlatform.getPlatform();
Platform platform = databasePlatform.getPlatform();
String concatOperator = databasePlatform.getConcatOperator();
switch (platform) {
case POSTGRES:
return new PostgresJsonExpression();
return new PostgresDbExpression(concatOperator);
case ORACLE:
return new OracleDbExpression();
return new OracleDbExpression(concatOperator);
case SQLSERVER:
return new SqlServerJsonExpression();
return new SqlServerDbExpression(concatOperator);
default:
return new NotSupportedDbExpression();
return new BasicDbExpression(concatOperator);
}
}
@@ -6,7 +6,11 @@ import io.ebeaninternal.server.expression.Op;
/**
* Oracle JSON expression handler, ARRAY expressions not supported.
*/
public class OracleDbExpression implements DbExpressionHandler {
public class OracleDbExpression extends BaseDbExpression {
OracleDbExpression(String concatOperator) {
super(concatOperator);
}
@Override
public void json(SpiExpressionRequest request, String propName, String path, Op operator, Object value) {
@@ -6,7 +6,11 @@ import io.ebeaninternal.server.expression.Op;
/**
* Postgres JSON and ARRAY expression handler
*/
public class PostgresJsonExpression implements DbExpressionHandler {
public class PostgresDbExpression extends BaseDbExpression {
PostgresDbExpression(String concatOperator) {
super(concatOperator);
}
@Override
public void json(SpiExpressionRequest request, String propName, String path, Op operator, Object value) {
@@ -6,7 +6,11 @@ import io.ebeaninternal.server.expression.Op;
/**
* Microsoft SQL Server JSON. ARRAY expressions not supported.
*/
public class SqlServerJsonExpression implements DbExpressionHandler {
public class SqlServerDbExpression extends BaseDbExpression {
SqlServerDbExpression(String concatOperator) {
super(concatOperator);
}
@Override
public void json(final SpiExpressionRequest request, final String propName,
@@ -6,6 +6,7 @@ import io.ebean.ExpressionFactory;
import io.ebean.ExpressionList;
import io.ebean.Junction;
import io.ebean.LikeType;
import io.ebean.Pairs;
import io.ebean.Query;
import io.ebean.bean.EntityBean;
import io.ebean.search.Match;
@@ -361,6 +362,14 @@ public class DefaultExpressionFactory implements SpiExpressionFactory {
return new LikeExpression(propertyName, value, true, LikeType.CONTAINS);
}
/**
* In - property has a value in the collection of values.
*/
@Override
public Expression inPairs(Pairs pairs) {
return new InPairsExpression(pairs, false);
}
/**
* In - property has a value in the array of values.
*/
@@ -11,6 +11,7 @@ import io.ebean.FutureRowCount;
import io.ebean.Junction;
import io.ebean.OrderBy;
import io.ebean.PagedList;
import io.ebean.Pairs;
import io.ebean.Query;
import io.ebean.QueryIterator;
import io.ebean.Version;
@@ -21,13 +22,13 @@ import io.ebean.search.TextCommonTerms;
import io.ebean.search.TextQueryString;
import io.ebean.search.TextSimple;
import io.ebeaninternal.api.ManyWhereJoins;
import io.ebeaninternal.api.NaturalKeyQueryData;
import io.ebeaninternal.api.SpiExpression;
import io.ebeaninternal.api.SpiExpressionList;
import io.ebeaninternal.api.SpiExpressionRequest;
import io.ebeaninternal.api.SpiExpressionValidation;
import io.ebeaninternal.api.SpiJunction;
import io.ebeaninternal.server.deploy.BeanDescriptor;
import io.ebeaninternal.api.NaturalKeyQueryData;
import java.io.IOException;
import java.sql.Timestamp;
@@ -789,6 +790,12 @@ public class DefaultExpressionList<T> implements SpiExpressionList<T> {
return this;
}
@Override
public ExpressionList<T> inPairs(Pairs pairs) {
add(expr.inPairs(pairs));
return this;
}
@Override
public ExpressionList<T> in(String propertyName, Query<?> subQuery) {
add(expr.in(propertyName, subQuery));
@@ -0,0 +1,142 @@
package io.ebeaninternal.server.expression;
import io.ebean.Pairs;
import io.ebean.event.BeanQueryRequest;
import io.ebeaninternal.api.NaturalKeyQueryData;
import io.ebeaninternal.api.SpiExpression;
import io.ebeaninternal.api.SpiExpressionRequest;
import io.ebeaninternal.server.persist.MultiValueWrapper;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
class InPairsExpression extends AbstractExpression {
private final boolean not;
private final Pairs pairs;
private final String property0, property1;
private final List<Pairs.Entry> entries;
private boolean multiValueSupported;
private final String separator;
private final String suffix;
private List<Object> concatBindValues;
InPairsExpression(Pairs pairs, boolean not) {
super(pairs.getProperty0());
this.pairs = pairs;
this.property0 = pairs.getProperty0();
this.property1 = pairs.getProperty1();
this.entries = pairs.getEntries();
this.not = not;
this.separator = pairs.getConcatSeparator();
this.suffix = pairs.getConcatSuffix();
}
@Override
public boolean naturalKey(NaturalKeyQueryData data) {
return !not && data.matchInPairs(pairs);
}
@Override
public void prepareExpression(BeanQueryRequest<?> request) {
// at this stage translating pairs into varchar via DB concat
multiValueSupported = request.isMultiValueSupported(String.class);
}
@Override
public void writeDocQuery(DocQueryContext context) throws IOException {
throw new RuntimeException("Not supported with document query");
}
@Override
public void addBindValues(SpiExpressionRequest request) {
// Note at this point entries may have been removed when used with l2 caching
// ... for each l2 cache hit an entry was removed
this.concatBindValues = new ArrayList<>(entries.size());
for (Pairs.Entry entry : entries) {
concatBindValues.add(concat(entry.getA(), entry.getB()));
}
request.addBindValue(new MultiValueWrapper(concatBindValues, String.class));
}
/**
* Using DB concat at this stage. Usually a DB expression index should match the concat.
*/
private String concat(Object key, Object value) {
StringBuilder sb = new StringBuilder(30);
sb.append(key);
sb.append(separator);
sb.append(value);
if (suffix != null) {
sb.append(suffix);
}
return sb.toString();
}
@Override
public void addSql(SpiExpressionRequest request) {
if (entries.isEmpty()) {
String expr = not ? "1=1" : "1=0";
request.append(expr);
return;
}
String concat = request.getDbPlatformHandler().getConcatOperator();
String concatFormula = "(" + property0 + concat + "'" + separator + "'" + concat + property1;
if (suffix != null && !suffix.isEmpty()) {
concatFormula += concat + "'" + suffix + "'";
}
concatFormula += ")";
request.append(concatFormula);
request.appendInExpression(not, concatBindValues);
}
/**
* Based on the number of values in the in clause.
*/
@Override
public void queryPlanHash(StringBuilder builder) {
if (not) {
builder.append("NotInPairs[");
} else {
builder.append("InPairs[");
}
builder.append(property0).append("-");
builder.append(property1).append("-");
builder.append(separator).append("-");
builder.append(suffix).append(" ?");
if (!multiValueSupported) {
// query plan specific to the number of parameters in the IN clause
builder.append(entries.size());
}
builder.append("]");
}
@Override
public int queryBindHash() {
int hc = 92821;
for (Pairs.Entry entry : entries) {
hc = 92821 * hc + entry.hashCode();
}
return hc;
}
@Override
public boolean isSameByBind(SpiExpression other) {
InPairsExpression that = (InPairsExpression) other;
return this.entries.size() == that.entries.size() && entries.equals(that.entries);
}
}
@@ -10,6 +10,7 @@ import io.ebean.FutureRowCount;
import io.ebean.Junction;
import io.ebean.OrderBy;
import io.ebean.PagedList;
import io.ebean.Pairs;
import io.ebean.Query;
import io.ebean.QueryIterator;
import io.ebean.Version;
@@ -20,12 +21,12 @@ import io.ebean.search.TextCommonTerms;
import io.ebean.search.TextQueryString;
import io.ebean.search.TextSimple;
import io.ebeaninternal.api.ManyWhereJoins;
import io.ebeaninternal.api.NaturalKeyQueryData;
import io.ebeaninternal.api.SpiExpression;
import io.ebeaninternal.api.SpiExpressionRequest;
import io.ebeaninternal.api.SpiExpressionValidation;
import io.ebeaninternal.api.SpiJunction;
import io.ebeaninternal.server.deploy.BeanDescriptor;
import io.ebeaninternal.api.NaturalKeyQueryData;
import java.io.IOException;
import java.sql.Timestamp;
@@ -586,6 +587,11 @@ class JunctionExpression<T> implements SpiJunction<T>, SpiExpression, Expression
return exprList.ilike(propertyName, value);
}
@Override
public ExpressionList<T> inPairs(Pairs pairs) {
return exprList.inPairs(pairs);
}
@Override
public ExpressionList<T> in(String propertyName, Collection<?> values) {
return exprList.in(propertyName, values);
@@ -10,6 +10,11 @@ public class MultiValueWrapper {
private final Collection<?> values;
private Class<?> type;
public MultiValueWrapper(Collection<?> values, Class<?> type) {
this.values = values;
this.type = type;
}
public MultiValueWrapper(Collection<?> values) {
this.values = values;
this.type = values.iterator().next().getClass();