#1214 - ENH: Add inPairs() expression - support L2 cache hits for complex natural key with findList() and in pairs expression

src/main/java/io/ebeaninternal/api/NaturalKeyEntry.java

@@ -1,6 +1,8 @@
 package io.ebeaninternal.api;
 
 
+import io.ebean.Pairs;
+
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -18,16 +20,15 @@ public class NaturalKeyEntry {
    * Used when query query just has a series of EQ expressions (no IN clause).
    */
   public NaturalKeyEntry(String[] naturalKey, List<NaturalKeyEq> eqList) {
-    this(naturalKey, eqList, null, null);
+    load(eqList);
+    this.key = calculateKey(naturalKey);
   }
 
   /**
    * Create when query uses an IN clause.
    */
   public NaturalKeyEntry(String[] naturalKey, List<NaturalKeyEq> eqList, String inProperty, Object inValue) {
-    for (NaturalKeyEq eq : eqList) {
-      map.put(eq.property, eq.value);
-    }
+    load(eqList);
     if (inProperty != null) {
       map.put(inProperty, inValue);
       this.inValue = inValue;
@@ -35,6 +36,23 @@ public class NaturalKeyEntry {
     this.key = calculateKey(naturalKey);
   }
 
+  /**
+   * Create when query uses an IN PAIRS clause.
+   */
+  public NaturalKeyEntry(String[] naturalKey, List<NaturalKeyEq> eqList,
+                         String inMapProperty0, String inMapProperty1, Pairs.Entry pair) {
+    load(eqList);
+    map.put(inMapProperty0, pair.getA());
+    map.put(inMapProperty1, pair.getB());
+    this.inValue = pair;
+    this.key = calculateKey(naturalKey);
+  }
+
+  private void load(List<NaturalKeyEq> eqList) {
+    for (NaturalKeyEq eq : eqList) {
+      map.put(eq.property, eq.value);
+    }
+  }
 
   private Object calculateKey(String[] naturalKey) {
 
@@ -51,7 +69,7 @@ public class NaturalKeyEntry {
   }
 
   /**
-   * Return the natural cache key.
+   * Return the natural cache key (String concatenation of values).
    */
   public Object key() {
     return key;

src/main/java/io/ebeaninternal/api/NaturalKeyQueryData.java

@@ -1,5 +1,7 @@
 package io.ebeaninternal.api;
 
+import io.ebean.Pairs;
+
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.HashSet;
@@ -13,10 +15,20 @@ public class NaturalKeyQueryData<T> {
 
   private final String[] naturalKey;
 
-  private Collection<?> inValues;
+  /**
+   * Only one of IN or IN PAIRS is allowed.
+   */
+  private boolean hasIn;
 
+  // IN Pairs clause - only one allowed
+  private String inProperty0, inProperty1;
+  private List<Pairs.Entry> inPairs;
+
+  // IN clause - only one allowed
+  private Collection<?> inValues;
   private String inProperty;
 
+  // normal EQ expressions
   private List<NaturalKeyEq> eqList;
 
   private NaturalKeySet set;
@@ -37,15 +49,34 @@ public class NaturalKeyQueryData<T> {
     return false;
   }
 
+  /**
+   * Match for In Pairs expression. We only allow one IN clause.
+   */
+  public boolean matchInPairs(Pairs pairs) {
+    if (hasIn) {
+      // only 1 IN allowed (to project naturalIds)
+      return false;
+    }
+    if (matchProperty(pairs.getProperty0()) && matchProperty(pairs.getProperty1())) {
+      this.hasIn = true;
+      this.inProperty0 = pairs.getProperty0();
+      this.inProperty1 = pairs.getProperty1();
+      this.inPairs = pairs.getEntries();
+      return true;
+    }
+    return false;
+  }
+
   /**
    * Match for IN expression. We only allow one IN clause.
    */
   public boolean matchIn(String propName, Collection<?> sourceValues) {
-    if (inProperty != null) {
+    if (hasIn) {
       // only 1 IN allowed (to project naturalIds)
       return false;
     }
     if (matchProperty(propName)) {
+      this.hasIn = true;
       this.inProperty = propName;
       this.inValues = sourceValues;
       return true;
@@ -77,16 +108,22 @@ public class NaturalKeyQueryData<T> {
     }
 
     this.set = new NaturalKeySet();
-    if (inValues == null) {
-      // only one - a findOne()
-      set.add(new NaturalKeyEntry(naturalKey, eqList));
-
-    } else {
+    if (inValues != null) {
       // a findList() with an IN clause so we project
       // for every IN value a natural key combination
       for (Object inValue : inValues) {
         set.add(new NaturalKeyEntry(naturalKey, eqList, inProperty, inValue));
       }
+    } else if (inPairs != null) {
+      // a findList() with an IN Map clause so we project
+      // for every IN value a natural key combination
+      for (Pairs.Entry entry : inPairs) {
+        set.add(new NaturalKeyEntry(naturalKey, eqList, inProperty0, inProperty1, entry));
+      }
+
+    } else {
+      // only one - a findOne()
+      set.add(new NaturalKeyEntry(naturalKey, eqList));
     }
 
     return set;
@@ -110,6 +147,12 @@ public class NaturalKeyQueryData<T> {
     if (inProperty != null) {
       exprProps.add(inProperty);
     }
+    if (inProperty0 != null) {
+      exprProps.add(inProperty0);
+    }
+    if (inProperty1 != null) {
+      exprProps.add(inProperty1);
+    }
     if (eqList != null) {
       for (NaturalKeyEq eq : eqList) {
         exprProps.add(eq.property);
@@ -133,6 +176,7 @@ public class NaturalKeyQueryData<T> {
   private boolean expressionCount() {
 
     int defined = (inValues == null) ? 0 : 1;
+    defined += (inPairs == null) ? 0 : 2;
     defined += (eqList == null) ? 0 : eqList.size();
     return defined == naturalKey.length;
   }
@@ -161,6 +205,11 @@ public class NaturalKeyQueryData<T> {
         Object naturalKey = hit.getKey();
         Object inValue = set.getInValue(naturalKey);
         inValues.remove(inValue);
+
+      } else if (inPairs != null) {
+        Object naturalKey = hit.getKey();
+        Pairs.Entry inValue = (Pairs.Entry)set.getInValue(naturalKey);
+        inPairs.remove(inValue);
       }
       beans.add(hit.getBean());
     }

src/main/java/io/ebeaninternal/server/core/BaseDbExpression.java

@@ -0,0 +1,19 @@
+package io.ebeaninternal.server.core;
+
+/**
+ * Not supported JSON or ARRAY expression handler.
+ */
+abstract class BaseDbExpression implements DbExpressionHandler {
+
+  private final String concatOperator;
+
+  BaseDbExpression(String concatOperator) {
+    this.concatOperator = concatOperator;
+  }
+
+  @Override
+  public String getConcatOperator() {
+    return concatOperator;
+  }
+
+}

src/main/java/io/ebeaninternal/server/core/BasicDbExpression.java

@@ -6,7 +6,11 @@ import io.ebeaninternal.server.expression.Op;
 /**
  * Not supported JSON or ARRAY expression handler.
  */
-public class NotSupportedDbExpression implements DbExpressionHandler {
+public class BasicDbExpression extends BaseDbExpression {
+
+  BasicDbExpression(String concatOperator) {
+    super(concatOperator);
+  }
 
   @Override
   public void json(SpiExpressionRequest request, String propName, String path, Op operator, Object value) {

src/main/java/io/ebeaninternal/server/core/DbExpressionHandler.java

@@ -8,6 +8,11 @@ import io.ebeaninternal.server.expression.Op;
  */
 public interface DbExpressionHandler {
 
+  /**
+   * Return the DB concat operator (Usually SQL standard "||").
+   */
+  String getConcatOperator();
+
   /**
    * Write the db platform specific json expression.
    */

src/main/java/io/ebeaninternal/server/core/InternalConfiguration.java

@@ -91,6 +91,8 @@ public class InternalConfiguration {
 
   private final BootupClasses bootupClasses;
 
+  private final DatabasePlatform databasePlatform;
+
   private final DeployInherit deployInherit;
 
   private final TypeManager typeManager;
@@ -138,8 +140,8 @@ public class InternalConfiguration {
     this.serverConfig = serverConfig;
     this.bootupClasses = bootupClasses;
 
-    DatabasePlatform databasePlatform = serverConfig.getDatabasePlatform();
-    this.expressionFactory = initExpressionFactory(serverConfig, databasePlatform);
+    this.databasePlatform = serverConfig.getDatabasePlatform();
+    this.expressionFactory = initExpressionFactory(serverConfig);
     this.typeManager = new DefaultTypeManager(serverConfig, bootupClasses);
 
     this.multiValueBind = createMultiValueBind(databasePlatform.getPlatform());
@@ -161,7 +163,7 @@ public class InternalConfiguration {
   /**
    * Create and return the ExpressionFactory based on configuration and database platform.
    */
-  private ExpressionFactory initExpressionFactory(ServerConfig serverConfig, DatabasePlatform databasePlatform) {
+  private ExpressionFactory initExpressionFactory(ServerConfig serverConfig) {
 
     boolean nativeIlike = serverConfig.isExpressionNativeIlike() && databasePlatform.isSupportsNativeIlike();
     return new DefaultExpressionFactory(serverConfig.isExpressionEqualsWithNullAsNoop(), nativeIlike);
@@ -260,16 +262,17 @@ public class InternalConfiguration {
    * Return the JSON expression handler for the given database platform.
    */
   private DbExpressionHandler getDbExpressionHandler(DatabasePlatform databasePlatform) {
-    final Platform platform = databasePlatform.getPlatform();
+    Platform platform = databasePlatform.getPlatform();
+    String concatOperator = databasePlatform.getConcatOperator();
     switch (platform) {
       case POSTGRES:
-        return new PostgresJsonExpression();
+        return new PostgresDbExpression(concatOperator);
       case ORACLE:
-        return new OracleDbExpression();
+        return new OracleDbExpression(concatOperator);
       case SQLSERVER:
-        return new SqlServerJsonExpression();
+        return new SqlServerDbExpression(concatOperator);
       default:
-        return new NotSupportedDbExpression();
+        return new BasicDbExpression(concatOperator);
     }
   }
 

src/main/java/io/ebeaninternal/server/core/OracleDbExpression.java

@@ -6,7 +6,11 @@ import io.ebeaninternal.server.expression.Op;
 /**
  * Oracle JSON expression handler, ARRAY expressions not supported.
  */
-public class OracleDbExpression implements DbExpressionHandler {
+public class OracleDbExpression extends BaseDbExpression {
+
+  OracleDbExpression(String concatOperator) {
+    super(concatOperator);
+  }
 
   @Override
   public void json(SpiExpressionRequest request, String propName, String path, Op operator, Object value) {

src/main/java/io/ebeaninternal/server/core/PostgresDbExpression.java

@@ -6,7 +6,11 @@ import io.ebeaninternal.server.expression.Op;
 /**
  * Postgres JSON and ARRAY expression handler
  */
-public class PostgresJsonExpression implements DbExpressionHandler {
+public class PostgresDbExpression extends BaseDbExpression {
+
+  PostgresDbExpression(String concatOperator) {
+    super(concatOperator);
+  }
 
   @Override
   public void json(SpiExpressionRequest request, String propName, String path, Op operator, Object value) {

src/main/java/io/ebeaninternal/server/core/SqlServerDbExpression.java

@@ -6,7 +6,11 @@ import io.ebeaninternal.server.expression.Op;
 /**
  * Microsoft SQL Server JSON. ARRAY expressions not supported.
  */
-public class SqlServerJsonExpression implements DbExpressionHandler {
+public class SqlServerDbExpression extends BaseDbExpression {
+
+  SqlServerDbExpression(String concatOperator) {
+    super(concatOperator);
+  }
 
   @Override
   public void json(final SpiExpressionRequest request, final String propName,

src/main/java/io/ebeaninternal/server/expression/DefaultExpressionFactory.java

@@ -6,6 +6,7 @@ import io.ebean.ExpressionFactory;
 import io.ebean.ExpressionList;
 import io.ebean.Junction;
 import io.ebean.LikeType;
+import io.ebean.Pairs;
 import io.ebean.Query;
 import io.ebean.bean.EntityBean;
 import io.ebean.search.Match;
@@ -361,6 +362,14 @@ public class DefaultExpressionFactory implements SpiExpressionFactory {
     return new LikeExpression(propertyName, value, true, LikeType.CONTAINS);
   }
 
+  /**
+   * In - property has a value in the collection of values.
+   */
+  @Override
+  public Expression inPairs(Pairs pairs) {
+    return new InPairsExpression(pairs, false);
+  }
+
   /**
    * In - property has a value in the array of values.
    */

src/main/java/io/ebeaninternal/server/expression/DefaultExpressionList.java

@@ -11,6 +11,7 @@ import io.ebean.FutureRowCount;
 import io.ebean.Junction;
 import io.ebean.OrderBy;
 import io.ebean.PagedList;
+import io.ebean.Pairs;
 import io.ebean.Query;
 import io.ebean.QueryIterator;
 import io.ebean.Version;
@@ -21,13 +22,13 @@ import io.ebean.search.TextCommonTerms;
 import io.ebean.search.TextQueryString;
 import io.ebean.search.TextSimple;
 import io.ebeaninternal.api.ManyWhereJoins;
+import io.ebeaninternal.api.NaturalKeyQueryData;
 import io.ebeaninternal.api.SpiExpression;
 import io.ebeaninternal.api.SpiExpressionList;
 import io.ebeaninternal.api.SpiExpressionRequest;
 import io.ebeaninternal.api.SpiExpressionValidation;
 import io.ebeaninternal.api.SpiJunction;
 import io.ebeaninternal.server.deploy.BeanDescriptor;
-import io.ebeaninternal.api.NaturalKeyQueryData;
 
 import java.io.IOException;
 import java.sql.Timestamp;
@@ -789,6 +790,12 @@ public class DefaultExpressionList<T> implements SpiExpressionList<T> {
     return this;
   }
 
+  @Override
+  public ExpressionList<T> inPairs(Pairs pairs) {
+    add(expr.inPairs(pairs));
+    return this;
+  }
+
   @Override
   public ExpressionList<T> in(String propertyName, Query<?> subQuery) {
     add(expr.in(propertyName, subQuery));

src/main/java/io/ebeaninternal/server/expression/InPairsExpression.java

@@ -0,0 +1,142 @@
+package io.ebeaninternal.server.expression;
+
+import io.ebean.Pairs;
+import io.ebean.event.BeanQueryRequest;
+import io.ebeaninternal.api.NaturalKeyQueryData;
+import io.ebeaninternal.api.SpiExpression;
+import io.ebeaninternal.api.SpiExpressionRequest;
+import io.ebeaninternal.server.persist.MultiValueWrapper;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+class InPairsExpression extends AbstractExpression {
+
+  private final boolean not;
+
+  private final Pairs pairs;
+
+  private final String property0, property1;
+
+  private final List<Pairs.Entry> entries;
+
+  private boolean multiValueSupported;
+
+  private final String separator;
+
+  private final String suffix;
+
+  private List<Object> concatBindValues;
+
+  InPairsExpression(Pairs pairs, boolean not) {
+    super(pairs.getProperty0());
+    this.pairs = pairs;
+    this.property0 = pairs.getProperty0();
+    this.property1 = pairs.getProperty1();
+    this.entries = pairs.getEntries();
+    this.not = not;
+    this.separator = pairs.getConcatSeparator();
+    this.suffix = pairs.getConcatSuffix();
+  }
+
+  @Override
+  public boolean naturalKey(NaturalKeyQueryData data) {
+    return !not && data.matchInPairs(pairs);
+  }
+
+  @Override
+  public void prepareExpression(BeanQueryRequest<?> request) {
+
+    // at this stage translating pairs into varchar via DB concat
+    multiValueSupported = request.isMultiValueSupported(String.class);
+  }
+
+  @Override
+  public void writeDocQuery(DocQueryContext context) throws IOException {
+    throw new RuntimeException("Not supported with document query");
+  }
+
+  @Override
+  public void addBindValues(SpiExpressionRequest request) {
+
+    // Note at this point entries may have been removed when used with l2 caching
+    // ... for each l2 cache hit an entry was removed
+    this.concatBindValues = new ArrayList<>(entries.size());
+    for (Pairs.Entry entry : entries) {
+      concatBindValues.add(concat(entry.getA(), entry.getB()));
+    }
+    request.addBindValue(new MultiValueWrapper(concatBindValues, String.class));
+  }
+
+  /**
+   * Using DB concat at this stage. Usually a DB expression index should match the concat.
+   */
+  private String concat(Object key, Object value) {
+    StringBuilder sb = new StringBuilder(30);
+    sb.append(key);
+    sb.append(separator);
+    sb.append(value);
+    if (suffix != null) {
+      sb.append(suffix);
+    }
+    return sb.toString();
+  }
+
+  @Override
+  public void addSql(SpiExpressionRequest request) {
+
+    if (entries.isEmpty()) {
+      String expr = not ? "1=1" : "1=0";
+      request.append(expr);
+      return;
+    }
+
+    String concat = request.getDbPlatformHandler().getConcatOperator();
+
+    String concatFormula = "(" + property0 + concat + "'" + separator + "'" + concat + property1;
+    if (suffix != null && !suffix.isEmpty()) {
+      concatFormula += concat + "'" + suffix + "'";
+    }
+    concatFormula += ")";
+    request.append(concatFormula);
+    request.appendInExpression(not, concatBindValues);
+  }
+
+  /**
+   * Based on the number of values in the in clause.
+   */
+  @Override
+  public void queryPlanHash(StringBuilder builder) {
+    if (not) {
+      builder.append("NotInPairs[");
+    } else {
+      builder.append("InPairs[");
+    }
+    builder.append(property0).append("-");
+    builder.append(property1).append("-");
+    builder.append(separator).append("-");
+    builder.append(suffix).append(" ?");
+    if (!multiValueSupported) {
+      // query plan specific to the number of parameters in the IN clause
+      builder.append(entries.size());
+    }
+    builder.append("]");
+  }
+
+  @Override
+  public int queryBindHash() {
+    int hc = 92821;
+    for (Pairs.Entry entry : entries) {
+      hc = 92821 * hc + entry.hashCode();
+    }
+    return hc;
+  }
+
+  @Override
+  public boolean isSameByBind(SpiExpression other) {
+
+    InPairsExpression that = (InPairsExpression) other;
+    return this.entries.size() == that.entries.size() && entries.equals(that.entries);
+  }
+}

src/main/java/io/ebeaninternal/server/expression/JunctionExpression.java

@@ -10,6 +10,7 @@ import io.ebean.FutureRowCount;
 import io.ebean.Junction;
 import io.ebean.OrderBy;
 import io.ebean.PagedList;
+import io.ebean.Pairs;
 import io.ebean.Query;
 import io.ebean.QueryIterator;
 import io.ebean.Version;
@@ -20,12 +21,12 @@ import io.ebean.search.TextCommonTerms;
 import io.ebean.search.TextQueryString;
 import io.ebean.search.TextSimple;
 import io.ebeaninternal.api.ManyWhereJoins;
+import io.ebeaninternal.api.NaturalKeyQueryData;
 import io.ebeaninternal.api.SpiExpression;
 import io.ebeaninternal.api.SpiExpressionRequest;
 import io.ebeaninternal.api.SpiExpressionValidation;
 import io.ebeaninternal.api.SpiJunction;
 import io.ebeaninternal.server.deploy.BeanDescriptor;
-import io.ebeaninternal.api.NaturalKeyQueryData;
 
 import java.io.IOException;
 import java.sql.Timestamp;
@@ -586,6 +587,11 @@ class JunctionExpression<T> implements SpiJunction<T>, SpiExpression, Expression
     return exprList.ilike(propertyName, value);
   }
 
+  @Override
+  public ExpressionList<T> inPairs(Pairs pairs) {
+    return exprList.inPairs(pairs);
+  }
+
   @Override
   public ExpressionList<T> in(String propertyName, Collection<?> values) {
     return exprList.in(propertyName, values);

src/main/java/io/ebeaninternal/server/persist/MultiValueWrapper.java

@@ -10,6 +10,11 @@ public class MultiValueWrapper {
   private final Collection<?> values;
   private Class<?> type;
 
+  public MultiValueWrapper(Collection<?> values, Class<?> type) {
+    this.values = values;
+    this.type = type;
+  }
+
   public MultiValueWrapper(Collection<?> values) {
     this.values = values;
     this.type = values.iterator().next().getClass();