Change such that filterMany can be included in the main query.

That is, currently a filterMany automatically marks that path as a fetchQuery path. This change turns that off and the filterMany path and predicates can then be included in the main query.

This is better for performance (to include the filterMany path in with the main query) when the query is a findEach and looking to return a large number of results.

ebean-core/src/main/java/io/ebeaninternal/api/SpiExpression.java

@@ -105,4 +105,9 @@ public interface SpiExpression extends Expression {
    * Check for match to a natural key query returning false if it doesn't match.
    */
   boolean naturalKey(NaturalKeyQueryData<?> data);
+
+  /**
+   * Apply property prefix when filterMany expressions included into main query.
+   */
+  void prefixProperty(String path);
 }

ebean-core/src/main/java/io/ebeaninternal/api/SpiExpressionList.java

@@ -43,4 +43,9 @@ public interface SpiExpressionList<T> extends ExpressionList<T>, SpiExpression {
   default void applyRowLimits(SpiQuery<?> query) {
     // do nothing by default
   }
+
+  /**
+   * Apply property prefix when filterMany expressions included in main query.
+   */
+  void prefixProperty(String path);
 }

ebean-core/src/main/java/io/ebeaninternal/server/core/OrmQueryRequest.java

@@ -219,10 +219,10 @@ public final class OrmQueryRequest<T> extends BeanRequest implements SpiOrmQuery
    */
   @Override
   public void prepareQuery() {
+    secondaryQueries = query.convertJoins();
     beanDescriptor.prepareQuery(query);
     adapterPreQuery();
-    this.secondaryQueries = query.convertJoins();
-    this.queryPlanKey = query.prepare(this);
+    queryPlanKey = query.prepare(this);
   }
 
   public boolean isNativeSql() {

ebean-core/src/main/java/io/ebeaninternal/server/expression/AbstractExpression.java

@@ -16,12 +16,17 @@ import io.ebeaninternal.api.NaturalKeyQueryData;
  */
 public abstract class AbstractExpression implements SpiExpression {
 
-  protected final String propName;
+  protected String propName;
 
   protected AbstractExpression(String propName) {
     this.propName = propName;
   }
 
+  @Override
+  public void prefixProperty(String path) {
+    this.propName = path + "." + propName;
+  }
+
   @Override
   public boolean naturalKey(NaturalKeyQueryData<?> data) {
     // by default can't use naturalKey cache

ebean-core/src/main/java/io/ebeaninternal/server/expression/BetweenPropertyExpression.java

@@ -17,8 +17,8 @@ class BetweenPropertyExpression extends NonPrepareExpression {
 
   private static final String BETWEEN = " between ";
 
-  private final String lowProperty;
-  private final String highProperty;
+  private String lowProperty;
+  private String highProperty;
   private final Object value;
 
   BetweenPropertyExpression(String lowProperty, String highProperty, Object value) {
@@ -27,6 +27,12 @@ class BetweenPropertyExpression extends NonPrepareExpression {
     this.value = value;
   }
 
+  @Override
+  public void prefixProperty(String path) {
+    this.lowProperty = path + "." + lowProperty;
+    this.highProperty = path + "." + highProperty;
+  }
+
   protected String name(String propName) {
     return propName;
   }

ebean-core/src/main/java/io/ebeaninternal/server/expression/DefaultExampleExpression.java

@@ -88,6 +88,13 @@ public class DefaultExampleExpression implements SpiExpression, ExampleExpressio
     }
   }
 
+  @Override
+  public void prefixProperty(String path) {
+    for (SpiExpression exp : list) {
+      exp.prefixProperty(path);
+    }
+  }
+
   @Override
   public boolean naturalKey(NaturalKeyQueryData<?> data) {
     // can't use naturalKey cache

ebean-core/src/main/java/io/ebeaninternal/server/expression/DefaultExpressionList.java

@@ -126,6 +126,13 @@ public class DefaultExpressionList<T> implements SpiExpressionList<T> {
     }
   }
 
+  @Override
+  public void prefixProperty(String path) {
+    for (SpiExpression exp : list) {
+      exp.prefixProperty(path);
+    }
+  }
+
   @Override
   public Junction<T> toJunction() {
     return new JunctionExpression<>(Junction.Type.FILTER, this);

ebean-core/src/main/java/io/ebeaninternal/server/expression/ExistsQueryExpression.java

@@ -36,6 +36,11 @@ class ExistsQueryExpression implements SpiExpression, UnsupportedDocStoreExpress
     this.subQuery = null;
   }
 
+  @Override
+  public void prefixProperty(String path) {
+    // do nothing
+  }
+
   @Override
   public boolean naturalKey(NaturalKeyQueryData<?> data) {
     // can't use naturalKey cache

ebean-core/src/main/java/io/ebeaninternal/server/expression/IdExpression.java

@@ -19,6 +19,11 @@ class IdExpression extends NonPrepareExpression implements SpiExpression {
     this.value = value;
   }
 
+  @Override
+  public void prefixProperty(String path) {
+    throw new IllegalStateException("Not allowed?");
+  }
+
   @Override
   public void writeDocQuery(DocQueryContext context) throws IOException {
     context.writeId(value);

ebean-core/src/main/java/io/ebeaninternal/server/expression/JunctionExpression.java

@@ -66,6 +66,11 @@ class JunctionExpression<T> implements SpiJunction<T>, SpiExpression, Expression
     this.exprList = exprList;
   }
 
+  @Override
+  public void prefixProperty(String path) {
+    exprList.prefixProperty(path);
+  }
+
   @Override
   public boolean naturalKey(NaturalKeyQueryData<?> data) {
     // can't use naturalKey cache

ebean-core/src/main/java/io/ebeaninternal/server/expression/LogicExpression.java

@@ -57,6 +57,12 @@ abstract class LogicExpression implements SpiExpression {
     this.expTwo = (SpiExpression) expTwo;
   }
 
+  @Override
+  public void prefixProperty(String path) {
+    expOne.prefixProperty(path);
+    expTwo.prefixProperty(path);
+  }
+
   @Override
   public boolean naturalKey(NaturalKeyQueryData<?> data) {
     // can't use naturalKey cache

ebean-core/src/main/java/io/ebeaninternal/server/expression/NestedPathWrapperExpression.java

@@ -24,6 +24,11 @@ class NestedPathWrapperExpression implements SpiExpression {
     this.delegate = delegate;
   }
 
+  @Override
+  public void prefixProperty(String path) {
+    // do nothing
+  }
+
   @Override
   public boolean naturalKey(NaturalKeyQueryData<?> data) {
     // can't use naturalKey cache

ebean-core/src/main/java/io/ebeaninternal/server/expression/NonPrepareExpression.java

@@ -9,6 +9,11 @@ import io.ebeaninternal.api.NaturalKeyQueryData;
  */
 abstract class NonPrepareExpression implements SpiExpression {
 
+  @Override
+  public void prefixProperty(String path) {
+    // do nothing
+  }
+
   @Override
   public boolean naturalKey(NaturalKeyQueryData<?> data) {
     // can't use naturalKey cache

ebean-core/src/main/java/io/ebeaninternal/server/expression/NoopExpression.java

@@ -17,6 +17,11 @@ class NoopExpression implements SpiExpression {
 
   protected static final NoopExpression INSTANCE = new NoopExpression();
 
+  @Override
+  public void prefixProperty(String path) {
+    // do nothing
+  }
+
   @Override
   public boolean naturalKey(NaturalKeyQueryData<?> data) {
     // can't use naturalKey cache

ebean-core/src/main/java/io/ebeaninternal/server/expression/NotExpression.java

@@ -22,6 +22,11 @@ final class NotExpression implements SpiExpression {
     this.exp = (SpiExpression) exp;
   }
 
+  @Override
+  public void prefixProperty(String path) {
+    exp.prefixProperty(path);
+  }
+
   @Override
   public boolean naturalKey(NaturalKeyQueryData<?> data) {
     // can't use naturalKey cache

ebean-core/src/main/java/io/ebeaninternal/server/querydefn/DefaultOrmQuery.java

@@ -1238,6 +1238,7 @@ public class DefaultOrmQuery<T> implements SpiQuery<T> {
    * Prepare the expressions (compile sub-queries etc).
    */
   private void prepareExpressions(BeanQueryRequest<?> request) {
+    detail.prepareExpressions(request);
     if (whereExpressions != null) {
       whereExpressions.prepareExpression(request);
     }

ebean-core/src/main/java/io/ebeaninternal/server/querydefn/OrmQueryDetail.java

@@ -1,6 +1,7 @@
 package io.ebeaninternal.server.querydefn;
 
 import io.ebean.FetchConfig;
+import io.ebean.event.BeanQueryRequest;
 import io.ebean.util.SplitName;
 import io.ebeaninternal.server.deploy.BeanDescriptor;
 import io.ebeaninternal.server.deploy.BeanPropertyAssoc;
@@ -9,25 +10,16 @@ import io.ebeaninternal.server.el.ElPropertyValue;
 
 import javax.persistence.PersistenceException;
 import java.io.Serializable;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Iterator;
-import java.util.LinkedHashMap;
-import java.util.LinkedHashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
+import java.util.*;
 
 /**
  * Represents the internal structure of an Object Relational query.
  * <p>
  * Holds the select() and join() details of a ORM query.
- * </p>
  * <p>
  * It is worth noting that for AutoTune a "tuned fetch info" builds an instance of OrmQueryDetail.
  * Tuning a query is a matter of replacing an instance of this class with one that has been tuned
  * with select() and join() set.
- * </p>
  */
 public class OrmQueryDetail implements Serializable {
 
@@ -81,7 +73,6 @@ public class OrmQueryDetail implements Serializable {
    * Return true if equal in terms of autoTune (select and fetch without property ordering).
    */
   public boolean isAutoTuneEqual(OrmQueryDetail otherDetail) {
-
     if (!isSameByAutoTune(baseProps, otherDetail.baseProps)) {
       return false;
     }
@@ -175,9 +166,7 @@ public class OrmQueryDetail implements Serializable {
   }
 
   private List<OrmQueryProperties> removeSecondaryQueries(boolean lazyQuery) {
-
     ArrayList<String> matchingPaths = new ArrayList<>(2);
-
     for (OrmQueryProperties chunk : fetchPaths.values()) {
       boolean match = lazyQuery ? chunk.isLazyFetch() : chunk.isQueryFetch();
       if (match) {
@@ -191,7 +180,6 @@ public class OrmQueryDetail implements Serializable {
 
     // sort into depth order to remove
     Collections.sort(matchingPaths);
-
     // the list of secondary queries
     ArrayList<OrmQueryProperties> props = new ArrayList<>();
 
@@ -225,14 +213,11 @@ public class OrmQueryDetail implements Serializable {
       OrmQueryProperties chunk = getChunk(split[0], true);
       chunk.addSecondaryQueryJoin(split[1]);
     }
-
     return props;
   }
 
   boolean tuneFetchProperties(OrmQueryDetail tunedDetail) {
-
     boolean tuned = false;
-
     OrmQueryProperties tunedRoot = tunedDetail.getChunk(null, false);
     if (tunedRoot != null) {
       tuned = true;
@@ -257,10 +242,7 @@ public class OrmQueryDetail implements Serializable {
   }
 
   /**
-   * Remove all joins and properties.
-   * <p>
-   * Typically for the row count query.
-   * </p>
+   * Remove all joins and properties. Typically for the row count query.
    */
   public void clear() {
     fetchPaths.clear();
@@ -306,7 +288,6 @@ public class OrmQueryDetail implements Serializable {
   }
 
   private void sortFetchPaths(BeanDescriptor<?> d, boolean addIds) {
-
     if (!fetchPaths.isEmpty()) {
       LinkedHashMap<String, OrmQueryProperties> sorted = new LinkedHashMap<>();
       for (OrmQueryProperties p : fetchPaths.values()) {
@@ -317,7 +298,6 @@ public class OrmQueryDetail implements Serializable {
   }
 
   private void sortFetchPaths(BeanDescriptor<?> d, OrmQueryProperties p, LinkedHashMap<String, OrmQueryProperties> sorted, boolean addId) {
-
     String path = p.getPath();
     if (!sorted.containsKey(path)) {
       String parentPath = p.getParentPath();
@@ -339,7 +319,6 @@ public class OrmQueryDetail implements Serializable {
             parentProp = new OrmQueryProperties(parentPath, Collections.emptySet());
           }
         }
-
         sortFetchPaths(d, parentProp, sorted, addId);
         sorted.put(path, p);
       }
@@ -350,14 +329,12 @@ public class OrmQueryDetail implements Serializable {
    * Mark 'fetch joins' to 'many' properties over to 'query joins' where needed.
    */
   void markQueryJoins(BeanDescriptor<?> beanDescriptor, String lazyLoadManyPath, boolean allowOne, boolean addIds) {
-
     if (fetchPaths.isEmpty()) {
       return;
     }
 
     // the name of the many fetch property if there is one
     String manyFetchProperty = null;
-
     // flag that is set once the many fetch property is chosen
     boolean fetchJoinFirstMany = allowOne;
 
@@ -375,6 +352,7 @@ public class OrmQueryDetail implements Serializable {
             // letting the first one remain a 'fetch join'
             fetchJoinFirstMany = false;
             manyFetchProperty = pair.getPath();
+            chunk.filterManyInline();
           } else {
             // convert this one over to a 'query join'
             chunk.markForQueryJoin();
@@ -388,7 +366,6 @@ public class OrmQueryDetail implements Serializable {
    * Sort the fetch entries taking into account fetchPreference on the path.
    */
   private List<FetchEntry> sortByFetchPreference(BeanDescriptor<?> desc) {
-
     List<FetchEntry> entries = new ArrayList<>(fetchPaths.size());
     int idx = 0;
     for (Map.Entry<String, OrmQueryProperties> entry : fetchPaths.entrySet()) {
@@ -452,11 +429,9 @@ public class OrmQueryDetail implements Serializable {
    * </p>
    */
   public void setDefaultSelectClause(BeanDescriptor<?> desc) {
-
     if (desc.hasDefaultSelectClause() && !hasSelectClause()) {
       baseProps = new OrmQueryProperties(null, desc.getDefaultSelectClause());
     }
-
     for (OrmQueryProperties joinProps : fetchPaths.values()) {
       if (!joinProps.hasSelectClause()) {
         BeanDescriptor<?> assocDesc = desc.getBeanDescriptor(joinProps.getPath());
@@ -506,7 +481,6 @@ public class OrmQueryDetail implements Serializable {
       props = new OrmQueryProperties(path);
       fetch(props);
       return props;
-
     } else {
       return props;
     }
@@ -516,9 +490,7 @@ public class OrmQueryDetail implements Serializable {
    * Return true if the fetch path is included.
    */
   public boolean includesPath(String path) {
-
     OrmQueryProperties chunk = fetchPaths.get(path);
-
     // may not have fetch properties if just +cache etc
     return chunk != null && !chunk.isCache();
   }
@@ -537,6 +509,15 @@ public class OrmQueryDetail implements Serializable {
     return fetchPaths.entrySet();
   }
 
+  /**
+   * Prepare filterMany expressions that are being included into the main query.
+   */
+  public void prepareExpressions(BeanQueryRequest<?> request) {
+    for (OrmQueryProperties value : fetchPaths.values()) {
+      value.prepareExpressions(request);
+    }
+  }
+
   private static class FetchEntry implements Comparable<FetchEntry> {
 
     private final int index;

ebean-core/src/main/java/io/ebeaninternal/server/querydefn/OrmQueryProperties.java

@@ -4,6 +4,7 @@ import io.ebean.ExpressionFactory;
 import io.ebean.FetchConfig;
 import io.ebean.OrderBy;
 import io.ebean.Query;
+import io.ebean.event.BeanQueryRequest;
 import io.ebean.util.SplitName;
 import io.ebeaninternal.api.SpiExpression;
 import io.ebeaninternal.api.SpiExpressionFactory;
@@ -178,8 +179,6 @@ public class OrmQueryProperties implements Serializable {
       SpiExpressionFactory queryEf = (SpiExpressionFactory) rootQuery.getExpressionFactory();
       ExpressionFactory filterEf = queryEf.createExpressionFactory();// exprPath);
       filterMany = new FilterExpressionList(exprPath, filterEf, rootQuery);
-      // by default we need to make this a 'query join' now
-      markForQueryJoin = true;
     }
     return filterMany;
   }
@@ -194,6 +193,24 @@ public class OrmQueryProperties implements Serializable {
     return filterMany.trimPath(trimPath);
   }
 
+  /**
+   * Adjust filterMany expressions for inclusion in main query.
+   */
+  public void filterManyInline() {
+    if (filterMany != null){
+      filterMany.prefixProperty(path);
+    }
+  }
+
+  /**
+   * Prepare filterMany expressions for query plan key.
+   */
+  public void prepareExpressions(BeanQueryRequest<?> request) {
+    if (filterMany != null) {
+      filterMany.prepareExpression(request);
+    }
+  }
+
   /**
    * Return the filterMany expression list (can be null).
    */
@@ -206,7 +223,6 @@ public class OrmQueryProperties implements Serializable {
    */
   public void setFilterMany(SpiExpressionList<?> filterMany) {
     this.filterMany = filterMany;
-    this.markForQueryJoin = true;
   }
 
   /**

ebean-core/src/test/java/io/ebeaninternal/server/core/DefaultServer_createOrmQueryRequestTest.java

@@ -288,7 +288,7 @@ public class DefaultServer_createOrmQueryRequestTest extends BaseTestCase {
   }
 
   @Test
-  public void test_removeJoinToMany_when_filterMany() {
+  public void test_filterMany_included() {
 
     Query<Order> query = Ebean.find(Order.class)
       .fetch("details")
@@ -301,6 +301,40 @@ public class DefaultServer_createOrmQueryRequestTest extends BaseTestCase {
     OrmQueryRequest<Order> queryRequest = queryRequest(query);
     OrmQueryDetail detail = queryRequest.getQuery().getDetail();
 
+    assertThat(detail.getFetchPaths()).containsExactly("details", "details.product", "customer");
+  }
+
+  @Test
+  public void test_filterMany_excludedByOrdering() {
+
+    Query<Order> query = Ebean.find(Order.class)
+      .fetch("customer")
+      .fetch("customer.contacts")
+      .fetch("details")
+      .fetch("details.product")
+      .filterMany("details").eq("orderQuantity", 10)
+      .query();
+
+    OrmQueryRequest<Order> queryRequest = queryRequest(query);
+    OrmQueryDetail detail = queryRequest.getQuery().getDetail();
+
+    assertThat(detail.getFetchPaths()).containsExactly("customer", "customer.contacts");
+  }
+
+  @Test
+  public void test_filterMany_excludedExplicitly() {
+
+    Query<Order> query = Ebean.find(Order.class)
+      .fetchQuery("details")
+      .fetch("details.product")
+      .fetch("customer")
+      .fetch("customer.contacts")
+      .filterMany("details").eq("orderQuantity", 10)
+      .query();
+
+    OrmQueryRequest<Order> queryRequest = queryRequest(query);
+    OrmQueryDetail detail = queryRequest.getQuery().getDetail();
+
     assertThat(detail.getFetchPaths()).containsExactly("customer", "customer.contacts");
   }
 

ebean-core/src/test/java/org/tests/query/TestQueryFilterMany.java

@@ -172,7 +172,6 @@ public class TestQueryFilterMany extends BaseTestCase {
     LoggedSqlCollector.start();
 
     Query<Customer> query = Ebean.find(Customer.class)
-      .fetch("orders")
       .filterMany("orders").raw("1=0")
       .where().isNotEmpty("orders")
       .query();
@@ -183,12 +182,10 @@ public class TestQueryFilterMany extends BaseTestCase {
     }
 
     List<String> sqlList = LoggedSqlCollector.stop();
-    assertEquals(2, sqlList.size());
-    assertThat(sqlList.get(0)).contains("where exists (select 1 from o_order x where x.kcustomer_id = t0.id)");
-    assertThat(sqlList.get(1)).contains("and 1=0");
+    assertEquals(1, sqlList.size());
+    assertThat(sqlList.get(0)).contains("from o_customer t0 left join o_order t1 on t1.kcustomer_id = t0.id and t1.order_date is not null left join o_customer t2 on t2.id = t1.kcustomer_id where exists (select 1 from o_order x where x.kcustomer_id = t0.id) and 1=0 order by t0.id");
   }
 
-
   @Test
   public void test_filterMany_in_findCount() {
 
@@ -212,7 +209,6 @@ public class TestQueryFilterMany extends BaseTestCase {
   public void test_filterMany_copy_findList() {
 
     ResetBasicData.reset();
-
     LoggedSqlCollector.start();
 
     Query<Customer> query = Ebean.find(Customer.class)
@@ -222,17 +218,35 @@ public class TestQueryFilterMany extends BaseTestCase {
 
     query.copy().findList();
 
+    List<String> sqlList = LoggedSqlCollector.stop();
+    assertEquals(1, sqlList.size());
+    assertThat(sqlList.get(0)).contains("from o_customer t0 left join o_order t1 on t1.kcustomer_id = t0.id and t1.order_date is not null left join o_customer t2 on t2.id = t1.kcustomer_id where t1.status in (?) order by t0.id");
+  }
+
+  @Test
+  public void test_filterMany_fetchQuery() {
+
+    ResetBasicData.reset();
+    LoggedSqlCollector.start();
+
+    Query<Customer> query = Ebean.find(Customer.class)
+      .fetchQuery("orders") // explicitly fetch orders separately
+      .filterMany("orders").in("status", Order.Status.NEW)
+      .order().asc("id");
+
+    query.findList();
+
     List<String> sqlList = LoggedSqlCollector.stop();
     assertEquals(2, sqlList.size());
     assertThat(sqlList.get(0)).contains("from o_customer t0");
-    assertThat(sqlList.get(1)).contains("from o_order t0 join o_customer t1");
+    assertThat(sqlList.get(1)).contains("from o_order t0 join o_customer t1 on t1.id = t0.kcustomer_id where t0.order_date is not null and (t0.kcustomer_id) in ");
+    assertThat(sqlList.get(1)).contains(" and t0.status in ");
   }
 
   @Test
   public void testDisjunction() {
 
     ResetBasicData.reset();
-
     LoggedSqlCollector.start();
 
     Ebean.find(Customer.class)
@@ -243,8 +257,8 @@ public class TestQueryFilterMany extends BaseTestCase {
       .findList();
 
     List<String> sql = LoggedSqlCollector.stop();
-    assertEquals(2, sql.size());
-    assertSql(sql.get(1)).contains("and (t0.status = ? or t0.order_date = ?");
+    assertEquals(1, sql.size());
+    assertSql(sql.get(0)).contains(" from o_customer t0 left join o_order t1 on t1.kcustomer_id = t0.id and t1.order_date is not null left join o_customer t2 on t2.id = t1.kcustomer_id where (t1.status = ? or t1.order_date = ?) order by t0.id");
   }
 
   @Test
@@ -259,12 +273,10 @@ public class TestQueryFilterMany extends BaseTestCase {
 
     List<String> sql = LoggedSqlCollector.stop();
 
-    assertThat(sql).hasSize(3);
-    assertSql(sql.get(0)).contains(" from o_customer t0; --bind()");
-    platformAssertIn(sql.get(1), " from contact t0 where (t0.customer_id)");
-    assertSql(sql.get(1)).contains(" and t0.first_name is not null");
-    platformAssertIn(sql.get(2), " from contact_note t0 where (t0.contact_id)");
-    assertSql(sql.get(2)).contains(" and lower(t0.title) like");
+    assertThat(sql).hasSize(2);
+    assertSql(sql.get(0)).contains(" from o_customer t0 left join contact t1 on t1.customer_id = t0.id where t1.first_name is not null order by t0.id; --bind()");
+    platformAssertIn(sql.get(1), " from contact_note t0 where (t0.contact_id)");
+    assertSql(sql.get(1)).contains(" and lower(t0.title) like");
   }
 
   @Test
@@ -280,9 +292,7 @@ public class TestQueryFilterMany extends BaseTestCase {
 
     List<String> sql = LoggedSqlCollector.stop();
 
-    assertThat(sql).hasSize(2);
-    assertSql(sql.get(0)).contains(" from o_customer t0");
-    assertSql(sql.get(1)).contains("from contact t0 where ");
-    assertSql(sql.get(1)).contains("and (t0.first_name is not null and lower(t0.email) like ?");
+    assertThat(sql).hasSize(1);
+    assertSql(sql.get(0)).contains(" from o_customer t0 left join contact t1 on t1.customer_id = t0.id where (t1.first_name is not null and lower(t1.email) like ? escape'|' ) order by t0.id; --bind(rob%)");
   }
 }