From 43afaf1344936e19b466a7f03e991c1fe5376278 Mon Sep 17 00:00:00 2001 From: Robin Bygrave Date: Fri, 5 Aug 2016 13:01:42 +1200 Subject: [PATCH] #811 - ENH: Restore findIterate() as alternative to findEach / findEachWhile --- .../java/com/avaje/ebean/EbeanServer.java | 19 ++++++ .../java/com/avaje/ebean/ExpressionList.java | 7 ++ src/main/java/com/avaje/ebean/Query.java | 50 ++++++++++++++ .../java/com/avaje/ebean/QueryIterator.java | 67 +++++++++++++++++++ .../server/core/DefaultServer.java | 15 ++++- .../server/core/OrmQueryEngine.java | 2 +- .../server/core/OrmQueryRequest.java | 2 +- .../server/core/QueryIterator.java | 60 ----------------- .../server/core/SpiOrmQueryRequest.java | 1 + .../expression/DefaultExpressionList.java | 5 ++ .../server/expression/JunctionExpression.java | 6 ++ .../ebeaninternal/server/query/CQuery.java | 2 +- .../server/query/CQueryEngine.java | 2 +- .../server/query/CQueryIteratorSimple.java | 2 +- .../query/CQueryIteratorWithBuffer.java | 2 +- .../server/query/DefaultOrmQueryEngine.java | 3 +- .../server/querydefn/DefaultOrmQuery.java | 5 ++ .../ebeaninternal/api/TDSpiEbeanServer.java | 5 ++ .../tests/query/TestQueryFindIterate.java | 26 +++++++ 19 files changed, 211 insertions(+), 70 deletions(-) create mode 100644 src/main/java/com/avaje/ebean/QueryIterator.java delete mode 100644 src/main/java/com/avaje/ebeaninternal/server/core/QueryIterator.java diff --git a/src/main/java/com/avaje/ebean/EbeanServer.java b/src/main/java/com/avaje/ebean/EbeanServer.java index 6d13000b3..3490bd203 100644 --- a/src/main/java/com/avaje/ebean/EbeanServer.java +++ b/src/main/java/com/avaje/ebean/EbeanServer.java @@ -761,6 +761,25 @@ public interface EbeanServer { */ List findIds(Query query, Transaction transaction); + /** + * Return a QueryIterator for the query. + *

+ * Generally using {@link #findEach(Query, QueryEachConsumer, Transaction)} or + * {@link #findEachWhile(Query, QueryEachWhileConsumer, Transaction)} is preferred + * to findIterate(). The reason is that those methods automatically take care of + * closing the queryIterator (and the underlying jdbc statement and resultSet). + *

+ *

+ * This is similar to findEach in that not all the result beans need to be held + * in memory at the same time and as such is good for processing large queries. + *

+ * + * @see Query#findIterate() + * @see Query#findEach(QueryEachConsumer) + * @see Query#findEachWhile(QueryEachWhileConsumer) + */ + QueryIterator findIterate(Query query, Transaction transaction); + /** * Execute the query visiting the each bean one at a time. *

diff --git a/src/main/java/com/avaje/ebean/ExpressionList.java b/src/main/java/com/avaje/ebean/ExpressionList.java index 1937ee002..40db32065 100644 --- a/src/main/java/com/avaje/ebean/ExpressionList.java +++ b/src/main/java/com/avaje/ebean/ExpressionList.java @@ -147,6 +147,13 @@ public interface ExpressionList { */ int update(); + /** + * Execute the query iterating over the results. + * + * @see Query#findIterate() + */ + QueryIterator findIterate(); + /** * Execute the query process the beans one at a time. * diff --git a/src/main/java/com/avaje/ebean/Query.java b/src/main/java/com/avaje/ebean/Query.java index 0c60ecd23..47b4661cc 100644 --- a/src/main/java/com/avaje/ebean/Query.java +++ b/src/main/java/com/avaje/ebean/Query.java @@ -544,6 +544,46 @@ public interface Query { */ List findIds(); + /** + * Execute the query iterating over the results. + *

+ * Note that findIterate (and findEach and findEachWhile) uses a "per graph" + * persistence context scope and adjusts jdbc fetch buffer size for large + * queries. As such it is better to use findList for small queries. + *

+ *

+ * Remember that with {@link QueryIterator} you must call {@link QueryIterator#close()} + * when you have finished iterating the results (typically in a finally block). + *

+ *

+ * findEach() and findEachWhile() are preferred to findIterate() as they ensure + * the jdbc statement and resultSet are closed at the end of the iteration. + *

+ *

+ * This query will execute against the EbeanServer that was used to create it. + *

+ *
{@code
+   *
+   *  Query query =
+   *    ebeanServer.find(Customer.class)
+   *     .where().eq("status", Status.NEW)
+   *     .order().asc("id");
+   *
+   *  QueryIterator it = query.findIterate();
+   *  try {
+   *    while (it.hasNext()) {
+   *      Customer customer = it.next();
+   *      // do something with customer ...
+   *    }
+   *  } finally {
+   *    // close the underlying resources
+   *    it.close();
+   *  }
+   *
+   * }
+ */ + QueryIterator findIterate(); + /** * Execute the query processing the beans one at a time. *

@@ -552,6 +592,11 @@ public interface Query { * (unlike #findList #findSet etc) *

*

+ * Note that findEach (and findEachWhile and findIterate) uses a "per graph" + * persistence context scope and adjusts jdbc fetch buffer size for large + * queries. As such it is better to use findList for small queries. + *

+ *

* Note that internally Ebean can inform the JDBC driver that it is expecting larger * resultSet and specifically for MySQL this hint is required to stop it's JDBC driver * from buffering the entire resultSet. As such, for smaller resultSets findList() is @@ -588,6 +633,11 @@ public interface Query { * Execute the query using callbacks to a visitor to process the resulting * beans one at a time. *

+ * Note that findEachWhile (and findEach and findIterate) uses a "per graph" + * persistence context scope and adjusts jdbc fetch buffer size for large + * queries. As such it is better to use findList for small queries. + *

+ *

* This method is functionally equivalent to findIterate() but instead of using an * iterator uses the QueryEachWhileConsumer (SAM) interface which is better suited to use * with Java8 closures. diff --git a/src/main/java/com/avaje/ebean/QueryIterator.java b/src/main/java/com/avaje/ebean/QueryIterator.java new file mode 100644 index 000000000..23bfaea55 --- /dev/null +++ b/src/main/java/com/avaje/ebean/QueryIterator.java @@ -0,0 +1,67 @@ +package com.avaje.ebean; + +import java.util.Iterator; + +/** + * Used to provide iteration over query results. + *

+ * This can be used when you want to process a very large number of results and + * means that you don't have to hold all the results in memory at once (unlike + * findList(), findSet() etc where all the beans are held in the List or Set + * etc). + *

+ *

+ * Note that findIterate (and findEach and findEachWhile) uses a "per graph" + * persistence context scope and adjusts jdbc fetch buffer size for large + * queries. As such it is better to use findList for small queries. + *

+ *

+ * Remember that with {@link QueryIterator} you must call {@link QueryIterator#close()} + * when you have finished iterating the results (typically in a finally block). + *

+ * + *
{@code
+ * 
+ *  Query query = server.find(Customer.class)
+ *     .where().gt("id", 0)
+ *     .orderBy("id")
+ *     .setMaxRows(2);
+ *
+ *  QueryIterator it = query.findIterate();
+ *  try {
+ *    while (it.hasNext()) {
+ *      Customer customer = it.next();
+ *      // do something with customer ...
+ *    }
+ *  } finally {
+ *    // close the underlying resources
+ *    it.close();
+ *  }
+ *
+ * }
+ * + * @param + * the type of entity bean in the iteration + */ +public interface QueryIterator extends Iterator, java.io.Closeable { + + /** + * Returns true if the iteration has more elements. + */ + boolean hasNext(); + + /** + * Returns the next element in the iteration. + */ + T next(); + + /** + * Remove is not allowed. + */ + void remove(); + + /** + * Close the underlying resources held by this iterator. + */ + void close(); +} diff --git a/src/main/java/com/avaje/ebeaninternal/server/core/DefaultServer.java b/src/main/java/com/avaje/ebeaninternal/server/core/DefaultServer.java index 7c3e76a57..a87e07960 100644 --- a/src/main/java/com/avaje/ebeaninternal/server/core/DefaultServer.java +++ b/src/main/java/com/avaje/ebeaninternal/server/core/DefaultServer.java @@ -71,10 +71,8 @@ import javax.persistence.NonUniqueResultException; import javax.persistence.OptimisticLockException; import javax.persistence.PersistenceException; import javax.sql.DataSource; -import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; -import java.util.Collections; import java.util.Iterator; import java.util.List; import java.util.Map; @@ -1304,6 +1302,19 @@ public final class DefaultServer implements SpiServer, SpiEbeanServer { return new LimitOffsetPagedList(this, spiQuery); } + public QueryIterator findIterate(Query query, Transaction t) { + + SpiOrmQueryRequest request = createQueryRequest(Type.ITERATE, query, t); + try { + request.initTransIfRequired(); + return request.findIterate(); + + } catch (RuntimeException ex) { + request.endTransIfRequired(); + throw ex; + } + } + public void findEach(Query query, QueryEachConsumer consumer, Transaction t) { SpiOrmQueryRequest request = createQueryRequest(Type.ITERATE, query, t); diff --git a/src/main/java/com/avaje/ebeaninternal/server/core/OrmQueryEngine.java b/src/main/java/com/avaje/ebeaninternal/server/core/OrmQueryEngine.java index c38ce79ca..83acce984 100644 --- a/src/main/java/com/avaje/ebeaninternal/server/core/OrmQueryEngine.java +++ b/src/main/java/com/avaje/ebeaninternal/server/core/OrmQueryEngine.java @@ -1,8 +1,8 @@ package com.avaje.ebeaninternal.server.core; +import com.avaje.ebean.QueryIterator; import com.avaje.ebean.Version; import com.avaje.ebean.bean.BeanCollection; -import com.avaje.ebeaninternal.api.BeanIdList; import java.util.List; diff --git a/src/main/java/com/avaje/ebeaninternal/server/core/OrmQueryRequest.java b/src/main/java/com/avaje/ebeaninternal/server/core/OrmQueryRequest.java index c9bf91f82..5b1a74340 100644 --- a/src/main/java/com/avaje/ebeaninternal/server/core/OrmQueryRequest.java +++ b/src/main/java/com/avaje/ebeaninternal/server/core/OrmQueryRequest.java @@ -3,6 +3,7 @@ package com.avaje.ebeaninternal.server.core; import com.avaje.ebean.PersistenceContextScope; import com.avaje.ebean.QueryEachConsumer; import com.avaje.ebean.QueryEachWhileConsumer; +import com.avaje.ebean.QueryIterator; import com.avaje.ebean.RawSql; import com.avaje.ebean.Version; import com.avaje.ebean.bean.BeanCollection; @@ -12,7 +13,6 @@ import com.avaje.ebean.event.BeanFindController; import com.avaje.ebean.event.BeanQueryAdapter; import com.avaje.ebean.event.BeanQueryRequest; import com.avaje.ebean.text.json.JsonReadOptions; -import com.avaje.ebeaninternal.api.BeanIdList; import com.avaje.ebeaninternal.api.CQueryPlanKey; import com.avaje.ebeaninternal.api.HashQuery; import com.avaje.ebeaninternal.api.LoadContext; diff --git a/src/main/java/com/avaje/ebeaninternal/server/core/QueryIterator.java b/src/main/java/com/avaje/ebeaninternal/server/core/QueryIterator.java deleted file mode 100644 index 148a7d8c1..000000000 --- a/src/main/java/com/avaje/ebeaninternal/server/core/QueryIterator.java +++ /dev/null @@ -1,60 +0,0 @@ -package com.avaje.ebeaninternal.server.core; - -import java.util.Iterator; - -/** - * Used to provide iteration over query results. - *

- * This can be used when you want to process a very large number of results and - * means that you don't have to hold all the results in memory at once (unlike - * findList(), findSet() etc where all the beans are held in the List or Set - * etc). - *

- * - *
- * 
- * Query<Customer> query = server.find(Customer.class)
- *     .fetch("contacts", new FetchConfig().query(2))
- *     .where().gt("id", 0)
- *     .orderBy("id")
- *     .setMaxRows(2);
- * 
- * QueryIterator<Customer> it = query.findIterate();
- * try {
- *   while (it.hasNext()) {
- *     Customer customer = it.next();
- *     // do something with customer...
- *   }
- * } finally {
- *   // close the associated resources
- *   it.close();
- * }
- * 
- * - * @author rbygrave - * - * @param - * the type of entity bean in the iteration - */ -public interface QueryIterator extends Iterator, java.io.Closeable { - - /** - * Returns true if the iteration has more elements. - */ - boolean hasNext(); - - /** - * Returns the next element in the iteration. - */ - T next(); - - /** - * Remove is not allowed. - */ - void remove(); - - /** - * Close the underlying resources held by this iterator. - */ - void close(); -} diff --git a/src/main/java/com/avaje/ebeaninternal/server/core/SpiOrmQueryRequest.java b/src/main/java/com/avaje/ebeaninternal/server/core/SpiOrmQueryRequest.java index b7ed67df4..7c68e63c8 100644 --- a/src/main/java/com/avaje/ebeaninternal/server/core/SpiOrmQueryRequest.java +++ b/src/main/java/com/avaje/ebeaninternal/server/core/SpiOrmQueryRequest.java @@ -2,6 +2,7 @@ package com.avaje.ebeaninternal.server.core; import com.avaje.ebean.QueryEachConsumer; import com.avaje.ebean.QueryEachWhileConsumer; +import com.avaje.ebean.QueryIterator; import com.avaje.ebean.Version; import com.avaje.ebean.bean.BeanCollection; import com.avaje.ebeaninternal.api.SpiQuery; diff --git a/src/main/java/com/avaje/ebeaninternal/server/expression/DefaultExpressionList.java b/src/main/java/com/avaje/ebeaninternal/server/expression/DefaultExpressionList.java index d5840c3a0..a62f9a7bd 100644 --- a/src/main/java/com/avaje/ebeaninternal/server/expression/DefaultExpressionList.java +++ b/src/main/java/com/avaje/ebeaninternal/server/expression/DefaultExpressionList.java @@ -358,6 +358,11 @@ public class DefaultExpressionList implements SpiExpressionList { return query.findIds(); } + @Override + public QueryIterator findIterate() { + return query.findIterate(); + } + @Override public void findEach(QueryEachConsumer consumer) { query.findEach(consumer); diff --git a/src/main/java/com/avaje/ebeaninternal/server/expression/JunctionExpression.java b/src/main/java/com/avaje/ebeaninternal/server/expression/JunctionExpression.java index 9132b0807..1df86c1e7 100644 --- a/src/main/java/com/avaje/ebeaninternal/server/expression/JunctionExpression.java +++ b/src/main/java/com/avaje/ebeaninternal/server/expression/JunctionExpression.java @@ -12,6 +12,7 @@ import com.avaje.ebean.PagedList; import com.avaje.ebean.Query; import com.avaje.ebean.QueryEachConsumer; import com.avaje.ebean.QueryEachWhileConsumer; +import com.avaje.ebean.QueryIterator; import com.avaje.ebean.Version; import com.avaje.ebean.event.BeanQueryRequest; import com.avaje.ebean.search.Match; @@ -375,6 +376,11 @@ class JunctionExpression implements SpiJunction, SpiExpression, Expression return exprList.findIds(); } + @Override + public QueryIterator findIterate() { + return exprList.findIterate(); + } + @Override public void findEach(QueryEachConsumer consumer) { exprList.findEach(consumer); diff --git a/src/main/java/com/avaje/ebeaninternal/server/query/CQuery.java b/src/main/java/com/avaje/ebeaninternal/server/query/CQuery.java index ca37d7976..d3e1121bd 100644 --- a/src/main/java/com/avaje/ebeaninternal/server/query/CQuery.java +++ b/src/main/java/com/avaje/ebeaninternal/server/query/CQuery.java @@ -1,6 +1,6 @@ package com.avaje.ebeaninternal.server.query; -import com.avaje.ebeaninternal.server.core.QueryIterator; +import com.avaje.ebean.QueryIterator; import com.avaje.ebean.Version; import com.avaje.ebean.bean.BeanCollection; import com.avaje.ebean.bean.EntityBean; diff --git a/src/main/java/com/avaje/ebeaninternal/server/query/CQueryEngine.java b/src/main/java/com/avaje/ebeaninternal/server/query/CQueryEngine.java index bcf3d5da5..49d8625f9 100644 --- a/src/main/java/com/avaje/ebeaninternal/server/query/CQueryEngine.java +++ b/src/main/java/com/avaje/ebeaninternal/server/query/CQueryEngine.java @@ -10,7 +10,7 @@ import com.avaje.ebean.config.dbplatform.DatabasePlatform; import com.avaje.ebeaninternal.api.SpiQuery; import com.avaje.ebeaninternal.server.core.DiffHelp; import com.avaje.ebeaninternal.server.core.OrmQueryRequest; -import com.avaje.ebeaninternal.server.core.QueryIterator; +import com.avaje.ebean.QueryIterator; import com.avaje.ebeaninternal.server.deploy.BeanDescriptor; import com.avaje.ebeaninternal.server.lib.util.Str; import com.avaje.ebeaninternal.server.persist.Binder; diff --git a/src/main/java/com/avaje/ebeaninternal/server/query/CQueryIteratorSimple.java b/src/main/java/com/avaje/ebeaninternal/server/query/CQueryIteratorSimple.java index 2fe591939..88dba323a 100644 --- a/src/main/java/com/avaje/ebeaninternal/server/query/CQueryIteratorSimple.java +++ b/src/main/java/com/avaje/ebeaninternal/server/query/CQueryIteratorSimple.java @@ -4,7 +4,7 @@ import java.sql.SQLException; import javax.persistence.PersistenceException; -import com.avaje.ebeaninternal.server.core.QueryIterator; +import com.avaje.ebean.QueryIterator; import com.avaje.ebeaninternal.server.core.OrmQueryRequest; /** diff --git a/src/main/java/com/avaje/ebeaninternal/server/query/CQueryIteratorWithBuffer.java b/src/main/java/com/avaje/ebeaninternal/server/query/CQueryIteratorWithBuffer.java index fc042d264..163fdfc2f 100644 --- a/src/main/java/com/avaje/ebeaninternal/server/query/CQueryIteratorWithBuffer.java +++ b/src/main/java/com/avaje/ebeaninternal/server/query/CQueryIteratorWithBuffer.java @@ -5,7 +5,7 @@ import java.util.ArrayList; import javax.persistence.PersistenceException; -import com.avaje.ebeaninternal.server.core.QueryIterator; +import com.avaje.ebean.QueryIterator; import com.avaje.ebeaninternal.server.core.OrmQueryRequest; /** diff --git a/src/main/java/com/avaje/ebeaninternal/server/query/DefaultOrmQueryEngine.java b/src/main/java/com/avaje/ebeaninternal/server/query/DefaultOrmQueryEngine.java index 5a0a0112b..493d7030b 100644 --- a/src/main/java/com/avaje/ebeaninternal/server/query/DefaultOrmQueryEngine.java +++ b/src/main/java/com/avaje/ebeaninternal/server/query/DefaultOrmQueryEngine.java @@ -1,11 +1,10 @@ package com.avaje.ebeaninternal.server.query; -import com.avaje.ebeaninternal.server.core.QueryIterator; +import com.avaje.ebean.QueryIterator; import com.avaje.ebean.Version; import com.avaje.ebean.bean.BeanCollection; import com.avaje.ebean.bean.EntityBean; import com.avaje.ebean.event.BeanFindController; -import com.avaje.ebeaninternal.api.BeanIdList; import com.avaje.ebeaninternal.api.SpiQuery; import com.avaje.ebeaninternal.api.SpiTransaction; import com.avaje.ebeaninternal.server.core.OrmQueryEngine; diff --git a/src/main/java/com/avaje/ebeaninternal/server/querydefn/DefaultOrmQuery.java b/src/main/java/com/avaje/ebeaninternal/server/querydefn/DefaultOrmQuery.java index 2adcf19ad..370fd8bc8 100644 --- a/src/main/java/com/avaje/ebeaninternal/server/querydefn/DefaultOrmQuery.java +++ b/src/main/java/com/avaje/ebeaninternal/server/querydefn/DefaultOrmQuery.java @@ -1114,6 +1114,11 @@ public class DefaultOrmQuery implements SpiQuery { server.findEach(this, consumer, null); } + @Override + public QueryIterator findIterate() { + return server.findIterate(this, null); + } + @Override public List> findVersions() { this.temporalMode = TemporalMode.VERSIONS; diff --git a/src/test/java/com/avaje/ebeaninternal/api/TDSpiEbeanServer.java b/src/test/java/com/avaje/ebeaninternal/api/TDSpiEbeanServer.java index e7ea6a3bb..a05d5fc98 100644 --- a/src/test/java/com/avaje/ebeaninternal/api/TDSpiEbeanServer.java +++ b/src/test/java/com/avaje/ebeaninternal/api/TDSpiEbeanServer.java @@ -470,6 +470,11 @@ public class TDSpiEbeanServer implements SpiEbeanServer { return null; } + @Override + public QueryIterator findIterate(Query query, Transaction transaction) { + return null; + } + @Override public void findEach(Query query, QueryEachConsumer consumer, Transaction transaction) { diff --git a/src/test/java/com/avaje/tests/query/TestQueryFindIterate.java b/src/test/java/com/avaje/tests/query/TestQueryFindIterate.java index f03b6da6c..67b6bc466 100644 --- a/src/test/java/com/avaje/tests/query/TestQueryFindIterate.java +++ b/src/test/java/com/avaje/tests/query/TestQueryFindIterate.java @@ -27,6 +27,32 @@ public class TestQueryFindIterate extends BaseTestCase { EbeanServer server = Ebean.getServer(null); + Query query = server.find(Customer.class) + .setMaxRows(2); + + final AtomicInteger count = new AtomicInteger(); + + QueryIterator it = query.findIterate(); + try { + while (it.hasNext()) { + Customer customer = it.next(); + customer.getName(); + count.incrementAndGet(); + } + } finally { + it.close(); + } + + assertEquals(2, count.get()); + } + + @Test + public void findEach() { + + ResetBasicData.reset(); + + EbeanServer server = Ebean.getServer(null); + Query query = server.find(Customer.class) .setAutoTune(false) //.fetch("contacts", new FetchConfig().query(2)).where().gt("id", 0).orderBy("id")