#1334 - ENH: Convert ORM query into DTO query ... with dynamic formula support

src/main/java/io/ebean/Query.java

@@ -236,6 +236,14 @@ public interface Query<T> {
    */
   Query<T> asDraft();
 
+  /**
+   * Convert the query to a DTO bean query.
+   * <p>
+   * We effectively use the underlying ORM query to build the SQL and then execute
+   * and map it into DTO beans.
+   */
+  <D> DtoQuery<D> asDto(Class<D> dtoClass);
+
   /**
    * Cancel the query execution if supported by the underlying database and
    * driver.

src/main/java/io/ebeaninternal/api/SpiDtoQuery.java

@@ -17,7 +17,7 @@ public interface SpiDtoQuery<T> extends DtoQuery<T>, SpiSqlBinding {
   /**
    * Get the query plan for the cache.
    */
-  DtoQueryPlan getQueryPlan(String planKey);
+  DtoQueryPlan getQueryPlan(Object planKey);
 
   /**
    * Build the query plan.
@@ -27,7 +27,7 @@ public interface SpiDtoQuery<T> extends DtoQuery<T>, SpiSqlBinding {
   /**
    * Put the query plan into the cache.
    */
-  void putQueryPlan(String planKey, DtoQueryPlan plan);
+  void putQueryPlan(Object planKey, DtoQueryPlan plan);
 
   /**
    * Return true if the query is in relaxed mapping mode.
@@ -43,4 +43,10 @@ public interface SpiDtoQuery<T> extends DtoQuery<T>, SpiSqlBinding {
    * Return the associated DTO bean type.
    */
   Class<T> getType();
+
+  /**
+   * Return an underlying ORM query (if this query is built from an ORM query).
+   */
+  SpiQuery<?> getOrmQuery();
+
 }

src/main/java/io/ebeaninternal/api/SpiEbeanServer.java

@@ -1,5 +1,6 @@
 package io.ebeaninternal.api;
 
+import io.ebean.DtoQuery;
 import io.ebean.EbeanServer;
 import io.ebean.PersistenceContextScope;
 import io.ebean.Query;
@@ -14,6 +15,7 @@ import io.ebean.config.dbplatform.DatabasePlatform;
 import io.ebean.event.readaudit.ReadAuditLogger;
 import io.ebean.event.readaudit.ReadAuditPrepare;
 import io.ebeaninternal.dbmigration.ddlgeneration.DdlHandler;
+import io.ebeaninternal.server.core.SpiResultSet;
 import io.ebeaninternal.server.core.timezone.DataTimeZone;
 import io.ebeaninternal.server.deploy.BeanDescriptor;
 import io.ebeaninternal.server.query.CQuery;
@@ -246,4 +248,13 @@ public interface SpiEbeanServer extends EbeanServer, BeanLoader, BeanCollectionL
    */
   <T> void findDtoEachWhile(SpiDtoQuery<T> query, Predicate<T> consumer);
 
+  /**
+   * Return / wrap the ORM query as a DTO query.
+   */
+  <D> DtoQuery<D> findDto(Class<D> dtoType, SpiQuery<?> ormQuery);
+
+  /**
+   * Execute the underlying ORM query returning as a JDBC ResultSet to map to DTO beans.
+   */
+  SpiResultSet findResultSet(SpiQuery<?> ormQuery, SpiTransaction transaction);
 }

src/main/java/io/ebeaninternal/api/SpiQuery.java

@@ -215,6 +215,11 @@ public interface SpiQuery<T> extends Query<T>, TxnProfileEventCodes {
    */
   BeanDescriptor<T> getBeanDescriptor();
 
+  /**
+   * Return the query plan key.
+   */
+  Object getQueryPlanKey();
+
   /**
    * Return the RawSql that was set to use for this query.
    */
@@ -748,6 +753,16 @@ public interface SpiQuery<T> extends Query<T>, TxnProfileEventCodes {
    */
   boolean isDistinct();
 
+  /**
+   * Return true if the Id property is manually included in the query (DTO queries).
+   */
+  boolean isManualId();
+
+  /**
+   * Set to true when we only include the Id property if it is explicitly included in the select().
+   */
+  void setManualId(boolean manualId);
+
   /**
    * Set default select clauses where none have been explicitly defined.
    */

src/main/java/io/ebeaninternal/server/core/AbstractSqlQueryRequest.java

@@ -5,6 +5,7 @@ import io.ebean.Transaction;
 import io.ebean.util.JdbcClose;
 import io.ebeaninternal.api.BindParams;
 import io.ebeaninternal.api.SpiEbeanServer;
+import io.ebeaninternal.api.SpiQuery;
 import io.ebeaninternal.api.SpiSqlBinding;
 import io.ebeaninternal.api.SpiTransaction;
 import io.ebeaninternal.server.lib.util.Str;
@@ -25,7 +26,7 @@ public abstract class AbstractSqlQueryRequest {
 
   private final SpiSqlBinding query;
 
-  protected final SpiEbeanServer ebeanServer;
+  protected final SpiEbeanServer server;
 
   protected SpiTransaction trans;
 
@@ -45,7 +46,7 @@ public abstract class AbstractSqlQueryRequest {
    * Create the BeanFindRequest.
    */
   AbstractSqlQueryRequest(SpiEbeanServer server, SpiSqlBinding query, Transaction t) {
-    this.ebeanServer = server;
+    this.server = server;
     this.query = query;
     this.trans = (SpiTransaction) t;
   }
@@ -55,10 +56,10 @@ public abstract class AbstractSqlQueryRequest {
    */
   public void initTransIfRequired() {
     if (trans == null) {
-      trans = ebeanServer.currentServerTransaction();
+      trans = server.currentServerTransaction();
       if (trans == null || !trans.isActive()) {
         // create a local readOnly transaction
-        trans = ebeanServer.createQueryTransaction(null);
+        trans = server.createQueryTransaction(null);
         createdTransaction = true;
       }
     }
@@ -73,8 +74,8 @@ public abstract class AbstractSqlQueryRequest {
     }
   }
 
-  public EbeanServer getEbeanServer() {
-    return ebeanServer;
+  public EbeanServer getServer() {
+    return server;
   }
 
   public SpiTransaction getTransaction() {
@@ -85,7 +86,10 @@ public abstract class AbstractSqlQueryRequest {
     return trans.isLogSql();
   }
 
-  abstract void setResultSet(ResultSet resultSet) throws SQLException;
+  /**
+   * Set the resultSet and associated query plan if known.
+   */
+  abstract void setResultSet(ResultSet resultSet, Object queryPlanKey) throws SQLException;
 
   /**
    * Return the bindLog for this request.
@@ -132,7 +136,7 @@ public abstract class AbstractSqlQueryRequest {
     int firstRow = query.getFirstRow();
     int maxRows = query.getMaxRows();
     if (firstRow > 0 || maxRows > 0) {
-      return ebeanServer.getDatabasePlatform().getBasicSqlLimiter().limit(sql, firstRow, maxRows);
+      return server.getDatabasePlatform().getBasicSqlLimiter().limit(sql, firstRow, maxRows);
     }
     return sql;
   }
@@ -140,7 +144,11 @@ public abstract class AbstractSqlQueryRequest {
   /**
    * Prepare and execute the SQL using the Binder.
    */
-  public void executeSql(Binder binder) throws SQLException {
+  public void executeSql(Binder binder, SpiQuery.Type type) throws SQLException {
+    executeAsSql(binder);
+  }
+
+  protected void executeAsSql(Binder binder) throws SQLException {
 
     startNano = System.nanoTime();
 
@@ -169,7 +177,7 @@ public abstract class AbstractSqlQueryRequest {
       trans.logSql(logSql);
     }
 
-    setResultSet(pstmt.executeQuery());
+    setResultSet(pstmt.executeQuery(), null);
   }
 
   /**

src/main/java/io/ebeaninternal/server/core/DefaultServer.java

@@ -985,6 +985,21 @@ public final class DefaultServer implements SpiServer, SpiEbeanServer {
     return new DefaultDtoQuery<>(this, descriptor, sql.trim());
   }
 
+  @Override
+  public <T> DtoQuery<T> findDto(Class<T> dtoType, SpiQuery<?> ormQuery) {
+
+    DtoBeanDescriptor<T> descriptor = dtoBeanManager.getDescriptor(dtoType);
+    return new DefaultDtoQuery<>(this, descriptor, ormQuery);
+  }
+
+  @Override
+  public SpiResultSet findResultSet(SpiQuery<?> ormQuery, SpiTransaction transaction) {
+
+    SpiOrmQueryRequest<?> request = createQueryRequest(ormQuery.getType(), ormQuery, transaction);
+    request.initTransIfRequired();
+    return request.findResultSet();
+  }
+
   @Override
   public SqlQuery createSqlQuery(String sql) {
     return new DefaultRelationalQuery(this, sql.trim());

src/main/java/io/ebeaninternal/server/core/DtoQueryRequest.java

@@ -2,9 +2,11 @@ package io.ebeaninternal.server.core;
 
 import io.ebeaninternal.api.SpiDtoQuery;
 import io.ebeaninternal.api.SpiEbeanServer;
+import io.ebeaninternal.api.SpiQuery;
 import io.ebeaninternal.server.dto.DtoColumn;
 import io.ebeaninternal.server.dto.DtoMappingRequest;
 import io.ebeaninternal.server.dto.DtoQueryPlan;
+import io.ebeaninternal.server.persist.Binder;
 import io.ebeaninternal.server.query.dto.DtoQueryEngine;
 import io.ebeaninternal.server.type.DataReader;
 import io.ebeaninternal.server.type.RsetDataReader;
@@ -37,15 +39,39 @@ public final class DtoQueryRequest<T> extends AbstractSqlQueryRequest {
     this.query = query;
   }
 
+  /**
+   * Prepare and execute the SQL using the Binder.
+   */
   @Override
-  protected void setResultSet(ResultSet resultSet) throws SQLException {
-    this.resultSet = resultSet;
-    this.dataReader = new RsetDataReader(ebeanServer.getDataTimeZone(), resultSet);
-    obtainPlan();
+  public void executeSql(Binder binder, SpiQuery.Type type) throws SQLException {
+    SpiQuery<?> ormQuery = query.getOrmQuery();
+    if (ormQuery != null) {
+      ormQuery.setType(type);
+      ormQuery.setManualId(true);
+
+      // execute the underlying ORM query returning the ResultSet
+      SpiResultSet result = server.findResultSet(ormQuery, trans);
+      this.pstmt = result.getStatement();
+      this.sql = ormQuery.getGeneratedSql();
+      setResultSet(result.getResultSet(), ormQuery.getQueryPlanKey());
+
+    } else {
+      // native SQL query execution
+      executeAsSql(binder);
+    }
   }
 
-  private void obtainPlan() throws SQLException {
-    String planKey = query.planKey();
+  @Override
+  protected void setResultSet(ResultSet resultSet, Object queryPlanKey) throws SQLException {
+    this.resultSet = resultSet;
+    this.dataReader = new RsetDataReader(server.getDataTimeZone(), resultSet);
+    obtainPlan(queryPlanKey);
+  }
+
+  private void obtainPlan(Object planKey) throws SQLException {
+    if (planKey == null) {
+      planKey = query.planKey();
+    }
     plan = query.getQueryPlan(planKey);
     if (plan == null) {
       plan = query.buildPlan(mappingRequest());

src/main/java/io/ebeaninternal/server/core/OrmQueryEngine.java

@@ -13,6 +13,11 @@ import java.util.List;
  */
 public interface OrmQueryEngine {
 
+  /**
+   * Execute the ORM query returning the JDBC ResultSet (for DTO query processing).
+   */
+  <T> SpiResultSet findResultSet(OrmQueryRequest<T> request);
+
   /**
    * Execute the 'find by id' query returning a single bean.
    */

src/main/java/io/ebeaninternal/server/core/OrmQueryRequest.java

@@ -353,6 +353,13 @@ public final class OrmQueryRequest<T> extends BeanRequest implements SpiOrmQuery
     return query.getType() == Type.BEAN;
   }
 
+  /**
+   * Return true if this is a findEach, findIterate type query where we expect many results.
+   */
+  public boolean isFindIterate() {
+    return query.getType() == Type.ITERATE;
+  }
+
   /**
    * Execute the query as a delete.
    */
@@ -376,6 +383,11 @@ public final class OrmQueryRequest<T> extends BeanRequest implements SpiOrmQuery
     return rows;
   }
 
+  @Override
+  public SpiResultSet findResultSet() {
+    return queryEngine.findResultSet(this);
+  }
+
   /**
    * Execute the query as findById.
    */

src/main/java/io/ebeaninternal/server/core/RelationalQueryRequest.java

@@ -36,7 +36,7 @@ public final class RelationalQueryRequest extends AbstractSqlQueryRequest {
   }
 
   @Override
-  protected void setResultSet(ResultSet resultSet) throws SQLException {
+  protected void setResultSet(ResultSet resultSet, Object planKey) throws SQLException {
     this.resultSet = resultSet;
     this.propertyNames = getPropertyNames();
     // calculate the initialCapacity of the Map to reduce rehashing

src/main/java/io/ebeaninternal/server/core/SpiOrmQueryRequest.java

@@ -114,6 +114,11 @@ public interface SpiOrmQueryRequest<T> extends BeanQueryRequest<T>, DocQueryRequ
    */
   <A> List<A> findSingleAttributeList();
 
+  /**
+   * Execute returning the ResultSet.
+   */
+  SpiResultSet findResultSet();
+
   /**
    * Try to get the query result from the query cache.
    */

src/main/java/io/ebeaninternal/server/core/SpiResultSet.java

@@ -0,0 +1,29 @@
+package io.ebeaninternal.server.core;
+
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+
+/**
+ * Hold the JDBC PreparedStatement and ResultSet.
+ *
+ * These both must be closed properly when done.
+ */
+public class SpiResultSet {
+
+  private final PreparedStatement statement;
+
+  private final ResultSet resultSet;
+
+  public SpiResultSet(PreparedStatement statement, ResultSet resultSet) {
+    this.statement = statement;
+    this.resultSet = resultSet;
+  }
+
+  public PreparedStatement getStatement() {
+    return statement;
+  }
+
+  public ResultSet getResultSet() {
+    return resultSet;
+  }
+}

src/main/java/io/ebeaninternal/server/dto/DtoBeanDescriptor.java

@@ -10,7 +10,7 @@ import java.util.concurrent.ConcurrentHashMap;
  */
 public class DtoBeanDescriptor<T> {
 
-  private final Map<String, DtoQueryPlan> plans = new ConcurrentHashMap<>();
+  private final Map<Object, DtoQueryPlan> plans = new ConcurrentHashMap<>();
 
   private final Class<T> dtoType;
 
@@ -25,7 +25,7 @@ public class DtoBeanDescriptor<T> {
     return dtoType;
   }
 
-  public DtoQueryPlan getQueryPlan(String planKey) {
+  public DtoQueryPlan getQueryPlan(Object planKey) {
     return plans.get(planKey);
   }
 
@@ -33,7 +33,7 @@ public class DtoBeanDescriptor<T> {
     return meta.match(request);
   }
 
-  public void putQueryPlan(String planKey, DtoQueryPlan plan) {
+  public void putQueryPlan(Object planKey, DtoQueryPlan plan) {
     plans.put(planKey, plan);
   }
 

src/main/java/io/ebeaninternal/server/dto/DtoMeta.java

@@ -1,5 +1,7 @@
 package io.ebeaninternal.server.dto;
 
+import io.ebean.util.StringHelper;
+
 import java.util.Arrays;
 import java.util.LinkedHashMap;
 import java.util.List;
@@ -81,7 +83,7 @@ class DtoMeta {
     int pos = 0;
     for (int i = firstOnes; i < cols.length; i++) {
       String label = cols[i].getLabel();
-      DtoReadSet property = propMap.get(label.toUpperCase());
+      DtoReadSet property = findProperty(label);
       if (property == null || property.isReadOnly()) {
         if (request.isRelaxedMode()) {
           property = DtoReadSetColumnSkip.INSTANCE;
@@ -103,7 +105,7 @@ class DtoMeta {
 
     for (int i = 0; i < cols.length; i++) {
       String label = cols[i].getLabel();
-      DtoReadSet property = propMap.get(label.toUpperCase());
+      DtoReadSet property = findProperty(label);
       if (property == null || property.isReadOnly()) {
         if (request.isRelaxedMode()) {
           property = DtoReadSetColumnSkip.INSTANCE;
@@ -116,4 +118,15 @@ class DtoMeta {
 
     return new DtoQueryPlanConSetter(request, defaultConstructor, setterProps);
   }
+
+  private DtoReadSet findProperty(String label) {
+
+    String upperLabel = label.toUpperCase();
+    DtoMetaProperty property = propMap.get(upperLabel);
+    if (property == null) {
+      upperLabel = StringHelper.replaceString(upperLabel, "_", "");
+      property = propMap.get(upperLabel);
+    }
+    return property;
+  }
 }

src/main/java/io/ebeaninternal/server/profile/DQueryPlanMetric.java

@@ -41,7 +41,7 @@ class DQueryPlanMetric implements QueryPlanMetric {
 
     @Override
     public String toString() {
-      return meta +" "+ stats;
+      return meta + " " + stats + " sql:" + getSql();
     }
 
     @Override

src/main/java/io/ebeaninternal/server/query/CQuery.java

@@ -310,11 +310,21 @@ public class CQuery<T> implements DbReadContext, CancelableQuery, SpiProfileTran
 
   private boolean prepareBindExecuteQueryWithOption(boolean forwardOnlyHint) throws SQLException {
 
+    ResultSet resultSet = prepareResultSet(forwardOnlyHint);
+    if (resultSet == null) {
+      return false;
+    }
+    dataReader = queryPlan.createDataReader(resultSet);
+    return true;
+  }
+
+  ResultSet prepareResultSet(boolean forwardOnlyHint) throws SQLException {
+
     synchronized (this) {
       if (cancelled || query.isCancelled()) {
         // cancelled before we started
         cancelled = true;
-        return false;
+        return null;
       }
 
       startNano = System.nanoTime();
@@ -328,9 +338,8 @@ public class CQuery<T> implements DbReadContext, CancelableQuery, SpiProfileTran
         ResultSet suppliedResultSet = query.getRawSql().getResultSet();
         if (suppliedResultSet != null) {
           // this is a user supplied ResultSet so use that
-          dataReader = queryPlan.createDataReader(suppliedResultSet);
           bindLog = "";
-          return true;
+          return suppliedResultSet;
         }
       }
 
@@ -353,10 +362,7 @@ public class CQuery<T> implements DbReadContext, CancelableQuery, SpiProfileTran
       bindLog = predicates.bind(dataBind);
 
       // executeQuery
-      ResultSet rset = pstmt.executeQuery();
-      dataReader = queryPlan.createDataReader(rset);
-
-      return true;
+      return pstmt.executeQuery();
     }
   }
 
@@ -793,4 +799,10 @@ public class CQuery<T> implements DbReadContext, CancelableQuery, SpiProfileTran
     }
   }
 
+  /**
+   * Return the underlying PreparedStatement.
+   */
+  PreparedStatement getPstmt() {
+    return pstmt;
+  }
 }

src/main/java/io/ebeaninternal/server/query/CQueryEngine.java

@@ -8,12 +8,14 @@ import io.ebean.bean.EntityBean;
 import io.ebean.bean.ObjectGraphNode;
 import io.ebean.config.ServerConfig;
 import io.ebean.config.dbplatform.DatabasePlatform;
+import io.ebean.util.JdbcClose;
 import io.ebean.util.StringHelper;
 import io.ebeaninternal.api.SpiQuery;
 import io.ebeaninternal.api.SpiTransaction;
 import io.ebeaninternal.server.core.DiffHelp;
 import io.ebeaninternal.server.core.Message;
 import io.ebeaninternal.server.core.OrmQueryRequest;
+import io.ebeaninternal.server.core.SpiResultSet;
 import io.ebeaninternal.server.deploy.BeanDescriptor;
 import io.ebeaninternal.server.lib.util.Str;
 import io.ebeaninternal.server.persist.Binder;
@@ -22,6 +24,7 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import javax.persistence.PersistenceException;
+import java.sql.ResultSet;
 import java.sql.SQLException;
 import java.util.LinkedHashMap;
 import java.util.List;
@@ -325,6 +328,37 @@ public class CQueryEngine {
     return historySupport.getSysPeriodLower(rootTableAlias);
   }
 
+  /**
+   * Execute returning the ResultSet and PreparedStatement for processing (by DTO query usually).
+   */
+  public <T> SpiResultSet findResultSet(OrmQueryRequest<T> request) {
+    CQuery<T> cquery = queryBuilder.buildQuery(request);
+    try {
+      boolean fwdOnly;
+      if (request.isFindIterate()) {
+        // findEach ...
+        fwdOnly = forwardOnlyHintOnFindIterate;
+        if (defaultFetchSizeFindEach > 0) {
+          request.setDefaultFetchBuffer(defaultFetchSizeFindEach);
+        }
+      } else {
+        // findList - aggressive fetch
+        fwdOnly = false;
+        if (defaultFetchSizeFindList > 0) {
+          request.setDefaultFetchBuffer(defaultFetchSizeFindList);
+        }
+      }
+      ResultSet resultSet = cquery.prepareResultSet(fwdOnly);
+      if (request.isLogSql()) {
+        logSql(cquery);
+      }
+      return new SpiResultSet(cquery.getPstmt(), resultSet);
+
+    } catch (SQLException e) {
+      JdbcClose.close(cquery.getPstmt());
+      throw cquery.createPersistenceException(e);
+    }
+  }
 
   /**
    * Find a list/map/set of beans.

src/main/java/io/ebeaninternal/server/query/DefaultOrmQueryEngine.java

@@ -9,6 +9,7 @@ import io.ebeaninternal.api.SpiQuery;
 import io.ebeaninternal.api.SpiTransaction;
 import io.ebeaninternal.server.core.OrmQueryEngine;
 import io.ebeaninternal.server.core.OrmQueryRequest;
+import io.ebeaninternal.server.core.SpiResultSet;
 import io.ebeaninternal.server.deploy.BeanDescriptor;
 import io.ebeaninternal.server.persist.Binder;
 
@@ -77,6 +78,12 @@ public class DefaultOrmQueryEngine implements OrmQueryEngine {
     return queryEngine.update(request);
   }
 
+  @Override
+  public <T> SpiResultSet findResultSet(OrmQueryRequest<T> request) {
+    flushJdbcBatchOnQuery(request);
+    return queryEngine.findResultSet(request);
+  }
+
   @Override
   public <T> int findCount(OrmQueryRequest<T> request) {
 

src/main/java/io/ebeaninternal/server/query/DefaultRelationalQueryEngine.java

@@ -1,6 +1,7 @@
 package io.ebeaninternal.server.query;
 
 import io.ebean.SqlRow;
+import io.ebeaninternal.api.SpiQuery;
 import io.ebeaninternal.server.core.Message;
 import io.ebeaninternal.server.core.RelationalQueryEngine;
 import io.ebeaninternal.server.core.RelationalQueryRequest;
@@ -39,7 +40,7 @@ public class DefaultRelationalQueryEngine implements RelationalQueryEngine {
   public void findEach(RelationalQueryRequest request, Predicate<SqlRow> consumer) {
 
     try {
-      request.executeSql(binder);
+      request.executeSql(binder, SpiQuery.Type.ITERATE);
       while (request.next()) {
         if (!consumer.test(readRow(request))) {
           break;
@@ -59,7 +60,7 @@ public class DefaultRelationalQueryEngine implements RelationalQueryEngine {
   public void findEach(RelationalQueryRequest request, Consumer<SqlRow> consumer) {
 
     try {
-      request.executeSql(binder);
+      request.executeSql(binder, SpiQuery.Type.ITERATE);
       while (request.next()) {
         consumer.accept(readRow(request));
       }
@@ -77,7 +78,7 @@ public class DefaultRelationalQueryEngine implements RelationalQueryEngine {
   public List<SqlRow> findList(RelationalQueryRequest request) {
 
     try {
-      request.executeSql(binder);
+      request.executeSql(binder, SpiQuery.Type.LIST);
       List<SqlRow> rows = new ArrayList<>();
       while (request.next()) {
         rows.add(readRow(request));

src/main/java/io/ebeaninternal/server/query/SqlTreeBuilder.java

@@ -442,7 +442,7 @@ public final class SqlTreeBuilder {
         p = desc.findBeanProperty("id");
         selectProps.add(p);
 
-      } else if (p.isId() && isNotSingleAttribute()) {
+      } else if (p.isId() && excludeIdProperty()) {
         // do not bother to include id for normal queries as the
         // id is always added (except for subQueries)
 
@@ -722,6 +722,13 @@ public final class SqlTreeBuilder {
 
   }
 
+  /**
+   * Return true if the Id property should be excluded (as it is automatically included).
+   */
+  private boolean excludeIdProperty() {
+    return query == null || !query.isSingleAttribute() && !query.isManualId();
+  }
+
   /**
    * Return true if the query is not a single attribute query.
    */

src/main/java/io/ebeaninternal/server/query/dto/DtoQueryEngine.java

@@ -1,5 +1,6 @@
 package io.ebeaninternal.server.query.dto;
 
+import io.ebeaninternal.api.SpiQuery;
 import io.ebeaninternal.server.core.DtoQueryRequest;
 import io.ebeaninternal.server.core.Message;
 import io.ebeaninternal.server.persist.Binder;
@@ -20,7 +21,7 @@ public class DtoQueryEngine {
 
   public <T> List<T> findList(DtoQueryRequest<T> request) {
     try {
-      request.executeSql(binder);
+      request.executeSql(binder, SpiQuery.Type.LIST);
       List<T> rows = new ArrayList<>();
       while (request.next()) {
         rows.add(request.readNextBean());
@@ -36,7 +37,7 @@ public class DtoQueryEngine {
 
   public <T> void findEach(DtoQueryRequest<T> request, Consumer<T> consumer) {
      try {
-      request.executeSql(binder);
+      request.executeSql(binder, SpiQuery.Type.ITERATE);
       while (request.next()) {
         consumer.accept(request.readNextBean());
       }
@@ -50,7 +51,7 @@ public class DtoQueryEngine {
 
   public <T> void findEachWhile(DtoQueryRequest<T> request, Predicate<T> consumer) {
     try {
-      request.executeSql(binder);
+      request.executeSql(binder, SpiQuery.Type.ITERATE);
       while (request.next()) {
         if (!consumer.test(request.readNextBean())) {
           break;

src/main/java/io/ebeaninternal/server/querydefn/DefaultDtoQuery.java

@@ -4,6 +4,7 @@ import io.ebean.DtoQuery;
 import io.ebeaninternal.api.BindParams;
 import io.ebeaninternal.api.SpiDtoQuery;
 import io.ebeaninternal.api.SpiEbeanServer;
+import io.ebeaninternal.api.SpiQuery;
 import io.ebeaninternal.server.dto.DtoBeanDescriptor;
 import io.ebeaninternal.server.dto.DtoMappingRequest;
 import io.ebeaninternal.server.dto.DtoQueryPlan;
@@ -22,6 +23,8 @@ public class DefaultDtoQuery<T> implements SpiDtoQuery<T> {
 
   private final DtoBeanDescriptor<T> descriptor;
 
+  private final SpiQuery<?> ormQuery;
+
   private String sql;
 
   private int firstRow;
@@ -42,11 +45,21 @@ public class DefaultDtoQuery<T> implements SpiDtoQuery<T> {
   private final BindParams bindParams = new BindParams();
 
   /**
-   * Additional supply a query detail object.
+   * Create given an underlying ORM query.
+   */
+  public DefaultDtoQuery(SpiEbeanServer server, DtoBeanDescriptor<T> descriptor, SpiQuery<?> ormQuery) {
+    this.server = server;
+    this.descriptor = descriptor;
+    this.ormQuery = ormQuery;
+  }
+
+  /**
+   * Create given a native SQL query.
    */
   public DefaultDtoQuery(SpiEbeanServer server, DtoBeanDescriptor<T> descriptor, String sql) {
     this.server = server;
     this.descriptor = descriptor;
+    this.ormQuery = null;
     this.sql = sql;
   }
 
@@ -56,7 +69,7 @@ public class DefaultDtoQuery<T> implements SpiDtoQuery<T> {
   }
 
   @Override
-  public DtoQueryPlan getQueryPlan(String planKey) {
+  public DtoQueryPlan getQueryPlan(Object planKey) {
     return descriptor.getQueryPlan(planKey);
   }
 
@@ -66,7 +79,7 @@ public class DefaultDtoQuery<T> implements SpiDtoQuery<T> {
   }
 
   @Override
-  public void putQueryPlan(String planKey, DtoQueryPlan plan) {
+  public void putQueryPlan(Object planKey, DtoQueryPlan plan) {
     descriptor.putQueryPlan(planKey, plan);
   }
 
@@ -97,13 +110,21 @@ public class DefaultDtoQuery<T> implements SpiDtoQuery<T> {
 
   @Override
   public DtoQuery<T> setParameter(int position, Object value) {
-    bindParams.setParameter(position, value);
+    if (ormQuery != null) {
+      ormQuery.setParameter(position, value);
+    } else {
+      bindParams.setParameter(position, value);
+    }
     return this;
   }
 
   @Override
   public DtoQuery<T> setParameter(String paramName, Object value) {
-    bindParams.setParameter(paramName, value);
+    if (ormQuery != null) {
+      ormQuery.setParameter(paramName, value);
+    } else {
+      bindParams.setParameter(paramName, value);
+    }
     return this;
   }
 
@@ -117,6 +138,10 @@ public class DefaultDtoQuery<T> implements SpiDtoQuery<T> {
     return descriptor.getType();
   }
 
+  public SpiQuery<?> getOrmQuery() {
+    return ormQuery;
+  }
+
   @Override
   public DtoQuery<T> setRelaxedMode() {
     this.relaxedMode = true;
@@ -147,6 +172,9 @@ public class DefaultDtoQuery<T> implements SpiDtoQuery<T> {
   @Override
   public DtoQuery<T> setFirstRow(int firstRow) {
     this.firstRow = firstRow;
+    if (ormQuery != null) {
+      ormQuery.setFirstRow(firstRow);
+    }
     return this;
   }
 
@@ -158,6 +186,9 @@ public class DefaultDtoQuery<T> implements SpiDtoQuery<T> {
   @Override
   public DtoQuery<T> setMaxRows(int maxRows) {
     this.maxRows = maxRows;
+    if (ormQuery != null) {
+      ormQuery.setMaxRows(maxRows);
+    }
     return this;
   }
 

src/main/java/io/ebeaninternal/server/querydefn/DefaultOrmQuery.java

@@ -2,6 +2,7 @@ package io.ebeaninternal.server.querydefn;
 
 import io.ebean.CacheMode;
 import io.ebean.CountDistinctOrder;
+import io.ebean.DtoQuery;
 import io.ebean.Expression;
 import io.ebean.ExpressionFactory;
 import io.ebean.ExpressionList;
@@ -130,6 +131,11 @@ public class DefaultOrmQuery<T> implements SpiQuery<T> {
 
   private String lazyLoadManyPath;
 
+  /**
+   * Flag set for report/DTO beans when we may choose to explicitly include the Id property.
+   */
+  private boolean manualId;
+
   /**
    * Set to true by a user wanting a DISTINCT query (id property must be excluded).
    */
@@ -280,6 +286,11 @@ public class DefaultOrmQuery<T> implements SpiQuery<T> {
     this.nativeSql = nativeSql;
   }
 
+  @Override
+  public <D> DtoQuery<D> asDto(Class<D> dtoClass) {
+    return server.findDto(dtoClass, this);
+  }
+
   @Override
   public BeanDescriptor<T> getBeanDescriptor() {
     return beanDescriptor;
@@ -668,7 +679,7 @@ public class DefaultOrmQuery<T> implements SpiQuery<T> {
    */
   @Override
   public boolean isWithId() {
-    return !distinct && !singleAttribute;
+    return !manualId && !distinct && !singleAttribute;
   }
 
   @Override
@@ -1027,7 +1038,7 @@ public class DefaultOrmQuery<T> implements SpiQuery<T> {
       queryPlanKey = new NativeSqlQueryPlanKey(nativeSql + "-" + firstRow + "-" + maxRows);
     } else {
       queryPlanKey = new OrmQueryPlanKey(beanDescriptor.getDiscValue(), m2mIncludeJoin, type, detail, maxRows, firstRow,
-        disableLazyLoading, orderBy,
+        disableLazyLoading, manualId, orderBy,
         distinct, sqlDistinct, mapKey, id, bindParams, whereExpressions, havingExpressions,
         temporalMode, forUpdate, rootTableAlias, rawSql, updateProperties, countDistinctOrder);
     }
@@ -1044,6 +1055,11 @@ public class DefaultOrmQuery<T> implements SpiQuery<T> {
     return nativeSql;
   }
 
+  @Override
+  public Object getQueryPlanKey() {
+    return queryPlanKey;
+  }
+
   /**
    * Prepare the query which prepares any expressions (sub-query expressions etc) and calculates the query plan key.
    */
@@ -1475,6 +1491,16 @@ public class DefaultOrmQuery<T> implements SpiQuery<T> {
     return this;
   }
 
+  @Override
+  public boolean isManualId() {
+    return manualId;
+  }
+
+  @Override
+  public void setManualId(boolean manualId) {
+    this.manualId = manualId;
+  }
+
   /**
    * return true if user specified to use SQL DISTINCT (effectively excludes id property).
    */

src/main/java/io/ebeaninternal/server/querydefn/OrmQueryPlanKey.java

@@ -22,7 +22,7 @@ class OrmQueryPlanKey implements CQueryPlanKey {
   private final String description;
 
   OrmQueryPlanKey(String discValue, TableJoin m2mIncludeTable, SpiQuery.Type type, OrmQueryDetail detail, int maxRows, int firstRow, boolean disableLazyLoading,
-                  OrderBy<?> orderBy, boolean distinct, boolean sqlDistinct, String mapKey, Object id, BindParams bindParams,
+                  boolean manualId, OrderBy<?> orderBy, boolean distinct, boolean sqlDistinct, String mapKey, Object id, BindParams bindParams,
                   SpiExpression whereExpressions, SpiExpression havingExpressions, SpiQuery.TemporalMode temporalMode,
                   Query.ForUpdate forUpdate, String rootTableAlias, SpiRawSql rawSql, OrmUpdateProperties updateProperties, CountDistinctOrder countDistinctOrder) {
 
@@ -42,6 +42,9 @@ class OrmQueryPlanKey implements CQueryPlanKey {
     if (id != null) {
       sb.append(",id:");
     }
+    if (manualId) {
+      sb.append(",manId:");
+    }
     if (distinct) {
       sb.append(",dist:");
     }