#405 - ENH: Add @ReadAudit and associated functionality for auditing reads including queries and L2 cache access

src/main/java/com/avaje/ebean/Query.java

@@ -366,6 +366,16 @@ public interface Query<T> extends Serializable {
    */
   Query<T> setLazyLoadBatchSize(int lazyLoadBatchSize);
 
+  /**
+   * Disable read auditing for this query.
+   * <p>
+   * This is intended to be used when the query is not a user initiated query and instead
+   * part of the internal processing in an application to load a cache or document store etc.
+   * In these cases we don't want the query to be part of read auditing.
+   * </p>
+   */
+  Query<T> setDisableReadAuditing();
+
   /**
    * Explicitly set a comma delimited list of the properties to fetch on the
    * 'main' root level entity bean (aka partial object). Note that '*' means all

src/main/java/com/avaje/ebean/annotation/ReadAudit.java

@@ -0,0 +1,15 @@
+package com.avaje.ebean.annotation;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+/**
+ * Marks an entity bean as being included in read auditing.
+ */
+@Target({ElementType.TYPE})
+@Retention(RetentionPolicy.RUNTIME)
+public @interface ReadAudit {
+
+}

src/main/java/com/avaje/ebean/config/ServerConfig.java

@@ -11,6 +11,8 @@ import com.avaje.ebean.event.*;
 import com.avaje.ebean.event.changelog.ChangeLogListener;
 import com.avaje.ebean.event.changelog.ChangeLogPrepare;
 import com.avaje.ebean.event.changelog.ChangeLogRegister;
+import com.avaje.ebean.event.readaudit.ReadAuditLogger;
+import com.avaje.ebean.event.readaudit.ReadAuditPrepare;
 import com.avaje.ebean.meta.MetaInfoManager;
 import com.avaje.ebean.util.ClassUtil;
 import com.fasterxml.jackson.core.JsonFactory;
@@ -299,6 +301,10 @@ public class ServerConfig {
 
   private ChangeLogRegister changeLogRegister;
 
+  private ReadAuditLogger readAuditLogger;
+
+  private ReadAuditPrepare readAuditPrepare;
+
   private EncryptKeyManager encryptKeyManager;
 
   private EncryptDeployManager encryptDeployManager;
@@ -725,6 +731,41 @@ public class ServerConfig {
     this.changeLogIncludeInserts = changeLogIncludeInserts;
   }
 
+  /**
+   * Return the ReadAuditLogger to use.
+   */
+  public ReadAuditLogger getReadAuditLogger() {
+    return readAuditLogger;
+  }
+
+  /**
+   * Set the ReadAuditLogger to use. If not set the default implementation is used
+   * which logs the read events in JSON format to a standard named SLF4J logger
+   * (which can be configured in say logback to log to a separate log file).
+   */
+  public void setReadAuditLogger(ReadAuditLogger readAuditLogger) {
+    this.readAuditLogger = readAuditLogger;
+  }
+
+  /**
+   * Return the ReadAuditPrepare to use.
+   */
+  public ReadAuditPrepare getReadAuditPrepare() {
+    return readAuditPrepare;
+  }
+
+  /**
+   * Set the ReadAuditPrepare to use.
+   * <p>
+   * It is expected that an implementation is used that read user context information
+   * (user id, user ip address etc) and sets it on the ReadEvent bean before it is sent
+   * to the ReadAuditLogger.
+   * </p>
+   */
+  public void setReadAuditPrepare(ReadAuditPrepare readAuditPrepare) {
+    this.readAuditPrepare = readAuditPrepare;
+  }
+
   /**
    * Return the DB migration configuration.
    */

src/main/java/com/avaje/ebean/dbmigration/ddlgeneration/platform/DbTriggerBasedHistoryDdl.java

@@ -176,7 +176,7 @@ public abstract class DbTriggerBasedHistoryDdl implements PlatformHistoryDdl {
 
     String platformType = platformDdl.convert(type, false);
     buffer.append("  ");
-    buffer.append(platformDdl.lowerName(columnName), 30);
+    buffer.append(platformDdl.lowerName(columnName), 29);
     buffer.append(platformType);
   }
 

src/main/java/com/avaje/ebean/dbmigration/ddlgeneration/platform/PlatformDdl.java

@@ -136,7 +136,7 @@ public class PlatformDdl {
     String platformType = convert(column.getType(), identityColumn);
 
     buffer.append("  ");
-    buffer.append(lowerName(column.getName()), 30);
+    buffer.append(lowerName(column.getName()), 29);
     buffer.append(platformType);
     if (isTrue(column.isNotnull()) || isTrue(column.isPrimaryKey())) {
       buffer.append(" not null");

src/main/java/com/avaje/ebean/event/changelog/BeanChange.java

@@ -51,6 +51,10 @@ public class BeanChange {
   public BeanChange() {
   }
 
+  public String toString() {
+    return "table:" + table + " id:" + id+" values:"+values;
+  }
+
   /**
    * Return the object type (typically table name).
    */

src/main/java/com/avaje/ebean/event/readaudit/ReadAuditLogger.java

@@ -0,0 +1,37 @@
+package com.avaje.ebean.event.readaudit;
+
+/**
+ * Log that the query was executed
+ */
+public interface ReadAuditLogger {
+
+  /**
+   * Called when a new query plan is created.
+   * <p>
+   * The query plan has the full sql and logging the query plan separately means that each of
+   * the bean and many read events can log the query plan key and not the full sql (reducing the
+   * bulk size of the read audit logs).
+   * </p>
+   */
+  void queryPlan(ReadAuditQueryPlan queryPlan);
+
+  /**
+   * Audit a find bean query that returned a bean.
+   * <p>
+   * Finds that did not return a bean are excluded.
+   * </p>
+   */
+  void auditBean(ReadEvent readBean);
+
+  /**
+   * Audit a find many query that returned some beans.
+   * <p>
+   * Finds that did not return any beans are excluded.
+   * </p>
+   * <p>
+   * For large queries executed via findEach() etc the ids are collected in batches
+   * and logged. Hence the ids list has a maximum size of the batch size.
+   * </p>
+   */
+  void auditMany(ReadEvent readMany);
+}

src/main/java/com/avaje/ebean/event/readaudit/ReadAuditPrepare.java

@@ -0,0 +1,23 @@
+package com.avaje.ebean.event.readaudit;
+
+/**
+ * Set user context information into the read event prior to it being logged.
+ */
+public interface ReadAuditPrepare {
+
+  /**
+   * Prepare the read event by setting any user context information into the read event such as the
+   * application user id and ip address.
+   * <p>
+   * This method is called prior to the read event being sent to the ReadAuditLogger.
+   * </p>
+   * <p>
+   * Note that for findFutureList() queries prepare() is called early in the foreground thread
+   * prior to the query executing and at that point the ReadEvent bean only has the bean type
+   * and no other details (which are populated later when the query is executed in the background
+   * thread).
+   * </p>
+   */
+  void prepare(ReadEvent readEvent);
+
+}

src/main/java/com/avaje/ebean/event/readaudit/ReadAuditQueryPlan.java

@@ -0,0 +1,77 @@
+package com.avaje.ebean.event.readaudit;
+
+/**
+ * A SQL query and associated keys.
+ * <p>
+ * This is logged as a separate event so that the
+ * </p>
+ */
+public class ReadAuditQueryPlan {
+
+  String beanType;
+
+  String queryKey;
+
+  String sql;
+
+  /**
+   * Construct given the beanType, queryKey and sql.
+   */
+  public ReadAuditQueryPlan(String beanType, String queryKey, String sql) {
+    this.beanType = beanType;
+    this.queryKey = queryKey;
+    this.sql = sql;
+  }
+
+  /**
+   * Construct for JSON tools.
+   */
+  public ReadAuditQueryPlan() {
+  }
+
+  public String toString() {
+    return "beanType:" + beanType + " queryKey:" + queryKey + " sql:" + sql;
+  }
+
+  /**
+   * Return the bean type.
+   */
+  public String getBeanType() {
+    return beanType;
+  }
+
+  /**
+   * Set the bean type.
+   */
+  public void setBeanType(String beanType) {
+    this.beanType = beanType;
+  }
+
+  /**
+   * Return the query key (relative to the bean type).
+   */
+  public String getQueryKey() {
+    return queryKey;
+  }
+
+  /**
+   * Set the query key.
+   */
+  public void setQueryKey(String queryKey) {
+    this.queryKey = queryKey;
+  }
+
+  /**
+   * Return the sql statement.
+   */
+  public String getSql() {
+    return sql;
+  }
+
+  /**
+   * Set the sql statement.
+   */
+  public void setSql(String sql) {
+    this.sql = sql;
+  }
+}

src/main/java/com/avaje/ebean/event/readaudit/ReadEvent.java

@@ -0,0 +1,261 @@
+package com.avaje.ebean.event.readaudit;
+
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Read event sent to the ReadEventLogger.
+ * <p>
+ * This is a flattened in that it contains either a read bean or list of beans. It is flattened
+ * in this way to simplify logging and processing and simply means that it either contains an
+ * id or a list of ids.
+ * </p>
+ */
+public class ReadEvent {
+
+  /**
+   * User defined 'source' such as the application name.
+   */
+  protected String source;
+
+  /**
+   * Application user id expected to be optionally populated by ChangeLogPrepare.
+   */
+  protected String userId;
+
+  /**
+   * Application user ip address expected to be optionally populated by ChangeLogPrepare.
+   */
+  protected String userIpAddress;
+
+  /**
+   * Arbitrary user context information expected to be optionally populated by ChangeLogPrepare.
+   */
+  protected Map<String, String> userContext;
+
+  /**
+   * The time the bean change was created.
+   */
+  protected long eventTime;
+
+  /**
+   * The type of the bean(s) read.
+   */
+  protected String beanType;
+
+  /**
+   * The query key (relative to the bean type).
+   */
+  protected String queryKey;
+
+  /**
+   * The bind log when the query was executed.
+   */
+  protected String bindLog;
+
+  /**
+   * The id of the bean read.
+   */
+  protected Object id;
+
+  /**
+   * The ids of the beans read.
+   */
+  protected List<Object> ids;
+
+  /**
+   * Common constructor for single bean and multi-bean read events.
+   */
+  protected ReadEvent(String beanType, String queryKey, String bindLog) {
+    this.beanType = beanType;
+    this.queryKey = queryKey;
+    this.bindLog = bindLog;
+    this.eventTime = System.currentTimeMillis();
+  }
+
+  /**
+   * Construct for a single bean read.
+   */
+  public ReadEvent(String beanType, String queryKey, String bindLog, Object id) {
+    this(beanType, queryKey, bindLog);
+    this.id = id;
+  }
+
+  /**
+   * Construct for many beans read.
+   */
+  public ReadEvent(String beanType, String queryKey, String bindLog, List<Object> ids) {
+    this(beanType, queryKey, bindLog);
+    this.ids = ids;
+  }
+
+  /**
+   * Construct for many future list query.
+   */
+  public ReadEvent(String beanType) {
+    this.beanType = beanType;
+    this.eventTime = System.currentTimeMillis();
+  }
+
+  /**
+   * Constructor for JSON tools.
+   */
+  public ReadEvent() {
+  }
+
+  /**
+   * Return a code that identifies the source of the change (like the name of the application).
+   */
+  public String getSource() {
+    return source;
+  }
+
+  /**
+   * Set the source of the change (like the name of the application).
+   */
+  public void setSource(String source) {
+    this.source = source;
+  }
+
+  /**
+   * Return the application user Id.
+   */
+  public String getUserId() {
+    return userId;
+  }
+
+  /**
+   * Set the application user Id.
+   * <p>
+   * This can be set by the ChangeLogListener in the prepare() method which is called
+   * in the foreground thread.
+   * </p>
+   */
+  public void setUserId(String userId) {
+    this.userId = userId;
+  }
+
+  /**
+   * Return the application users ip address.
+   */
+  public String getUserIpAddress() {
+    return userIpAddress;
+  }
+
+  /**
+   * Set the application users ip address.
+   * <p>
+   * This can be set by the ChangeLogListener in the prepare() method which is called
+   * in the foreground thread.
+   * </p>
+   */
+  public void setUserIpAddress(String userIpAddress) {
+    this.userIpAddress = userIpAddress;
+  }
+
+  /**
+   * Return a user context value - anything you set yourself in ChangeLogListener prepare().
+   */
+  public Map<String, String> getUserContext() {
+    if (userContext == null) {
+      userContext = new LinkedHashMap<String, String>();
+    }
+    return userContext;
+  }
+
+  /**
+   * Set a user context value (anything you like).
+   * <p>
+   * This can be set by the ChangeLogListener in the prepare() method which is called
+   * in the foreground thread.
+   * </p>
+   */
+  public void setUserContext(Map<String, String> userContext) {
+    this.userContext = userContext;
+  }
+
+  /**
+   * Return the type of bean read.
+   */
+  public String getBeanType() {
+    return beanType;
+  }
+
+  /**
+   * Set the type of bean read.
+   */
+  public void setBeanType(String beanType) {
+    this.beanType = beanType;
+  }
+
+  /**
+   * Return the query key (relative to the bean type).
+   */
+  public String getQueryKey() {
+    return queryKey;
+  }
+
+  /**
+   * Set the query key (relative to the bean type).
+   */
+  public void setQueryKey(String queryKey) {
+    this.queryKey = queryKey;
+  }
+
+  /**
+   * Return the bind log used when executing the query.
+   */
+  public String getBindLog() {
+    return bindLog;
+  }
+
+  /**
+   * Set the bind log used when executing the query.
+   */
+  public void setBindLog(String bindLog) {
+    this.bindLog = bindLog;
+  }
+
+  /**
+   * Return the event date time.
+   */
+  public long getEventTime() {
+    return eventTime;
+  }
+
+  /**
+   * Set the event date time.
+   */
+  public void setEventTime(long eventTime) {
+    this.eventTime = eventTime;
+  }
+
+  /**
+   * Return the id of the bean read.
+   */
+  public Object getId() {
+    return id;
+  }
+
+  /**
+   * Set the id of the bean read.
+   */
+  public void setId(Object id) {
+    this.id = id;
+  }
+
+  /**
+   * Return the ids of the beans read.
+   */
+  public List<Object> getIds() {
+    return ids;
+  }
+
+  /**
+   * Set the ids of the beans read.
+   */
+  public void setIds(List<Object> ids) {
+    this.ids = ids;
+  }
+}

src/main/java/com/avaje/ebean/text/json/EJson.java

@@ -3,6 +3,7 @@ package com.avaje.ebean.text.json;
 import java.io.IOException;
 import java.io.Reader;
 import java.io.Writer;
+import java.util.Collection;
 import java.util.List;
 import java.util.Map;
 
@@ -36,6 +37,13 @@ public class EJson {
     EJsonWriter.write(object, jsonGenerator);
   }
 
+  /**
+   * Write the collection as json array to the jsonGenerator.
+   */
+  public static void writeCollection(Collection<Object> collection, JsonGenerator jsonGenerator) throws IOException {
+    EJsonWriter.writeCollection(collection, jsonGenerator);
+  }
+
   /**
    * Parse the json and return as a Map additionally specifying if the returned map should
    * be modify aware meaning that it can detect when it has been modified.

src/main/java/com/avaje/ebean/text/json/EJsonWriter.java

@@ -37,6 +37,10 @@ class EJsonWriter {
     new EJsonWriter(jsonGenerator).writeJson(object);
   }
 
+  static void writeCollection(Collection<Object> collection, JsonGenerator jsonGenerator) throws IOException {
+    new EJsonWriter(jsonGenerator).writeCollection(null, collection);
+  }
+
   private final JsonGenerator jsonGenerator;
 
   private EJsonWriter(JsonGenerator jsonGenerator) {
@@ -53,6 +57,12 @@ class EJsonWriter {
       if (object == null) {
         writeNull(name);
 
+      } else if (object instanceof Number) {
+        writeNumber(name, (Number) object);
+
+      } else if (object instanceof String) {
+        writeString(name, (String) object);
+
       } else if (object instanceof Map) {
         writeMap(name, (Map<Object, Object>) object);
 
@@ -62,15 +72,9 @@ class EJsonWriter {
       } else if (object instanceof Boolean) {
         writeBoolean(name, (Boolean) object);
 
-      } else if (object instanceof Number) {
-        writeNumber(name, (Number) object);
-
       } else if (object instanceof Date) {
         writeDate(name, (Date) object);
 
-      } else if (object instanceof String) {
-        writeString(name, (String) object);
-
       } else if (object instanceof Map.Entry<?, ?>) {
         Map.Entry<?, ?> entry = (Map.Entry<?, ?>) object;
         writeJson(entry.getKey().toString(), entry.getValue());

src/main/java/com/avaje/ebeaninternal/api/HashQueryPlan.java

@@ -20,7 +20,16 @@ public class HashQueryPlan {
   public String toString() {
     return planHash+":"+bindCount+(rawSql != null ? ":r" : "");
   }
-  
+
+  /**
+   * Return as a partial key. For rawSql hash the sql is part of the key and as such
+   * needs to be included in order to have a complete key. Typically the MD5 of the sql
+   * can be used as a shot form proxy for the actual sql.
+   */
+  public String getPartialKey() {
+    return planHash+"_"+bindCount;
+  }
+
   public int hashCode() {
     int hc = planHash;
     hc = hc * 31 + bindCount;

src/main/java/com/avaje/ebeaninternal/api/SpiEbeanServer.java

@@ -11,6 +11,8 @@ import com.avaje.ebean.bean.CallStack;
 import com.avaje.ebean.bean.ObjectGraphNode;
 import com.avaje.ebean.config.ServerConfig;
 import com.avaje.ebean.config.dbplatform.DatabasePlatform;
+import com.avaje.ebean.event.readaudit.ReadAuditLogger;
+import com.avaje.ebean.event.readaudit.ReadAuditPrepare;
 import com.avaje.ebeaninternal.server.autofetch.AutoFetchManager;
 import com.avaje.ebeaninternal.server.core.SpiOrmQueryRequest;
 import com.avaje.ebean.dbmigration.DdlGenerator;
@@ -195,4 +197,14 @@ public interface SpiEbeanServer extends EbeanServer, BeanLoader, BeanCollectionL
    */
   void collectQueryStats(ObjectGraphNode objectGraphNode, long loadedBeanCount, long timeMicros);
 
+  /**
+   * Return the ReadAuditLogger to use for logging all read audit events.
+   */
+  ReadAuditLogger getReadAuditLogger();
+
+  /**
+   * Return the ReadAuditPrepare used to populate the read audit events with
+   * user context information (user id, user ip address etc).
+   */
+  ReadAuditPrepare getReadAuditPrepare();
 }

src/main/java/com/avaje/ebeaninternal/api/SpiQuery.java

@@ -10,6 +10,7 @@ import com.avaje.ebean.bean.CallStack;
 import com.avaje.ebean.bean.ObjectGraphNode;
 import com.avaje.ebean.bean.PersistenceContext;
 import com.avaje.ebean.event.BeanQueryRequest;
+import com.avaje.ebean.event.readaudit.ReadEvent;
 import com.avaje.ebeaninternal.server.autofetch.AutoFetchManager;
 import com.avaje.ebeaninternal.server.deploy.BeanDescriptor;
 import com.avaje.ebeaninternal.server.deploy.BeanPropertyAssocMany;
@@ -644,8 +645,13 @@ public interface SpiQuery<T> extends Query<T> {
   int getBufferFetchSizeHint();
 
   /**
-   * Return true if this is a query executing in the background.
+   * Return true if read auditing is disabled on this query.
    */
+  boolean isDisableReadAudit();
+
+    /**
+     * Return true if this is a query executing in the background.
+     */
   boolean isFutureFetch();
 
   /**
@@ -654,6 +660,16 @@ public interface SpiQuery<T> extends Query<T> {
    */
   void setFutureFetch(boolean futureFetch);
 
+  /**
+   * Set the readEvent for future queries (as prepared in foreground thread).
+   */
+  void setFutureFetchAudit(ReadEvent event);
+
+  /**
+   * Read the readEvent for future queries (null otherwise).
+   */
+  ReadEvent getFutureFetchAudit();
+
   /**
    * Set the underlying cancelable query (with the PreparedStatement).
    */

src/main/java/com/avaje/ebeaninternal/server/changelog/ChangeJsonBuilder.java

@@ -93,6 +93,7 @@ public class ChangeJsonBuilder {
    * For insert and update write the new/old values.
    */
   protected void writeBeanValues(JsonGenerator gen, BeanChange bean) throws IOException {
+
     if (bean.getType() != ChangeType.DELETE) {
       gen.writeFieldName("values");
       gen.writeStartObject();

src/main/java/com/avaje/ebeaninternal/server/changelog/DefaultChangeLogListener.java

@@ -9,7 +9,6 @@ import com.avaje.ebean.plugin.SpiServerPlugin;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.io.IOException;
 import java.io.StringWriter;
 import java.util.List;
 import java.util.Properties;
@@ -80,17 +79,17 @@ public class DefaultChangeLogListener implements ChangeLogListener, SpiServerPlu
   @Override
   public void log(ChangeSet changeSet) {
 
-    try {
-      List<BeanChange> changes = changeSet.getChanges();
-      for (int i = 0; i < changes.size(); i++) {
-        // log each bean change as a separate log entry
-        BeanChange beanChange = changes.get(i);
+    List<BeanChange> changes = changeSet.getChanges();
+    for (int i = 0; i < changes.size(); i++) {
+      // log each bean change as a separate log entry
+      BeanChange beanChange = changes.get(i);
+      try {
         StringWriter writer = new StringWriter(getBufferSize(beanChange));
         jsonBuilder.writeBeanJson(writer, beanChange, changeSet, i);
         changeLog.info(writer.toString());
+      } catch (Exception e) {
+        logger.error("Exception logging beanChange " + beanChange.toString(), e);
       }
-    } catch (IOException e) {
-      logger.error("Exception sending changeSet " + changeSet.toString(), e);
     }
   }
 

src/main/java/com/avaje/ebeaninternal/server/core/BootupClasses.java

@@ -13,6 +13,8 @@ import com.avaje.ebean.event.TransactionEventListener;
 import com.avaje.ebean.event.changelog.ChangeLogListener;
 import com.avaje.ebean.event.changelog.ChangeLogPrepare;
 import com.avaje.ebean.event.changelog.ChangeLogRegister;
+import com.avaje.ebean.event.readaudit.ReadAuditLogger;
+import com.avaje.ebean.event.readaudit.ReadAuditPrepare;
 import com.avaje.ebeaninternal.server.type.ScalarType;
 import com.avaje.ebeaninternal.server.util.ClassPathSearchMatcher;
 import org.slf4j.Logger;
@@ -69,10 +71,14 @@ public class BootupClasses implements ClassPathSearchMatcher {
   private Class<?> changeLogPrepareClass;
   private Class<?> changeLogListenerClass;
   private Class<?> changeLogRegisterClass;
+  private Class<?> readAuditPrepareClass;
+  private Class<?> readAuditLoggerClass;
 
   private ChangeLogPrepare changeLogPrepare;
   private ChangeLogListener changeLogListener;
   private ChangeLogRegister changeLogRegister;
+  private ReadAuditPrepare readAuditPrepare;
+  private ReadAuditLogger readAuditLogger;
 
   public BootupClasses() {
   }
@@ -192,6 +198,17 @@ public class BootupClasses implements ClassPathSearchMatcher {
   }
 
   public void addChangeLogInstances(ServerConfig serverConfig) {
+
+    readAuditPrepare = serverConfig.getReadAuditPrepare();
+    readAuditLogger = serverConfig.getReadAuditLogger();
+
+    if (readAuditPrepare == null && readAuditPrepareClass != null) {
+      readAuditPrepare = (ReadAuditPrepare)create(readAuditPrepareClass, false);
+    }
+    if (readAuditLogger == null && readAuditLoggerClass != null) {
+      readAuditLogger = (ReadAuditLogger)create(readAuditLoggerClass, false);
+    }
+
     changeLogListener = serverConfig.getChangeLogListener();
     changeLogRegister = serverConfig.getChangeLogRegister();
     changeLogPrepare = serverConfig.getChangeLogPrepare();
@@ -262,6 +279,14 @@ public class BootupClasses implements ClassPathSearchMatcher {
     return changeLogRegister;
   }
 
+  public ReadAuditPrepare getReadAuditPrepare() {
+    return readAuditPrepare;
+  }
+
+  public ReadAuditLogger getReadAuditLogger() {
+    return readAuditLogger;
+  }
+
   public List<BeanQueryAdapter> getBeanQueryAdapters() {
     // add class registered BeanQueryAdapter to the already created instances
     for (Class<?> cls : beanQueryAdapterList) {
@@ -440,6 +465,15 @@ public class BootupClasses implements ClassPathSearchMatcher {
       interesting = true;
     }
 
+    if (ReadAuditPrepare.class.isAssignableFrom(cls)) {
+      readAuditPrepareClass = cls;
+      interesting = true;
+    }
+    if (ReadAuditLogger.class.isAssignableFrom(cls)) {
+      readAuditLoggerClass = cls;
+      interesting = true;
+    }
+
     return interesting;
   }
 

src/main/java/com/avaje/ebeaninternal/server/core/DefaultServer.java

@@ -12,8 +12,11 @@ import com.avaje.ebean.cache.ServerCacheManager;
 import com.avaje.ebean.config.EncryptKeyManager;
 import com.avaje.ebean.config.ServerConfig;
 import com.avaje.ebean.config.dbplatform.DatabasePlatform;
+import com.avaje.ebean.dbmigration.DdlGenerator;
 import com.avaje.ebean.event.BeanPersistController;
 import com.avaje.ebean.event.BeanQueryAdapter;
+import com.avaje.ebean.event.readaudit.ReadAuditLogger;
+import com.avaje.ebean.event.readaudit.ReadAuditPrepare;
 import com.avaje.ebean.meta.MetaInfoManager;
 import com.avaje.ebean.plugin.SpiBeanType;
 import com.avaje.ebean.plugin.SpiServer;
@@ -34,7 +37,6 @@ import com.avaje.ebeaninternal.api.SpiSqlQuery;
 import com.avaje.ebeaninternal.api.SpiTransaction;
 import com.avaje.ebeaninternal.api.TransactionEventTable;
 import com.avaje.ebeaninternal.server.autofetch.AutoFetchManager;
-import com.avaje.ebean.dbmigration.DdlGenerator;
 import com.avaje.ebeaninternal.server.deploy.BeanDescriptor;
 import com.avaje.ebeaninternal.server.deploy.BeanDescriptorManager;
 import com.avaje.ebeaninternal.server.deploy.BeanProperty;
@@ -134,6 +136,10 @@ public final class DefaultServer implements SpiServer, SpiEbeanServer {
 
   private final AutoFetchManager autoFetchManager;
 
+  private final ReadAuditPrepare readAuditPrepare;
+
+  private final ReadAuditLogger readAuditLogger;
+
   private final CQueryEngine cqueryEngine;
 
   private final List<SpiServerPlugin> serverPlugins;
@@ -237,6 +243,8 @@ public final class DefaultServer implements SpiServer, SpiEbeanServer {
 
     this.autoFetchManager = config.createAutoFetchManager(this);
     this.adminAutofetch = new MAdminAutofetch(autoFetchManager);
+    this.readAuditPrepare = config.getReadAuditPrepare();
+    this.readAuditLogger = config.getReadAuditLogger();
 
     this.beanLoader = new DefaultBeanLoader(this);
     this.jsonContext = config.createJsonContext(this);
@@ -353,6 +361,16 @@ public final class DefaultServer implements SpiServer, SpiEbeanServer {
     return autoFetchManager;
   }
 
+  @Override
+  public ReadAuditPrepare getReadAuditPrepare() {
+    return readAuditPrepare;
+  }
+
+  @Override
+  public ReadAuditLogger getReadAuditLogger() {
+    return readAuditLogger;
+  }
+
   /**
    * Run any initialisation required before registering with the ClusterManager.
    */
@@ -1393,6 +1411,11 @@ public final class DefaultServer implements SpiServer, SpiEbeanServer {
     // FutureList query always run in it's own persistence content
     spiQuery.setPersistenceContext(new DefaultPersistenceContext());
 
+    if (!spiQuery.isDisableReadAudit()) {
+      BeanDescriptor<T> desc = beanDescriptorManager.getBeanDescriptor(spiQuery.getBeanType());
+      desc.readAuditFutureList(spiQuery);
+    }
+
     // Create a new transaction solely to execute the findList() at some future time
     Transaction newTxn = createTransaction();
     CallableQueryList<T> call = new CallableQueryList<T>(this, spiQuery, newTxn);

src/main/java/com/avaje/ebeaninternal/server/core/DiffHelp.java

@@ -60,19 +60,19 @@ public class DiffHelp {
 	
 	public void diff(String prefix, Map<String, ValuePair> map, EntityBean newBean, EntityBean oldBean, BeanDescriptor<?> desc) {
 
-		// check the simple properties
-		BeanProperty[] base = desc.propertiesBaseScalar();
-		for (int i = 0; i < base.length; i++) {
-			Object newVal = (newBean == null) ? null : base[i].getValue(newBean);
-			Object oldVal = (oldBean == null) ? null : base[i].getValue(oldBean);
-			if (!ValueUtil.areEqual(newVal, oldVal)) {
-			  String propName = (prefix == null) ? base[i].getName() : prefix + base[i].getName();
-				map.put(propName, new ValuePair(newVal, oldVal));
-			}
-		}
+    if (flatMode) {
+      desc.diff(prefix, map, newBean, oldBean);
+    } else {
 
-		diffAssocOne(prefix, newBean, oldBean, desc, map);
-		diffEmbedded(prefix, newBean, oldBean, desc, map);
+      // check the simple properties
+      BeanProperty[] base = desc.propertiesBaseScalar();
+      for (int i = 0; i < base.length; i++) {
+        base[i].diff(prefix, map, newBean, oldBean);
+      }
+
+      diffAssocOne(prefix, newBean, oldBean, desc, map);
+      diffEmbedded(prefix, newBean, oldBean, desc, map);
+    }
 	}
 
 	/**
@@ -96,7 +96,7 @@ public class DiffHelp {
 					// one of the embedded beans is null
           if (flatMode) {
             BeanDescriptor<?> embDesc = emb[i].getTargetDescriptor();
-            diff(emb[i].getName()+".", map, newVal, oldVal, embDesc);
+            diff(propName, map, newVal, oldVal, embDesc);
           } else {
             map.put(propName, new ValuePair(newVal, oldVal));
           }
@@ -104,7 +104,7 @@ public class DiffHelp {
 				} else {
 				  // recursively diff into the embedded bean
 				  BeanDescriptor<?> embDesc = emb[i].getTargetDescriptor();
-				  diff(emb[i].getName()+".", map, newVal, oldVal, embDesc);
+				  diff(propName, map, newVal, oldVal, embDesc);
 				}
 			}
 		}

src/main/java/com/avaje/ebeaninternal/server/core/DiffHelpInsert.java

@@ -1,102 +0,0 @@
-package com.avaje.ebeaninternal.server.core;
-
-import com.avaje.ebean.ValuePair;
-import com.avaje.ebean.bean.EntityBean;
-import com.avaje.ebeaninternal.server.deploy.BeanDescriptor;
-import com.avaje.ebeaninternal.server.deploy.BeanProperty;
-import com.avaje.ebeaninternal.server.deploy.BeanPropertyAssocOne;
-
-import java.util.LinkedHashMap;
-import java.util.Map;
-
-/**
- * Helper to perform a 'diff' for an insert.
- * <p>
- * This intentionally does not include any OneToMany or ManyToMany properties.
- * </p>
- */
-public class DiffHelpInsert {
-
-  private DiffHelpInsert() {
-  }
-
-  /**
-   * Return a map of the differences between a and b.
-   * <p>
-   * A and B must be of the same type. B can be null, in which case the 'dirty
-   * diff' of a is returned.
-   * </p>
-   * <p>
-   * This intentionally does not include as OneToMany or ManyToMany properties.
-   * </p>
-   */
-	public static Map<String, ValuePair> diff(EntityBean newBean, BeanDescriptor<?> desc) {
-
-    Map<String, ValuePair> map = new LinkedHashMap<String, ValuePair>();
-    diff(null, map, newBean, desc);
-    return map;
-	}
-	
-	private static void diff(String prefix, Map<String, ValuePair> map, EntityBean newBean,  BeanDescriptor<?> desc) {
-
-		// check the simple properties
-		BeanProperty[] base = desc.propertiesBaseScalar();
-		for (int i = 0; i < base.length; i++) {
-			Object newVal = (newBean == null) ? null : base[i].getValue(newBean);
-			if (newVal != null) {
-			  String propName = (prefix == null) ? base[i].getName() : prefix + base[i].getName();
-				map.put(propName, new ValuePair(newVal, null));
-			}
-		}
-
-		diffAssocOne(prefix, newBean, desc, map);
-		diffEmbedded(prefix, newBean, desc, map);
-	}
-
-	/**
-	 * Check the Embedded bean properties for differences.
-	 * <p>
-	 * If ANY of the properties are different then the whole Embedded bean is
-	 * determined to be different as is added to the map.
-	 * </p>
-	 */
-	private static void diffEmbedded(String prefix, EntityBean newBean, BeanDescriptor<?> desc, Map<String, ValuePair> map) {
-
-    BeanPropertyAssocOne<?>[] emb = desc.propertiesEmbedded();
-
-    for (int i = 0; i < emb.length; i++) {
-      EntityBean newVal = (EntityBean) emb[i].getValue(newBean);
-
-      if (newVal != null) {
-        String propName = (prefix == null) ? emb[i].getName() : prefix + emb[i].getName();
-
-        BeanDescriptor<?> embDesc = emb[i].getTargetDescriptor();
-        diff(propName + ".", map, newVal, embDesc);
-      }
-    }
-  }
-
-	/**
-	 * If the properties are different by null OR if the id value is different,
-	 * then add the Assoc One bean to the map.
-	 */
-	private static void diffAssocOne(String prefix, EntityBean newBean, BeanDescriptor<?> desc, Map<String, ValuePair> map) {
-
-		BeanPropertyAssocOne<?>[] ones = desc.propertiesOne();
-
-		for (int i = 0; i < ones.length; i++) {
-			Object newVal = ones[i].getValue(newBean);
-
-			if (newVal != null) {
-        BeanDescriptor<?> oneDesc = ones[i].getTargetDescriptor();
-        Object newId = oneDesc.getId((EntityBean)newVal);
-        if (newId != null) {
-          String propName = (prefix == null) ? ones[i].getName() : prefix + ones[i].getName();
-          String idName =  oneDesc.getIdProperty().getName();
-          map.put(propName + "." + idName, new ValuePair(newId, null));
-        }
-      }
-		}
-	}
-
-}

src/main/java/com/avaje/ebeaninternal/server/core/DiffHelpUpdate.java

@@ -0,0 +1,62 @@
+package com.avaje.ebeaninternal.server.core;
+
+import com.avaje.ebean.ValuePair;
+import com.avaje.ebean.bean.EntityBean;
+import com.avaje.ebeaninternal.server.deploy.BeanDescriptor;
+import com.avaje.ebeaninternal.server.deploy.BeanProperty;
+import com.avaje.ebeaninternal.server.deploy.BeanPropertyAssocMany;
+import com.avaje.ebeaninternal.server.deploy.BeanPropertyAssocOne;
+
+import java.util.Iterator;
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+/**
+ * Flattens an existing diff map converting assoc one beans into the associated id changes.
+ */
+public class DiffHelpUpdate {
+
+  public static Map<String, ValuePair> flatten(Map<String, ValuePair> values, BeanDescriptor<?> desc) {
+
+    Map<String, ValuePair> flattened = null;
+
+    Iterator<Map.Entry<String, ValuePair>> iterator = values.entrySet().iterator();
+    while (iterator.hasNext()) {
+      Map.Entry<String, ValuePair> entry = iterator.next();
+      BeanProperty beanProperty = desc.getBeanProperty(entry.getKey());
+      if (beanProperty instanceof BeanPropertyAssocMany) {
+        // filter out assoc many bean properties
+        iterator.remove();
+
+      } else if (beanProperty instanceof BeanPropertyAssocOne) {
+        BeanPropertyAssocOne assoc = (BeanPropertyAssocOne)beanProperty;
+        if (!assoc.isEmbedded()) {
+          // flatten for assoc one beans
+          if (flattened == null) {
+            flattened = new LinkedHashMap<String, ValuePair>();
+          }
+          flattenToId(flattened, entry, beanProperty, assoc);
+          iterator.remove();
+        }
+      }
+    }
+
+    if (flattened != null) {
+      values.putAll(flattened);
+    }
+
+    return values;
+  }
+
+  private static void flattenToId(Map<String, ValuePair> flattened, Map.Entry<String, ValuePair> entry, BeanProperty beanProperty, BeanPropertyAssocOne assoc) {
+
+    BeanDescriptor<?> oneDesc = assoc.getTargetDescriptor();
+
+    ValuePair value = entry.getValue();
+    Object newId = value.getNewValue() == null ? null :  oneDesc.getId((EntityBean)value.getNewValue());
+    Object oldId = value.getOldValue() == null ? null :  oneDesc.getId((EntityBean)value.getOldValue());
+
+    String propName = beanProperty.getName() + "." + oneDesc.getIdProperty().getName();
+    flattened.put(propName, new ValuePair(newId, oldId));
+  }
+}

src/main/java/com/avaje/ebeaninternal/server/core/InternalConfiguration.java

@@ -9,15 +9,17 @@ import com.avaje.ebean.config.dbplatform.DbHistorySupport;
 import com.avaje.ebean.event.changelog.ChangeLogListener;
 import com.avaje.ebean.event.changelog.ChangeLogPrepare;
 import com.avaje.ebean.event.changelog.ChangeLogRegister;
+import com.avaje.ebean.event.readaudit.ReadAuditLogger;
+import com.avaje.ebean.event.readaudit.ReadAuditPrepare;
 import com.avaje.ebean.plugin.SpiServerPlugin;
 import com.avaje.ebean.text.json.JsonContext;
 import com.avaje.ebeaninternal.api.SpiBackgroundExecutor;
 import com.avaje.ebeaninternal.api.SpiEbeanServer;
 import com.avaje.ebeaninternal.server.autofetch.AutoFetchManager;
 import com.avaje.ebeaninternal.server.autofetch.AutoFetchManagerFactory;
+import com.avaje.ebeaninternal.server.changelog.DefaultChangeLogListener;
 import com.avaje.ebeaninternal.server.changelog.DefaultChangeLogPrepare;
 import com.avaje.ebeaninternal.server.changelog.DefaultChangeLogRegister;
-import com.avaje.ebeaninternal.server.changelog.DefaultChangeLogListener;
 import com.avaje.ebeaninternal.server.cluster.ClusterManager;
 import com.avaje.ebeaninternal.server.deploy.BeanDescriptorManager;
 import com.avaje.ebeaninternal.server.deploy.DeployOrmXml;
@@ -32,6 +34,8 @@ import com.avaje.ebeaninternal.server.persist.DefaultPersister;
 import com.avaje.ebeaninternal.server.query.CQueryEngine;
 import com.avaje.ebeaninternal.server.query.DefaultOrmQueryEngine;
 import com.avaje.ebeaninternal.server.query.DefaultRelationalQueryEngine;
+import com.avaje.ebeaninternal.server.readaudit.DefaultReadAuditLogger;
+import com.avaje.ebeaninternal.server.readaudit.DefaultReadAuditPrepare;
 import com.avaje.ebeaninternal.server.resource.ResourceManager;
 import com.avaje.ebeaninternal.server.resource.ResourceManagerFactory;
 import com.avaje.ebeaninternal.server.text.json.DJsonContext;
@@ -55,8 +59,6 @@ import java.util.Map;
 /**
  * Used to extend the ServerConfig with additional objects used to configure and
  * construct an EbeanServer.
- * 
- * @author rbygrave
  */
 public class InternalConfiguration {
 
@@ -106,8 +108,8 @@ public class InternalConfiguration {
   private final List<SpiServerPlugin> plugins = new ArrayList<SpiServerPlugin>();
 
   public InternalConfiguration(XmlConfig xmlConfig, ClusterManager clusterManager,
-      ServerCacheManager cacheManager, SpiBackgroundExecutor backgroundExecutor,
-      ServerConfig serverConfig, BootupClasses bootupClasses) {
+                               ServerCacheManager cacheManager, SpiBackgroundExecutor backgroundExecutor,
+                               ServerConfig serverConfig, BootupClasses bootupClasses) {
 
     this.jsonFactory = serverConfig.getJsonFactory();
     this.xmlConfig = xmlConfig;
@@ -157,7 +159,7 @@ public class InternalConfiguration {
    */
   public <T> T plugin(T maybePlugin) {
     if (maybePlugin instanceof SpiServerPlugin) {
-      plugins.add((SpiServerPlugin)maybePlugin);
+      plugins.add((SpiServerPlugin) maybePlugin);
     }
     return maybePlugin;
   }
@@ -191,6 +193,22 @@ public class InternalConfiguration {
     return plugin((listener != null) ? listener : new DefaultChangeLogListener());
   }
 
+  /**
+   * Return the ReadAuditLogger implementation to use.
+   */
+  public ReadAuditLogger getReadAuditLogger() {
+    ReadAuditLogger found = bootupClasses.getReadAuditLogger();
+    return plugin(found != null ? found : new DefaultReadAuditLogger());
+  }
+
+  /**
+   * Return the ReadAuditPrepare implementation to use.
+   */
+  public ReadAuditPrepare getReadAuditPrepare() {
+    ReadAuditPrepare found = bootupClasses.getReadAuditPrepare();
+    return plugin(found != null ? found : new DefaultReadAuditPrepare());
+  }
+
   /**
    * For 'As Of' queries return the number of bind variables per predicate.
    */
@@ -206,14 +224,14 @@ public class InternalConfiguration {
    * Create the TransactionManager taking into account autoCommit mode.
    */
   private TransactionManager createTransactionManager() {
-    
+
     if (isAutoCommitMode()) {
       return new AutoCommitTransactionManager(clusterManager, backgroundExecutor, serverConfig, beanDescriptorManager, this.getBootupClasses());
     }
-    
+
     return new TransactionManager(clusterManager, backgroundExecutor, serverConfig, beanDescriptorManager, this.getBootupClasses());
   }
-  
+
   /**
    * Return true if autoCommit mode is on.
    */
@@ -251,7 +269,6 @@ public class InternalConfiguration {
     return new DefaultPersister(server, binder, beanDescriptorManager);
   }
 
-
   public ServerCacheManager getCacheManager() {
     return cacheManager;
   }

src/main/java/com/avaje/ebeaninternal/server/core/OrmQueryRequest.java

@@ -1,5 +1,7 @@
 package com.avaje.ebeaninternal.server.core;
 
+import java.util.ArrayList;
+import java.util.Collection;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -374,7 +376,37 @@ public final class OrmQueryRequest<T> extends BeanRequest implements BeanQueryRe
 
     cacheKey = query.queryHash();
 
-    return beanDescriptor.queryCacheGet(cacheKey);
+    BeanCollection<T> cached = beanDescriptor.queryCacheGet(cacheKey);
+
+    if (cached != null && isAuditReads() && readAuditQueryType()) {
+      // raw sql can't use L2 cache so normal queries only in here
+      Collection<T> actualDetails = cached.getActualDetails();
+      List<Object> ids = new ArrayList<Object>(actualDetails.size());
+      for (T bean : actualDetails) {
+        ids.add(beanDescriptor.getIdForJson(bean));
+      }
+      beanDescriptor.readAuditMany(queryPlanHash.getPartialKey(), "l2-query-cache", ids);
+    }
+
+    return cached;
+  }
+
+  /**
+   * Return true if the query type contains bean data (not just ids etc) and hence we want to include
+   * it in read auditing.  Return false for row count and find ids queries.
+   */
+  private boolean readAuditQueryType() {
+    Type type = query.getType();
+    switch (type) {
+      case BEAN:
+      case ITERATE:
+      case LIST:
+      case SET:
+      case MAP:
+        return true;
+      default:
+        return false;
+    }
   }
 
   public void putToQueryCache(BeanCollection<T> queryResult) {
@@ -410,4 +442,14 @@ public final class OrmQueryRequest<T> extends BeanRequest implements BeanQueryRe
     int batchSize = query.getLazyLoadBatchSize();
     return (batchSize > 0) ? batchSize : ebeanServer.getLazyLoadBatchSize();
   }
+
+  /**
+   * Return true if read auditing is on for this query request.
+   * <p>
+   * This means that read audit is on for this bean type and that query has not explicitly disabled it.
+   * </p>
+   */
+  public boolean isAuditReads() {
+    return !query.isDisableReadAudit() && beanDescriptor.isReadAuditing();
+  }
 }

src/main/java/com/avaje/ebeaninternal/server/deploy/BeanDescriptor.java

@@ -19,6 +19,9 @@ import com.avaje.ebean.event.BeanQueryAdapter;
 import com.avaje.ebean.event.changelog.BeanChange;
 import com.avaje.ebean.event.changelog.ChangeLogFilter;
 import com.avaje.ebean.event.changelog.ChangeType;
+import com.avaje.ebean.event.readaudit.ReadAuditLogger;
+import com.avaje.ebean.event.readaudit.ReadAuditPrepare;
+import com.avaje.ebean.event.readaudit.ReadEvent;
 import com.avaje.ebean.meta.MetaBeanInfo;
 import com.avaje.ebean.meta.MetaQueryPlanStatistic;
 import com.avaje.ebean.plugin.SpiBeanType;
@@ -31,7 +34,7 @@ import com.avaje.ebeaninternal.api.TransactionEventTable.TableIUD;
 import com.avaje.ebeaninternal.server.cache.CachedBeanData;
 import com.avaje.ebeaninternal.server.core.CacheOptions;
 import com.avaje.ebeaninternal.server.core.DefaultSqlUpdate;
-import com.avaje.ebeaninternal.server.core.DiffHelpInsert;
+import com.avaje.ebeaninternal.server.core.DiffHelpUpdate;
 import com.avaje.ebeaninternal.server.core.InternString;
 import com.avaje.ebeaninternal.server.core.PersistRequest;
 import com.avaje.ebeaninternal.server.core.PersistRequestBean;
@@ -143,6 +146,11 @@ public class BeanDescriptor<T> implements MetaBeanInfo, SpiBeanType<T> {
   private final String baseTableVersionsBetween;
   private final boolean historySupport;
 
+  /**
+   * Set to true if read auditing is on for this bean type.
+   */
+  private final boolean readAuditing;
+
   /**
    * Map of BeanProperty Linked so as to preserve order.
    */
@@ -362,6 +370,7 @@ public class BeanDescriptor<T> implements MetaBeanInfo, SpiBeanType<T> {
     this.updateChangesOnly = deploy.isUpdateChangesOnly();
     this.compoundUniqueConstraints = deploy.getCompoundUniqueConstraints();
 
+    this.readAuditing = deploy.isReadAuditing();
     this.historySupport = deploy.isHistorySupport();
     this.baseTable = InternString.intern(deploy.getBaseTable());
     this.baseTableAsOf = deploy.getBaseTableAsOf();
@@ -608,6 +617,20 @@ public class BeanDescriptor<T> implements MetaBeanInfo, SpiBeanType<T> {
     }
   }
 
+  /**
+   * Return the ReadAuditLogger for logging read audit events.
+   */
+  public ReadAuditLogger getReadAuditLogger() {
+    return ebeanServer.getReadAuditLogger();
+  }
+
+  /**
+   * Return the ReadAuditPrepare for preparing read audit events prior to logging.
+   */
+  public ReadAuditPrepare getReadAuditPrepare() {
+    return ebeanServer.getReadAuditPrepare();
+  }
+
   /**
    * Return true if this request should be included in the change log.
    */
@@ -641,27 +664,20 @@ public class BeanDescriptor<T> implements MetaBeanInfo, SpiBeanType<T> {
    * Return the bean change for an update.
    */
   private BeanChange updateBeanChange(PersistRequestBean<T> request) {
-    return newBeanChange(request.getBeanId(), ChangeType.UPDATE, request.getEntityBeanIntercept().getDirtyValues());
+    return newBeanChange(request.getBeanId(), ChangeType.UPDATE, diffFlatten(request.getEntityBeanIntercept().getDirtyValues()));
   }
 
   /**
    * Return the bean change for an insert.
    */
   private BeanChange insertBeanChange(PersistRequestBean<T> request) {
-    return newBeanChange(request.getBeanId(), ChangeType.INSERT, insertDiff(request.getEntityBean()));
+    return newBeanChange(request.getBeanId(), ChangeType.INSERT, diffForInsert(request.getEntityBean()));
   }
 
   private BeanChange newBeanChange(Object id, ChangeType changeType, Map<String, ValuePair> values) {
     return new BeanChange(getBaseTable(), id, changeType, values);
   }
 
-  /**
-   * For insert we create a Map of ValuePair to have the same structure as update.
-   */
-  private Map<String, ValuePair> insertDiff(EntityBean entityBean) {
-    return DiffHelpInsert.diff(entityBean, this);
-  }
-
   /**
    * Initialise the cache once the server has started.
    */
@@ -947,10 +963,55 @@ public class BeanDescriptor<T> implements MetaBeanInfo, SpiBeanType<T> {
   public void cacheHandleUpdate(Object id, PersistRequestBean<T> updateRequest) {
     cacheHelp.handleUpdate(id, updateRequest);
   }
-  
+
   /**
-   * Return the base table alias. This is always the first letter of the bean
-   * name.
+   * Prepare the read audit of a findFutureList() query.
+   */
+  public void readAuditFutureList(SpiQuery<T> spiQuery) {
+    if (isReadAuditing()) {
+      ReadEvent event = new ReadEvent(fullName);
+      // prepare in the foreground thread while we have the user context
+      // information (query is processed/executed later in bg thread)
+      readAuditPrepare(event);
+      spiQuery.setFutureFetchAudit(event);
+    }
+  }
+
+  /**
+   * Write a bean read to the read audit log.
+   */
+  public void readAuditBean(String queryKey, String bindLog, Object bean) {
+    ReadEvent event = new ReadEvent(fullName, queryKey, bindLog, getIdForJson(bean));
+    readAuditPrepare(event);
+    getReadAuditLogger().auditBean(event);
+  }
+
+  private void readAuditPrepare(ReadEvent event) {
+    ReadAuditPrepare prepare = getReadAuditPrepare();
+    if (prepare != null) {
+      prepare.prepare(event);
+    }
+  }
+
+  /**
+   * Write a many bean read to the read audit log.
+   */
+  public void readAuditMany(String queryKey, String bindLog, List<Object> ids) {
+    ReadEvent event = new ReadEvent(fullName, queryKey, bindLog, ids);
+    readAuditPrepare(event);
+    getReadAuditLogger().auditMany(event);
+  }
+
+  /**
+   * Write a futureList many read to the read audit log.
+   */
+  public void readAuditFutureMany(ReadEvent event) {
+    // this has already been prepared (in foreground thread)
+    getReadAuditLogger().auditMany(event);
+  }
+
+  /**
+   * Return the base table alias. This is always the first letter of the bean name.
    */
   public String getBaseTableAlias() {
     return baseTableAlias;
@@ -1208,6 +1269,9 @@ public class BeanDescriptor<T> implements MetaBeanInfo, SpiBeanType<T> {
       if (d != null) {
         Object shareableBean = d.getSharableBean();
         if (shareableBean != null) {
+          if (isReadAuditing()) {
+            readAuditBean("ref", "", shareableBean);
+          }
           return (T) shareableBean;
         }
       }
@@ -1362,6 +1426,26 @@ public class BeanDescriptor<T> implements MetaBeanInfo, SpiBeanType<T> {
     return getId((EntityBean) bean);
   }
 
+  /**
+   * Return the Id value for the bean with embeddedId beans converted into maps.
+   * <p>
+   * The usage is to provide simple id types for JSON processing (for embeddedId's).
+   * </p>
+   */
+  public Object getIdForJson(Object bean) {
+    return idBinder.getIdForJson((EntityBean) bean);
+  }
+
+  /**
+   * Convert the idValue assuming embeddedId values are Maps.
+   * <p>
+   * The usage is to provide simple id types for JSON processing (for embeddedId's).
+   * </p>
+   */
+  public Object convertIdFromJson(Object idValue) {
+    return idBinder.convertIdFromJson(idValue);
+  }
+  
   /**
    * Return the default order by that may need to be added if a many property is
    * included in the query.
@@ -1374,7 +1458,7 @@ public class BeanDescriptor<T> implements MetaBeanInfo, SpiBeanType<T> {
    * Convert the type of the idValue if required.
    */
   public Object convertId(Object idValue) {
-    return idBinder.convertSetId(idValue, null);
+    return idBinder.convertId(idValue);
   }
 
   /**
@@ -1777,6 +1861,13 @@ public class BeanDescriptor<T> implements MetaBeanInfo, SpiBeanType<T> {
     }
   }
 
+  /**
+   * Return true if read auditing is on this entity bean.
+   */
+  public boolean isReadAuditing() {
+    return readAuditing;
+  }
+
   /**
    * Return true if this entity bean has history support.
    */
@@ -1943,6 +2034,61 @@ public class BeanDescriptor<T> implements MetaBeanInfo, SpiBeanType<T> {
     }
   }
 
+  /**
+   * Flatten the diff that comes from the entity bean intercept.
+   */
+  Map<String, ValuePair> diffFlatten(Map<String, ValuePair> diff) {
+    return DiffHelpUpdate.flatten(diff, this);
+  }
+
+  /**
+   * Return a map of the differences between a and b.
+   * <p>
+   * A and B must be of the same type. B can be null, in which case the 'dirty
+   * diff' of a is returned.
+   * </p>
+   * <p>
+   * This intentionally does not include as OneToMany or ManyToMany properties.
+   * </p>
+   */
+  public Map<String, ValuePair> diffForInsert(EntityBean newBean) {
+
+    Map<String, ValuePair> map = new LinkedHashMap<String, ValuePair>();
+    diffForInsert(null, map, newBean);
+    return map;
+  }
+
+  /**
+   * Populate the diff for inserts with flattened non-null property values.
+   */
+  public void diffForInsert(String prefix, Map<String, ValuePair> map, EntityBean newBean) {
+    for (int i = 0; i < propertiesBaseScalar.length; i++) {
+      propertiesBaseScalar[i].diffForInsert(prefix, map, newBean);
+    }
+    for (int i = 0; i < propertiesOne.length; i++) {
+      propertiesOne[i].diffForInsert(prefix, map, newBean);
+    }
+    for (int i = 0; i < propertiesEmbedded.length; i++) {
+      propertiesEmbedded[i].diffForInsert(prefix, map, newBean);
+    }
+  }
+
+  /**
+   * Populate the diff for updates with flattened non-null property values.
+   */
+  public void diff(String prefix, Map<String, ValuePair> map, EntityBean newBean, EntityBean oldBean) {
+
+    for (int i = 0; i < propertiesBaseScalar.length; i++) {
+      propertiesBaseScalar[i].diff(prefix, map, newBean, oldBean);
+    }
+    for (int i = 0; i < propertiesOne.length; i++) {
+      propertiesOne[i].diff(prefix, map, newBean, oldBean);
+    }
+    for (int i = 0; i < propertiesEmbedded.length; i++) {
+      propertiesEmbedded[i].diff(prefix, map, newBean, oldBean);
+    }
+  }
+
   /**
    * All the BeanPropertyAssocOne that are not embedded. These are effectively
    * joined beans. For ManyToOne and OneToOne associations.

src/main/java/com/avaje/ebeaninternal/server/deploy/BeanDescriptorCacheHelp.java

@@ -432,6 +432,9 @@ public final class BeanDescriptorCacheHelp<T> {
         if (beanLog.isTraceEnabled()) {
           beanLog.trace("   GET {}({}) - hit shared bean", cacheName, id);
         }
+        if (desc.isReadAuditing()) {
+          desc.readAuditBean("l2", "", bean);
+        }
         return (T) bean;
       }
     }
@@ -450,6 +453,9 @@ public final class BeanDescriptorCacheHelp<T> {
     if (beanLog.isTraceEnabled()) {
       beanLog.trace("   GET {}({}) - hit", cacheName, id);
     }
+    if (desc.isReadAuditing()) {
+      desc.readAuditBean("l2", "", bean);
+    }
     return (T) bean;
   }
 

src/main/java/com/avaje/ebeaninternal/server/deploy/BeanProperty.java

@@ -1,5 +1,6 @@
 package com.avaje.ebeaninternal.server.deploy;
 
+import com.avaje.ebean.ValuePair;
 import com.avaje.ebean.bean.EntityBean;
 import com.avaje.ebean.config.EncryptKey;
 import com.avaje.ebean.config.dbplatform.DbEncryptFunction;
@@ -21,6 +22,7 @@ import com.avaje.ebeaninternal.server.text.json.ReadJson;
 import com.avaje.ebeaninternal.server.text.json.WriteJson;
 import com.avaje.ebeaninternal.server.type.DataBind;
 import com.avaje.ebeaninternal.server.type.ScalarType;
+import com.avaje.ebeaninternal.util.ValueUtil;
 import com.fasterxml.jackson.core.JsonToken;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -33,6 +35,7 @@ import java.lang.reflect.Field;
 import java.sql.SQLException;
 import java.sql.Types;
 import java.util.List;
+import java.util.Map;
 
 /**
  * Description of a property of a bean. Includes its deployment information such
@@ -1098,4 +1101,34 @@ public class BeanProperty implements ElPropertyValue {
       }
     }
   }
+
+  /**
+   * Populate diff map for insert if the property is not null.
+   */
+  public void diffForInsert(String prefix, Map<String, ValuePair> map, EntityBean newBean) {
+    Object newVal = (newBean == null) ? null : getValue(newBean);
+    if (newVal != null) {
+      String propName = (prefix == null) ? name : prefix + "." + name;
+      map.put(propName, new ValuePair(newVal, null));
+    }
+  }
+
+  /**
+   * Populate diff map comparing the property values between the beans.
+   */
+  public void diff(String prefix, Map<String, ValuePair> map, EntityBean newBean, EntityBean oldBean) {
+    Object newVal = (newBean == null) ? null : getValue(newBean);
+    Object oldVal = (oldBean == null) ? null : getValue(oldBean);
+    diffVal(prefix, map, newVal, oldVal);
+  }
+
+  /**
+   * Populate diff map comparing the property values.
+   */
+  public void diffVal(String prefix, Map<String, ValuePair> map, Object newVal, Object oldVal) {
+    if (!ValueUtil.areEqual(newVal, oldVal)) {
+      String propName = (prefix == null) ? name : prefix + "." + name;
+      map.put(propName, new ValuePair(newVal, oldVal));
+    }
+  }
 }

src/main/java/com/avaje/ebeaninternal/server/deploy/BeanPropertyAssocOne.java

@@ -4,6 +4,7 @@ import com.avaje.ebean.EbeanServer;
 import com.avaje.ebean.Query;
 import com.avaje.ebean.SqlUpdate;
 import com.avaje.ebean.Transaction;
+import com.avaje.ebean.ValuePair;
 import com.avaje.ebean.bean.EntityBean;
 import com.avaje.ebean.bean.EntityBeanIntercept;
 import com.avaje.ebean.bean.PersistenceContext;
@@ -26,6 +27,7 @@ import java.sql.SQLException;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 
 /**
  * Property mapped to a joined bean.
@@ -318,6 +320,53 @@ public class BeanPropertyAssocOne<T> extends BeanPropertyAssoc<T> {
     return importedPrimaryKey;
   }
 
+  @Override
+  public void diffForInsert(String prefix, Map<String, ValuePair> map, EntityBean newBean) {
+    Object newEmb = (newBean == null) ? null : getValue(newBean);
+    if (newEmb != null) {
+      prefix = (prefix == null) ? name : prefix + "." + name;
+      if (embedded) {
+        getTargetDescriptor().diffForInsert(prefix, map, (EntityBean) newEmb);
+      } else {
+        // we are only interested in the Id value
+        BeanDescriptor<T> targetDescriptor = getTargetDescriptor();
+        BeanProperty idProperty = targetDescriptor.getIdProperty();
+        idProperty.diffForInsert(prefix, map, (EntityBean) newEmb);
+      }
+    }
+  }
+
+  @Override
+  public void diff(String prefix, Map<String, ValuePair> map, EntityBean newBean, EntityBean oldBean) {
+
+    Object newEmb = (newBean == null) ? null : getValue(newBean);
+    Object oldEmb = (oldBean == null) ? null : getValue(oldBean);
+    if (newEmb == null && oldEmb == null) {
+      return;
+    }
+
+    if (embedded) {
+      prefix = (prefix == null) ? name : prefix + "." + name;
+      BeanDescriptor<T> targetDescriptor = getTargetDescriptor();
+      targetDescriptor.diff(prefix, map, (EntityBean) newEmb, (EntityBean) oldEmb);
+
+    } else {
+      // we are only interested in the Id value
+      newBean = (EntityBean)newEmb;
+      oldBean = (EntityBean)oldEmb;
+
+      BeanDescriptor<T> targetDescriptor = getTargetDescriptor();
+      BeanProperty idProperty = targetDescriptor.getIdProperty();
+
+      Object newId = (newBean == null) ? null : idProperty.getValue(newBean);
+      Object oldId = (oldBean == null) ? null : idProperty.getValue(oldBean);
+      if (newId != null || oldId != null) {
+        prefix = (prefix == null) ? name : prefix + "." + name;
+        idProperty.diffVal(prefix, map, newId, oldId);
+      }
+    }
+  }
+
   /**
    * Same as getPropertyType(). Return the type of the bean this property
    * represents.

src/main/java/com/avaje/ebeaninternal/server/deploy/id/IdBinder.java

@@ -84,6 +84,28 @@ public interface IdBinder {
    */
   Object[] getBindValues(Object idValue);
 
+
+  /**
+   * For EmbeddedId convert the idValue into a simple map.
+   * Otherwise the idValue is just returned as is.
+   * <p>
+   * This is used to provide a simple JSON serializable version of the id value.
+   * </p>
+   */
+  Object getIdForJson(EntityBean idValue);
+
+  /**
+   * For EmbeddedId the value is assumed to be a Map and this is
+   * takes the values from the map and builds an embedded id bean.
+   * <p>
+   * For other simple id's this just returns the value (no conversion required).
+   * </p>
+   * <p>
+   * This is used to provide a simple JSON serializable version of the id value.
+   * </p>
+   */
+  Object convertIdFromJson(Object value);
+
   /**
    * Return the id values for a given bean.
    */
@@ -171,4 +193,8 @@ public interface IdBinder {
    */
   Object convertSetId(Object idValue, EntityBean bean);
 
+  /**
+   * Cast or convert the Id value if necessary.
+   */
+  Object convertId(Object idValue);
 }

src/main/java/com/avaje/ebeaninternal/server/deploy/id/IdBinderEmbedded.java

@@ -15,7 +15,9 @@ import java.io.DataInput;
 import java.io.DataOutput;
 import java.io.IOException;
 import java.sql.SQLException;
+import java.util.LinkedHashMap;
 import java.util.List;
+import java.util.Map;
 
 /**
  * Bind an Id that is an Embedded bean.
@@ -225,6 +227,36 @@ public final class IdBinderEmbedded implements IdBinder {
     return bindvalues;
   }
 
+  /**
+   * Convert from embedded bean to Map.
+   */
+  @Override
+  public Object getIdForJson(EntityBean bean) {
+
+    EntityBean ebValue = (EntityBean)embIdProperty.getValue(bean);
+    Map<String,Object> map = new LinkedHashMap<String, Object>();
+    for (int i = 0; i < props.length; i++) {
+      map.put(props[i].getName(), props[i].getValue(ebValue));
+    }
+    return map;
+  }
+
+  /**
+   * Convert back from a Map to embedded bean.
+   */
+  public Object convertIdFromJson(Object value) {
+
+    Map<String,Object> map = (Map<String, Object>)value;
+
+    EntityBean idValue = idDesc.createEntityBean();
+    for (int i = 0; i < props.length; i++) {
+      Object val  = map.get(props[i].getName());
+      props[i].setValue(idValue, val);
+    }
+    return idValue;
+  }
+
+
   public void bindId(DefaultSqlUpdate sqlUpdate, Object value) {
     for (int i = 0; i < props.length; i++) {
       Object embFieldValue = props[i].getValue((EntityBean) value);
@@ -386,6 +418,12 @@ public final class IdBinderEmbedded implements IdBinder {
     return sb.toString();
   }
 
+  @Override
+  public Object convertId(Object idValue) {
+    // can not cast/convert if it is embedded
+    return idValue;
+  }
+
   public Object convertSetId(Object idValue, EntityBean bean) {
 
     // can not cast/convert if it is embedded

src/main/java/com/avaje/ebeaninternal/server/deploy/id/IdBinderEmpty.java

@@ -98,6 +98,16 @@ public final class IdBinderEmpty implements IdBinder {
     return new Object[]{idValue};
   }
 
+  @Override
+  public Object getIdForJson(EntityBean bean) {
+    return null;
+  }
+
+  @Override
+  public Object convertIdFromJson(Object value) {
+    return value;
+  }
+
   public void bindId(DefaultSqlUpdate sqlUpdate, Object value) {
 
   }
@@ -124,6 +134,11 @@ public final class IdBinderEmpty implements IdBinder {
     return idValue;
   }
 
+  @Override
+  public Object convertId(Object idValue) {
+    return idValue;
+  }
+
   public Object readData(DataInput dataOutput) throws IOException {
     return null;
   }

src/main/java/com/avaje/ebeaninternal/server/deploy/id/IdBinderSimple.java

@@ -131,6 +131,17 @@ public final class IdBinderSimple implements IdBinder {
     request.addBindValue(value);
   }
 
+  @Override
+  public Object getIdForJson(EntityBean bean) {
+    return idProperty.getValue(bean);
+  }
+
+  @Override
+  public Object convertIdFromJson(Object value) {
+    // handle simple type conversion if required
+    return convertId(value);
+  }
+
   public void bindId(DefaultSqlUpdate sqlUpdate, Object value) {
     sqlUpdate.addParameter(value);
   }
@@ -193,6 +204,14 @@ public final class IdBinderSimple implements IdBinder {
     return sb.toString();
   }
 
+  public Object convertId(Object idValue) {
+
+    if (!idValue.getClass().equals(expectedType)) {
+      return scalarType.toBeanType(idValue);
+    }
+    return idValue;
+  }
+
   public Object convertSetId(Object idValue, EntityBean bean) {
 
     if (!idValue.getClass().equals(expectedType)) {

src/main/java/com/avaje/ebeaninternal/server/deploy/meta/DeployBeanDescriptor.java

@@ -120,6 +120,8 @@ public class DeployBeanDescriptor<T> {
 
   private boolean historySupport;
 
+  private boolean readAuditing;
+
   private TableName baseTableFull;
 
   private String[] properties;
@@ -185,6 +187,20 @@ public class DeployBeanDescriptor<T> {
     return historySupport;
   }
 
+  /**
+   * Set read auditing on for this entity bean.
+   */
+  public void setReadAuditing() {
+    readAuditing = true;
+  }
+
+  /**
+   * Return true if read auditing is on for this entity bean.
+   */
+  public boolean isReadAuditing() {
+    return readAuditing;
+  }
+
   public boolean isScalaObject() {
     Class<?>[] interfaces = beanType.getInterfaces();
     for (int i = 0; i < interfaces.length; i++) {
@@ -741,7 +757,7 @@ public class DeployBeanDescriptor<T> {
     }
     DeployBeanProperty p = ids.get(0);
     if (p instanceof DeployBeanPropertyAssocOne<?>) {
-      return ((DeployBeanPropertyAssocOne<?>)p).isCompound();
+      return ((DeployBeanPropertyAssocOne<?>) p).isCompound();
     } else {
       return !p.isDbNumberType();
     }
@@ -859,17 +875,17 @@ public class DeployBeanDescriptor<T> {
   }
 
   /**
-   * Check valid mapping annotations on the class hierarchy. 
+   * Check valid mapping annotations on the class hierarchy.
    */
   private void checkInheritance(Class<?> beanType) {
-    
+
     Class<?> parent = beanType.getSuperclass();
     if (parent == null || Object.class.equals(parent)) {
       // all good
       return;
     }
     if (parent.isAnnotationPresent(Entity.class)) {
-      String msg = "Checking "+getBeanType()+" and found "+parent+" that has @Entity annotation rather than MappedSuperclass?";
+      String msg = "Checking " + getBeanType() + " and found " + parent + " that has @Entity annotation rather than MappedSuperclass?";
       throw new IllegalStateException(msg);
     }
     if (parent.isAnnotationPresent(MappedSuperclass.class)) {

src/main/java/com/avaje/ebeaninternal/server/deploy/parse/AnnotationClass.java

@@ -14,6 +14,7 @@ import com.avaje.ebean.annotation.History;
 import com.avaje.ebean.annotation.Index;
 import com.avaje.ebean.annotation.NamedUpdate;
 import com.avaje.ebean.annotation.NamedUpdates;
+import com.avaje.ebean.annotation.ReadAudit;
 import com.avaje.ebean.annotation.UpdateMode;
 import com.avaje.ebean.config.TableName;
 import com.avaje.ebeaninternal.server.core.CacheOptions;
@@ -97,6 +98,11 @@ public class AnnotationClass extends AnnotationParser {
       }
     }
 
+    ReadAudit readAudit = cls.getAnnotation(ReadAudit.class);
+    if (readAudit != null) {
+      descriptor.setReadAuditing();
+    }
+
     History history = cls.getAnnotation(History.class);
     if (history != null) {
       descriptor.setHistorySupport();

src/main/java/com/avaje/ebeaninternal/server/loadcontext/DLoadBeanContext.java

@@ -54,6 +54,9 @@ public class DLoadBeanContext extends DLoadBaseContext implements LoadBeanContex
     query.asOf(parent.getAsOf());
     query.setParentNode(objectGraphNode);
     query.setLazyLoadProperty(lazyLoadProperty);
+    if (parent.isDisableReadAudit()) {
+      query.setDisableReadAuditing();
+    }
 
     if (queryProps != null) {
       queryProps.configureBeanQuery(query);

src/main/java/com/avaje/ebeaninternal/server/loadcontext/DLoadContext.java

@@ -41,6 +41,7 @@ public class DLoadContext implements LoadContext {
   private final boolean excludeBeanCache;
   private final int defaultBatchSize;
   private final boolean disableLazyLoading;
+  private final boolean disableReadAudit;
 
   /**
    * The path relative to the root of the object graph.
@@ -65,6 +66,7 @@ public class DLoadContext implements LoadContext {
     SpiQuery<?> query = request.getQuery();
     this.asOf = query.getAsOf();
     this.readOnly = query.isReadOnly();
+    this.disableReadAudit = query.isDisableReadAudit();
     this.disableLazyLoading = query.isDisableLazyLoading();
     this.excludeBeanCache = Boolean.FALSE.equals(query.isUseBeanCache());
     this.useAutofetchManager = query.getAutoFetchManager() != null;
@@ -221,10 +223,23 @@ public class DLoadContext implements LoadContext {
     return readOnly;
   }
 
+  /**
+   * Return the 'as of' timestamp that should propagate to secondary queries.
+   */
   protected Timestamp getAsOf() {
     return asOf;
   }
 
+  /**
+   * Return true if disable read auditing should propagate to secondary queries.
+   */
+  protected boolean isDisableReadAudit() {
+    return disableReadAudit;
+  }
+
+  /**
+   * Return true if disable lazy loading should propagate to secondary queries.
+   */
   protected boolean isDisableLazyLoading() {
     return disableLazyLoading;
   }

src/main/java/com/avaje/ebeaninternal/server/loadcontext/DLoadManyContext.java

@@ -65,6 +65,9 @@ public class DLoadManyContext extends DLoadBaseContext implements LoadManyContex
     query.setDisableLazyLoading(parent.isDisableLazyLoading());
     query.asOf(parent.getAsOf());
     query.setParentNode(objectGraphNode);
+    if (parent.isDisableReadAudit()) {
+      query.setDisableReadAuditing();
+    }
 
     if (queryProps != null) {
       queryProps.configureBeanQuery(query);

src/main/java/com/avaje/ebeaninternal/server/query/CQuery.java

@@ -2,7 +2,14 @@ package com.avaje.ebeaninternal.server.query;
 
 import com.avaje.ebean.QueryIterator;
 import com.avaje.ebean.Version;
-import com.avaje.ebean.bean.*;
+import com.avaje.ebean.bean.BeanCollection;
+import com.avaje.ebean.bean.EntityBean;
+import com.avaje.ebean.bean.EntityBeanIntercept;
+import com.avaje.ebean.bean.NodeUsageCollector;
+import com.avaje.ebean.bean.NodeUsageListener;
+import com.avaje.ebean.bean.ObjectGraphNode;
+import com.avaje.ebean.bean.PersistenceContext;
+import com.avaje.ebean.event.readaudit.ReadEvent;
 import com.avaje.ebeaninternal.api.SpiQuery;
 import com.avaje.ebeaninternal.api.SpiQuery.Mode;
 import com.avaje.ebeaninternal.api.SpiTransaction;
@@ -10,7 +17,11 @@ import com.avaje.ebeaninternal.server.autofetch.AutoFetchManager;
 import com.avaje.ebeaninternal.server.core.Message;
 import com.avaje.ebeaninternal.server.core.OrmQueryRequest;
 import com.avaje.ebeaninternal.server.core.SpiOrmQueryRequest;
-import com.avaje.ebeaninternal.server.deploy.*;
+import com.avaje.ebeaninternal.server.deploy.BeanCollectionHelp;
+import com.avaje.ebeaninternal.server.deploy.BeanCollectionHelpFactory;
+import com.avaje.ebeaninternal.server.deploy.BeanDescriptor;
+import com.avaje.ebeaninternal.server.deploy.BeanPropertyAssocMany;
+import com.avaje.ebeaninternal.server.deploy.DbReadContext;
 import com.avaje.ebeaninternal.server.lib.util.StringHelper;
 import com.avaje.ebeaninternal.server.type.DataBind;
 import com.avaje.ebeaninternal.server.type.DataReader;
@@ -24,6 +35,7 @@ import java.sql.PreparedStatement;
 import java.sql.ResultSet;
 import java.sql.SQLException;
 import java.util.ArrayList;
+import java.util.Collection;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.TimeUnit;
@@ -163,6 +175,16 @@ public class CQuery<T> implements DbReadContext, CancelableQuery {
 
   private long executionTimeMicros;
 
+  /**
+   * Flag set when findIterate is being read audited.
+   */
+  private boolean auditFindIterate;
+
+  /**
+   * A buffer of Ids collected for findIterate auditing.
+   */
+  private List<Object> auditFindIterateIds;
+
   /**
    * Create the Sql select based on the request.
    */
@@ -295,7 +317,7 @@ public class CQuery<T> implements DbReadContext, CancelableQuery {
       }
 
       if (forwardOnlyHint) {
-        // Use forward only hints for large resultset processing (Issue 56, MySql specific)
+        // Use forward only hints for large resultSet processing (Issue 56, MySql specific)
         pstmt = conn.prepareStatement(sql, ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
         pstmt.setFetchSize(Integer.MIN_VALUE);
       } else {
@@ -327,18 +349,24 @@ public class CQuery<T> implements DbReadContext, CancelableQuery {
   /**
    * Close the resources.
    * <p>
-   * The jdbc resultSet and statement need to be closed. Its important that this
-   * method is called.
+   * The JDBC resultSet and statement need to be closed. Its important that this method is called.
    * </p>
    */
   public void close() {
+    try {
+      if (auditFindIterateIds != null && !auditFindIterateIds.isEmpty()) {
+        auditIterateLogMessage();
+      }
+    } catch (Throwable e) {
+      logger.error("Error logging read audit logs", e);
+    }
     try {
       if (dataReader != null) {
         dataReader.close();
         dataReader = null;
       }
     } catch (SQLException e) {
-      logger.error(null, e);
+      logger.error("Error closing dataReader", e);
     }
     try {
       if (pstmt != null) {
@@ -346,7 +374,7 @@ public class CQuery<T> implements DbReadContext, CancelableQuery {
         pstmt = null;
       }
     } catch (SQLException e) {
-      logger.error(null, e);
+      logger.error("Error closing preparedStatement", e);
     }
   }
 
@@ -464,6 +492,9 @@ public class CQuery<T> implements DbReadContext, CancelableQuery {
   }
 
   protected EntityBean next() {
+    if (auditFindIterate) {
+      auditIterateNextBean();
+    }
     return nextBean;
   }
 
@@ -610,8 +641,7 @@ public class CQuery<T> implements DbReadContext, CancelableQuery {
   }
 
   /**
-   * Create a PersistenceException including interesting information like the
-   * bindLog and sql used.
+   * Create a PersistenceException including interesting information like the bindLog and sql used.
    */
   public PersistenceException createPersistenceException(SQLException e) {
 
@@ -619,8 +649,7 @@ public class CQuery<T> implements DbReadContext, CancelableQuery {
   }
 
   /**
-   * Create a PersistenceException including interesting information like the
-   * bindLog and sql used.
+   * Create a PersistenceException including interesting information like the bindLog and sql used.
    */
   public static PersistenceException createPersistenceException(SQLException e, SpiTransaction t, String bindLog, String sql) {
 
@@ -680,4 +709,71 @@ public class CQuery<T> implements DbReadContext, CancelableQuery {
     this.currentPathMap = currentPathMap;
   }
 
+  /**
+   * A find bean query with read auditing so build and log the ReadEvent.
+   */
+  public void auditFind(EntityBean bean) {
+    if (bean != null) {
+      // only audit when a bean was actually found
+      desc.readAuditBean(queryPlan.getAuditQueryKey(), bindLog, bean);
+    }
+  }
+
+  /**
+   * a find many query with read auditing so build the ReadEvent and log it.
+   */
+  public void auditFindMany() {
+
+    if (!collection.isEmpty()) {
+      // get the id values of the underlying collection
+      List<Object> ids = new ArrayList<Object>(collection.size());
+      Collection<T> underlyingBeans = collection.getActualDetails();
+      for (T underlyingBean : underlyingBeans) {
+        ids.add(desc.getIdForJson(underlyingBean));
+      }
+      ReadEvent futureReadEvent = query.getFutureFetchAudit();
+      if (futureReadEvent == null) {
+        // normal query execution
+        desc.readAuditMany(queryPlan.getAuditQueryKey(), bindLog, ids);
+      } else {
+        // this query was executed via findFutureList() and the prepare()
+        // has already been called so set the details and log
+        futureReadEvent.setQueryKey(queryPlan.getAuditQueryKey());
+        futureReadEvent.setBindLog(bindLog);
+        futureReadEvent.setIds(ids);
+        desc.readAuditFutureMany(futureReadEvent);
+      }
+    }
+  }
+
+  /**
+   * Indicate that read auditing is occurring on a this findIterate query.
+   */
+  public void auditFindIterate() {
+    auditFindIterate = true;
+  }
+
+  /**
+   * Send the current buffer of findIterate collected ids to the audit log.
+   */
+  private void auditIterateLogMessage() {
+    desc.readAuditMany(queryPlan.getAuditQueryKey(), bindLog, auditFindIterateIds);
+    // create a new list on demand with the next bean/id
+    auditFindIterateIds = null;
+  }
+
+  /**
+   * Add the id to the audit id buffer and flush if needed in batches of 100.
+   */
+  private void auditIterateNextBean() {
+
+    if (auditFindIterateIds == null) {
+      auditFindIterateIds = new ArrayList<Object>(100);
+    }
+    auditFindIterateIds.add(desc.getIdForJson(nextBean));
+    if (auditFindIterateIds.size() >= 100) {
+      auditIterateLogMessage();
+    }
+  }
+
 }

src/main/java/com/avaje/ebeaninternal/server/query/CQueryBuilder.java

@@ -8,6 +8,7 @@ import com.avaje.ebean.config.dbplatform.DatabasePlatform;
 import com.avaje.ebean.config.dbplatform.SqlLimitRequest;
 import com.avaje.ebean.config.dbplatform.SqlLimitResponse;
 import com.avaje.ebean.config.dbplatform.SqlLimiter;
+import com.avaje.ebean.event.readaudit.ReadAuditQueryPlan;
 import com.avaje.ebean.text.PathProperties;
 import com.avaje.ebeaninternal.api.ManyWhereJoins;
 import com.avaje.ebeaninternal.api.SpiQuery;
@@ -269,6 +270,12 @@ public class CQueryBuilder {
       queryPlan = new CQueryPlan(request, res, sqlTree, false, predicates.getLogWhereSql());
     }
 
+    BeanDescriptor<T> desc = request.getBeanDescriptor();
+    if (desc.isReadAuditing()) {
+      // log the query plan based bean type (i.e. ignoring query disabling for logging the sql/plan)
+      desc.getReadAuditLogger().queryPlan(new ReadAuditQueryPlan(desc.getFullName(), queryPlan.getAuditQueryKey(), queryPlan.getSql()));
+    }
+
     // cache the query plan because we can reuse it and also
     // gather query performance statistics based on it.
     request.putQueryPlan(queryPlan);

src/main/java/com/avaje/ebeaninternal/server/query/CQueryEngine.java

@@ -180,6 +180,11 @@ public class CQueryEngine {
         logFindManySummary(cquery);
       }
 
+      if (request.isAuditReads()) {
+        // indicates we need to audit as the iterator progresses
+        cquery.auditFindIterate();
+      }
+
       return readIterate;
 
     } catch (SQLException e) {
@@ -218,6 +223,10 @@ public class CQueryEngine {
         logFindManySummary(cquery);
       }
 
+      if (request.isAuditReads()) {
+        cquery.auditFindMany();
+      }
+
       return versions;
 
     } catch (SQLException e) {
@@ -297,6 +306,10 @@ public class CQueryEngine {
         logFindManySummary(cquery);
       }
 
+      if (request.isAuditReads()) {
+        cquery.auditFindMany();
+      }
+
       request.executeSecondaryQueries();
 
       return beanCollection;
@@ -342,6 +355,10 @@ public class CQueryEngine {
         logFindBeanSummary(cquery);
       }
 
+      if (request.isAuditReads()) {
+        cquery.auditFind(bean);
+      }
+
       request.executeSecondaryQueries();
 
       return (T) bean;

src/main/java/com/avaje/ebeaninternal/server/query/CQueryPlan.java

@@ -1,5 +1,6 @@
 package com.avaje.ebeaninternal.server.query;
 
+import java.security.MessageDigest;
 import java.sql.ResultSet;
 import java.sql.SQLException;
 
@@ -14,6 +15,8 @@ import com.avaje.ebeaninternal.server.query.CQueryPlanStats.Snapshot;
 import com.avaje.ebeaninternal.server.type.DataBind;
 import com.avaje.ebeaninternal.server.type.DataReader;
 import com.avaje.ebeaninternal.server.type.RsetDataReader;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  * Represents a query for a given SQL statement.
@@ -36,6 +39,8 @@ import com.avaje.ebeaninternal.server.type.RsetDataReader;
  */
 public class CQueryPlan {
 
+  private static final Logger logger = LoggerFactory.getLogger(CQueryPlan.class);
+
   private final SpiEbeanServer server;
 
   private final boolean autofetchTuned;
@@ -61,6 +66,11 @@ public class CQueryPlan {
 
   private final Class<?> beanType;
 
+  /**
+   * Key used to identify the query plan in audit logging.
+   */
+  private volatile String auditQueryHash;
+
   /**
    * Create a query plan based on a OrmQueryRequest.
    */
@@ -141,6 +151,48 @@ public class CQueryPlan {
     return hash;
   }
 
+  /**
+   * Return a key used in audit logging to identify the query.
+   */
+  public String getAuditQueryKey() {
+    if (auditQueryHash == null) {
+      // volatile object assignment (so happy for multithreaded access)
+      auditQueryHash = calcAuditQueryKey();
+    }
+    return auditQueryHash;
+  }
+
+  private String calcAuditQueryKey() {
+    // rawSql needs to include the MD5 hash of the sql
+    return rawSql ? hash.getPartialKey() + "_" + getSqlMd5Hash() : hash.getPartialKey();
+  }
+
+  /**
+   * Return the MD5 hash of the underlying sql.
+   */
+  private String getSqlMd5Hash() {
+    try {
+      MessageDigest md = MessageDigest.getInstance("MD5");
+      byte[] digest = md.digest(sql.getBytes("UTF-8"));
+      return digestToHex(digest);
+    } catch (Exception e) {
+      logger.error("Failed to MD5 hash the rawSql query", e);
+      return "error";
+    }
+  }
+
+  /**
+   * Convert the digest into a hex value.
+   */
+  private String digestToHex(byte[] digest) {
+
+    StringBuilder sb = new StringBuilder();
+    for (int i = 0; i < digest.length; i++) {
+      sb.append(Integer.toString((digest[i] & 0xff) + 0x100, 16).substring(1));
+    }
+    return sb.toString();
+  }
+
   public String getSql() {
     return sql;
   }

src/main/java/com/avaje/ebeaninternal/server/querydefn/DefaultOrmQuery.java

@@ -17,6 +17,7 @@ import com.avaje.ebean.bean.ObjectGraphNode;
 import com.avaje.ebean.bean.ObjectGraphOrigin;
 import com.avaje.ebean.bean.PersistenceContext;
 import com.avaje.ebean.event.BeanQueryRequest;
+import com.avaje.ebean.event.readaudit.ReadEvent;
 import com.avaje.ebean.text.PathProperties;
 import com.avaje.ebeaninternal.api.BindParams;
 import com.avaje.ebeaninternal.api.HashQuery;
@@ -135,6 +136,11 @@ public class DefaultOrmQuery<T> implements SpiQuery<T> {
    */
   private boolean futureFetch;
 
+  /**
+   * Only used for read auditing with findFutureList() query.
+   */
+  private ReadEvent futureFetchAudit;
+
   private List<Object> partialIds;
 
   private int timeout;
@@ -173,6 +179,8 @@ public class DefaultOrmQuery<T> implements SpiQuery<T> {
   private Timestamp versionsStart;
   private Timestamp versionsEnd;
 
+  private boolean disableReadAudit;
+
   private int bufferFetchSizeHint;
 
   private boolean usageProfiling = true;
@@ -1320,6 +1328,16 @@ public class DefaultOrmQuery<T> implements SpiQuery<T> {
     return bufferFetchSizeHint;
   }
 
+  @Override
+  public Query<T> setDisableReadAuditing() {
+    this.disableReadAudit = true;
+    return this;
+  }
+
+  public boolean isDisableReadAudit() {
+    return disableReadAudit;
+  }
+
   public void setBeanCollectionTouched(BeanCollectionTouched notify) {
     this.beanCollectionTouched = notify;
   }
@@ -1344,6 +1362,16 @@ public class DefaultOrmQuery<T> implements SpiQuery<T> {
     this.futureFetch = backgroundFetch;
   }
 
+  @Override
+  public void setFutureFetchAudit(ReadEvent event) {
+    this.futureFetchAudit = event;
+  }
+
+  @Override
+  public ReadEvent getFutureFetchAudit() {
+    return futureFetchAudit;
+  }
+
   public void setCancelableQuery(CancelableQuery cancelableQuery) {
     synchronized (this) {
       this.cancelableQuery = cancelableQuery;

src/main/java/com/avaje/ebeaninternal/server/readaudit/DefaultReadAuditLogger.java

@@ -0,0 +1,141 @@
+package com.avaje.ebeaninternal.server.readaudit;
+
+import com.avaje.ebean.event.readaudit.ReadAuditLogger;
+import com.avaje.ebean.event.readaudit.ReadAuditQueryPlan;
+import com.avaje.ebean.event.readaudit.ReadEvent;
+import com.avaje.ebean.text.json.EJson;
+import com.fasterxml.jackson.core.JsonFactory;
+import com.fasterxml.jackson.core.JsonGenerator;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.io.StringWriter;
+import java.util.Map;
+
+/**
+ * Default implementation of ReadAuditLogger that writes the event in JSON format to standard loggers.
+ */
+public class DefaultReadAuditLogger implements ReadAuditLogger {
+
+  private static final Logger appLogger = LoggerFactory.getLogger(DefaultReadAuditLogger.class);
+
+  private static final Logger queryLogger = LoggerFactory.getLogger("org.avaje.ebean.ReadAuditQuery");
+
+  private static final Logger auditLogger = LoggerFactory.getLogger("org.avaje.ebean.ReadAudit");
+
+  protected final JsonFactory jsonFactory = new JsonFactory();
+
+  protected int defaultQueryBuffer = 500;
+
+  protected int defaultReadBuffer = 150;
+
+  /**
+   * Write the query plan details in JSON format to the logger.
+   */
+  @Override
+  public void queryPlan(ReadAuditQueryPlan queryPlan) {
+    try {
+      StringWriter writer = new StringWriter(defaultQueryBuffer);
+      JsonGenerator gen = jsonFactory.createGenerator(writer);
+
+      gen.writeStartObject();
+      String beanType = queryPlan.getBeanType();
+      if (beanType != null) {
+        gen.writeStringField("beanType", beanType);
+      }
+      String queryKey = queryPlan.getQueryKey();
+      if (queryKey != null) {
+        gen.writeStringField("queryKey", queryKey);
+      }
+      String sql = queryPlan.getSql();
+      if (sql != null) {
+        gen.writeStringField("sql", sql);
+      }
+      gen.writeEndObject();
+      gen.flush();
+      gen.close();
+
+      queryLogger.info(writer.toString());
+
+    } catch (IOException e) {
+      appLogger.error("Error writing Read audit event", e);
+    }
+  }
+
+  /**
+   * Write the bean read event details in JSON format to the logger.
+   */
+  @Override
+  public void auditBean(ReadEvent beanEvent) {
+
+    writeEvent(beanEvent);
+  }
+
+  /**
+   * Write the many beans read event details in JSON format to the logger.
+   */
+  @Override
+  public void auditMany(ReadEvent readMany) {
+    writeEvent(readMany);
+  }
+
+  protected void writeEvent(ReadEvent event) {
+
+    try {
+      StringWriter writer = new StringWriter(defaultReadBuffer);
+      JsonGenerator gen = jsonFactory.createGenerator(writer);
+      writeDetails(gen, event);
+
+      auditLogger.info(writer.toString());
+
+    } catch (IOException e) {
+      appLogger.error("Error writing Read audit event", e);
+    }
+  }
+
+  /**
+   * Write the details for the read bean or read many beans event.
+   */
+  protected void writeDetails(JsonGenerator gen, ReadEvent event) throws IOException {
+
+    gen.writeStartObject();
+    String source = event.getSource();
+    if (source != null) {
+      gen.writeStringField("source", source);
+    }
+    String userId = event.getUserId();
+    if (userId != null) {
+      gen.writeStringField("userId", userId);
+    }
+    String userIpAddress = event.getUserIpAddress();
+    if (userIpAddress != null) {
+      gen.writeStringField("userIpAddress", userIpAddress);
+    }
+    Map<String, String> userContext = event.getUserContext();
+    if (userContext != null && !userContext.isEmpty()) {
+      gen.writeObjectFieldStart("userContext");
+      for (Map.Entry<String, String> entry : userContext.entrySet()) {
+        gen.writeStringField(entry.getKey(), entry.getValue());
+      }
+      gen.writeEndObject();
+    }
+    gen.writeNumberField("eventTime", event.getEventTime());
+    gen.writeStringField("beanType", event.getBeanType());
+    gen.writeStringField("queryKey", event.getQueryKey());
+    gen.writeStringField("bindLog", event.getBindLog());
+    Object id = event.getId();
+    if (id != null) {
+      gen.writeFieldName("id");
+      EJson.write(id, gen);
+    } else {
+      gen.writeFieldName("ids");
+      EJson.writeCollection(event.getIds(), gen);
+    }
+
+    gen.writeEndObject();
+    gen.flush();
+    gen.close();
+  }
+
+}

src/main/java/com/avaje/ebeaninternal/server/readaudit/DefaultReadAuditPrepare.java

@@ -0,0 +1,19 @@
+package com.avaje.ebeaninternal.server.readaudit;
+
+import com.avaje.ebean.event.readaudit.ReadAuditPrepare;
+import com.avaje.ebean.event.readaudit.ReadEvent;
+
+/**
+ * A placeholder implementation for ReadAuditPrepare.
+ * <p>
+ * A real application specific implementation is required to obtain and set
+ * the user context information on the readEvent bean (like user id).
+ * </p>
+ */
+public class DefaultReadAuditPrepare implements ReadAuditPrepare {
+
+  @Override
+  public void prepare(ReadEvent readEvent) {
+    // do nothing by default.
+  }
+}