Merge pull request #2274 from ebean-orm/feature/json-dirty-detection-md5

Add support for JSON bean mutation detection via HASH and SOURCE of JSON string content

ebean-core/src/test/java/io/ebean/EbeanServer_refresh.java

@@ -11,7 +11,7 @@ import java.util.HashMap;
 import java.util.Map;
 
 import static org.assertj.core.api.Assertions.assertThat;
-import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.*;
 
 public class EbeanServer_refresh {
 
@@ -39,8 +39,11 @@ public class EbeanServer_refresh {
 
     assertEquals(rows, 1);
 
+    basic.setName("modify");
+    assertTrue(DB.getBeanState(basic).isDirty());
     server.refresh(basic);
     assertEquals(basic.getStatus(), EBasic.Status.ACTIVE);
+    assertFalse(DB.getBeanState(basic).isDirty());
   }
 
   @Test

ebean-core/src/test/java/io/ebeaninternal/server/util/ChecksumTest.java

@@ -0,0 +1,23 @@
+package io.ebeaninternal.server.util;
+
+import org.junit.Test;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+public class ChecksumTest {
+
+  @Test
+  public void checksum() {
+    final long val = Checksum.checksum("Hello world");
+    assertThat(val).isEqualTo(2346098258L);
+    assertThat(Checksum.checksum("Hello world")).isEqualTo(val);
+    assertThat(Checksum.checksum("hello world")).isNotEqualTo(val);
+  }
+
+  @Test
+  public void checksum_shortString() {
+    final long val0 = Checksum.checksum("2012-01-11");
+    final long val1 = Checksum.checksum("2012-10-02");
+    assertThat(val0).isNotEqualTo(val1);
+  }
+}

ebean-core/src/test/java/org/tests/json/TestDbJson_Jackson3.java

@@ -1,7 +1,11 @@
 package org.tests.json;
 
 import io.ebean.BaseTestCase;
+import io.ebean.BeanState;
 import io.ebean.DB;
+import io.ebean.ValuePair;
+import io.ebean.event.BeanPersistAdapter;
+import io.ebean.event.BeanPersistRequest;
 import io.ebeantest.LoggedSql;
 import org.junit.Test;
 import org.tests.model.json.EBasicJsonJackson3;
@@ -11,11 +15,33 @@ import org.tests.model.json.PlainBeanDirtyAware;
 
 import java.util.Arrays;
 import java.util.List;
+import java.util.Map;
 
 import static org.assertj.core.api.Assertions.assertThat;
 
 public class TestDbJson_Jackson3 extends BaseTestCase {
 
+  public static class EBasicJsonListPersistController extends BeanPersistAdapter {
+
+    private static Map<String, ValuePair> updatedValues;
+
+    @Override
+    public boolean isRegisterFor(Class<?> cls) {
+      return EBasicJsonList.class.isAssignableFrom(cls);
+    }
+
+    @Override
+    public boolean preInsert(BeanPersistRequest<?> request) {
+      updatedValues = request.getUpdatedValues();
+      return true;
+    }
+
+    @Override
+    public boolean preUpdate(BeanPersistRequest<?> request) {
+      updatedValues = request.getUpdatedValues();
+      return true;
+    }
+  }
   @Test
   public void updateIncludesJsonColumn_when_explicit_isMarkedDirty() {
 
@@ -24,6 +50,7 @@ public class TestDbJson_Jackson3 extends BaseTestCase {
     EBasicJsonJackson3 bean = new EBasicJsonJackson3();
     bean.setName("b1");
     bean.setPlainValue(contentBean);
+    bean.setPlainValue2(contentBean);
 
     bean.save();
 
@@ -32,20 +59,15 @@ public class TestDbJson_Jackson3 extends BaseTestCase {
 
     LoggedSql.start();
     found.save();
-
-    List<String> sql = LoggedSql.collect();
-    assertThat(sql).hasSize(1);
-    assertThat(sql.get(0)).contains("update ebasic_json_jackson3 set name=?, version=? where id=? and version=?");
+    expectedSql(0, "update ebasic_json_jackson3 set name=?, version=? where id=? and version=?");
 
     found.setName("b1-mod2");
     found.getPlainValue().setName("b");
-    found.getPlainValue().setMarkedDirty(true);
+    // found.getPlainValue().setMarkedDirty(true); // Irrelevant for SOURCE or HASH based mutation detection
 
     found.save();
-
-    sql = LoggedSql.stop();
-    assertThat(sql).hasSize(1);
-    assertThat(sql.get(0)).contains("update ebasic_json_jackson3 set name=?, plain_value=?, version=? where id=? and version=?");
+    expectedSql(0, "update ebasic_json_jackson3 set name=?, plain_value=?, version=? where id=? and version=?");
+    LoggedSql.stop();
 
     final EBasicJsonJackson3 found2 = DB.find(EBasicJsonJackson3.class, bean.getId());
 
@@ -69,11 +91,138 @@ public class TestDbJson_Jackson3 extends BaseTestCase {
     found.setName("p1-mod");
     found.setBeanList(null);
 
+    BeanState state = DB.getBeanState(found);
+    assertThat(state.getChangedProps()).containsExactlyInAnyOrder("name", "beanList");
+
+    ValuePair pair = state.getDirtyValues().get("name");
+    assertThat(pair.getNewValue()).isEqualTo("p1-mod");
+    assertThat(pair.getOldValue()).isEqualTo("p1");
+
+    pair = state.getDirtyValues().get("beanList");
+    assertThat(pair.getNewValue()).isEqualTo(null);
+    assertThat((List<PlainBean>)pair.getOldValue()).hasSize(1)
+      .extracting(PlainBean::getName).containsExactly("a");
+
+
     LoggedSql.start();
     DB.save(found);
 
-    final List<String> sql = LoggedSql.stop();
-    assertThat(sql).hasSize(1);
-    assertThat(sql.get(0)).contains("update ebasic_json_list set name=?, bean_list=?, plain_bean=?, version=? where id=?");
+    // plain_bean=?, no longer included with MD5 dirty detection
+    expectedSql(0, "update ebasic_json_list set name=?, bean_list=?, version=? where id=?");
+
+    assertThat(EBasicJsonListPersistController.updatedValues.entrySet())
+      .extracting(Map.Entry::toString)
+      .containsExactlyInAnyOrder("beanList=null,[name:a]","name=p1-mod,p1","version=2,1");
+
+    assertThat(DB.getBeanState(found).isDirty()).isFalse();
+
+    found.getPlainBean().setName("b");
+    assertThat(DB.getBeanState(found).isDirty()).isTrue();
+
+    state = DB.getBeanState(found);
+    assertThat(state.getChangedProps()).containsExactlyInAnyOrder("plainBean");
+    pair = state.getDirtyValues().get("plainBean");
+    assertThat(pair.getNewValue()).hasToString("name:b");
+    assertThat(pair.getOldValue()).hasToString("name:a");
+
+
+    LoggedSql.start();
+    DB.save(found);
+
+    // plain_bean=?, no longer included with MD5 dirty detection
+    expectedSql(0, "update ebasic_json_list set plain_bean=?, version=? where id=?");
+
+    assertThat(EBasicJsonListPersistController.updatedValues.entrySet())
+      .extracting(Map.Entry::toString)
+      .containsExactlyInAnyOrder("plainBean=name:b,name:a", "version=3,2");
+
+    LoggedSql.stop();
+  }
+
+  @Test
+  public void updateIncludesJsonColumn_when_list_loadedAndNotDirtyAware() {
+
+    PlainBean contentBean = new PlainBean("a", 42);
+    EBasicJsonList bean = new EBasicJsonList();
+    bean.setName("p1");
+    bean.setPlainBean(contentBean);
+    bean.setBeanList(Arrays.asList(contentBean));
+
+    DB.save(bean);
+    final EBasicJsonList found = DB.find(EBasicJsonList.class, bean.getId());
+    found.getBeanList().get(0).setName("p1-mod");
+
+    BeanState state = DB.getBeanState(found);
+    assertThat(state.getChangedProps()).containsExactlyInAnyOrder("beanList");
+  }
+
+  @Test
+  public void update_with_differentDbJsonSettings() {
+    PlainBeanDirtyAware contentBean1 = new PlainBeanDirtyAware("x", 42);
+    PlainBeanDirtyAware contentBean2 = new PlainBeanDirtyAware("y", 43);
+    PlainBeanDirtyAware contentBean3 = new PlainBeanDirtyAware("z", 44);
+
+    EBasicJsonJackson3 bean = new EBasicJsonJackson3();
+    bean.setName("b1");
+    bean.setPlainValue(contentBean1);
+    bean.setPlainValue2(contentBean2);
+    bean.setPlainValue3(contentBean3);
+
+    BeanState state = DB.getBeanState(bean);
+    // a new bean is not considered as dirty (thus have no changed props)
+    assertThat(state.isDirty()).isFalse();
+    assertThat(state.isNewOrDirty()).isTrue();
+    assertThat(state.getChangedProps()).isEmpty();
+
+    bean.save();
+
+    bean = DB.find(EBasicJsonJackson3.class, bean.getId());
+    state = DB.getBeanState(bean);
+    // a fresh loaded bean is also not considered as dirty
+    assertThat(state.isDirty()).isFalse();
+    assertThat(state.isNewOrDirty()).isFalse();
+    assertThat(state.getChangedProps()).isEmpty();
+
+    bean.getPlainValue().setName("a"); // has SOURCE
+
+    assertThat(state.isDirty()).isTrue();
+    assertThat(state.getChangedProps()).containsExactly("plainValue");
+
+    bean.getPlainValue2().setName("b");
+    assertThat(state.getChangedProps()).containsExactlyInAnyOrder("plainValue", "plainValue2");
+
+    bean.getPlainValue3().setName("c"); // has mutationDetection = NONE
+
+    Map<String, ValuePair> dirtyValues = state.getDirtyValues();
+    assertThat(dirtyValues).hasSize(2).containsKeys("plainValue", "plainValue2");
+
+    assertThat(dirtyValues.get("plainValue")).hasToString("name:a,name:x"); // SOURCE -> origValue present
+    assertThat(dirtyValues.get("plainValue2")).hasToString("name:b,null");  // without SOURCE no origValue present
+
+    LoggedSql.start();
+    bean.save();
+    expectedSql(0, "update ebasic_json_jackson3 set plain_value=?, plain_value2=?, version=? where id=?");
+
+    bean = DB.find(EBasicJsonJackson3.class, bean.getId());
+    LoggedSql.collect(); // ignore the select
+    assertThat(bean.getPlainValue().getName()).isEqualTo("a");
+    assertThat(bean.getPlainValue2().getName()).isEqualTo("b");
+    assertThat(bean.getPlainValue3().getName()).isEqualTo("z"); // value is not updated
+
+    bean.getPlainValue3().setName("c");
+    bean.getPlainValue3().setMarkedDirty(true); // This is ignored because it is MutationDetection.NONE
+    bean.save();
+    // no update as plainValue3 has MutationDetection.NONE (ModifyAwareType = NONE isn't an expected combination to me)
+    assertThat(LoggedSql.collect()).isEmpty();
+
+    bean.getPlainValue2().setName("b2"); // effectively HASH mode mutation detection
+    bean.save();
+    expectedSql(0, "update ebasic_json_jackson3 set plain_value2=?, version=? where id=? and version=?");
+
+    LoggedSql.stop();
+  }
+
+  private void expectedSql(int i, String s) {
+    assertThat(LoggedSql.collect().get(i)).contains(s);
   }
 }

ebean-core/src/test/java/org/tests/json/TestDbJson_List.java

@@ -113,7 +113,8 @@ public class TestDbJson_List extends BaseTestCase {
     List<String> sql = LoggedSqlCollector.stop();
 
     // we don't update the phone numbers (as they are not dirty)
-    assertSql(sql.get(0)).contains("update ebasic_json_list set name=?, plain_bean=?, version=? where");
+    // plain_bean=?, no longer included with MD5 dirty detection
+    assertSql(sql.get(0)).contains("update ebasic_json_list set name=?, version=? where");
   }
 
   public void update_when_dirty() {
@@ -126,7 +127,8 @@ public class TestDbJson_List extends BaseTestCase {
     List<String> sql = LoggedSqlCollector.stop();
 
     // we don't update the phone numbers (as they are not dirty)
-    assertSql(sql.get(0)).contains("update ebasic_json_list set plain_bean=?, tags=?, version=? where id=? and version=?");
+    // plain_bean=? not included using MD5 dirty detection
+    assertSql(sql.get(0)).contains("update ebasic_json_list set tags=?, version=? where id=? and version=?");
   }
 
   public void update_when_dirty_flags() {
@@ -139,7 +141,8 @@ public class TestDbJson_List extends BaseTestCase {
     List<String> sql = LoggedSqlCollector.stop();
 
     // we don't update the phone numbers (as they are not dirty)
-    assertSql(sql.get(0)).contains("update ebasic_json_list set plain_bean=?, flags=?, version=? where id=? and version=?;");
+    // plain_bean=? not included with MD5 dirty detection
+    assertSql(sql.get(0)).contains("update ebasic_json_list set flags=?, version=? where id=? and version=?;");
   }
 
   public void update_when_dirty_SetListMap() {
@@ -154,7 +157,8 @@ public class TestDbJson_List extends BaseTestCase {
     List<String> sql = LoggedSqlCollector.stop();
 
     // we don't update the phone numbers (as they are not dirty)
-    assertSql(sql.get(0)).contains("update ebasic_json_list set beans=?, bean_list=?, bean_map=?, plain_bean=?, version=? where id=? and version=?");
+    // plain_bean=? not included with MD5 dirty detection
+    assertSql(sql.get(0)).contains("update ebasic_json_list set beans=?, bean_list=?, bean_map=?, version=? where id=? and version=?");
   }
 
   @Test

ebean-core/src/test/java/org/tests/model/json/EBasicJsonJackson3.java

@@ -2,11 +2,15 @@ package org.tests.model.json;
 
 import io.ebean.Model;
 import io.ebean.annotation.DbJson;
+import io.ebean.annotation.MutationDetection;
 
 import javax.persistence.Entity;
 import javax.persistence.Id;
 import javax.persistence.Version;
 
+import static io.ebean.annotation.MutationDetection.NONE;
+import static io.ebean.annotation.MutationDetection.SOURCE;
+
 @Entity
 public class EBasicJsonJackson3 extends Model {
 
@@ -15,9 +19,15 @@ public class EBasicJsonJackson3 extends Model {
 
   String name;
 
-  @DbJson(length = 500)
+  @DbJson(length = 500, mutationDetection = SOURCE)
   PlainBeanDirtyAware plainValue;
 
+  @DbJson(length = 500)
+  PlainBeanDirtyAware plainValue2;
+
+  @DbJson(length = 500, mutationDetection = NONE)
+  PlainBeanDirtyAware plainValue3;
+
   @Version
   long version;
 
@@ -45,6 +55,22 @@ public class EBasicJsonJackson3 extends Model {
     this.plainValue = plainValue;
   }
 
+  public PlainBeanDirtyAware getPlainValue2() {
+    return plainValue2;
+  }
+
+  public void setPlainValue2(PlainBeanDirtyAware plainValue2) {
+    this.plainValue2 = plainValue2;
+  }
+
+  public PlainBeanDirtyAware getPlainValue3() {
+    return plainValue3;
+  }
+
+  public void setPlainValue3(PlainBeanDirtyAware plainValue3) {
+    this.plainValue3 = plainValue3;
+  }
+
   public long getVersion() {
     return version;
   }

ebean-core/src/test/java/org/tests/model/json/EBasicJsonList.java

@@ -7,12 +7,10 @@ import io.ebean.annotation.DbJsonType;
 import javax.persistence.Entity;
 import javax.persistence.Id;
 import javax.persistence.Version;
-import java.util.ArrayList;
-import java.util.LinkedHashMap;
-import java.util.LinkedHashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
+import java.util.*;
+
+import static io.ebean.annotation.MutationDetection.HASH;
+import static io.ebean.annotation.MutationDetection.SOURCE;
 
 @Entity
 public class EBasicJsonList {
@@ -25,13 +23,13 @@ public class EBasicJsonList {
   @DbJson(length = 700, name = "beans")
   Set<PlainBean> beanSet;
 
-  @DbJsonB
+  @DbJsonB(mutationDetection = HASH)
   List<PlainBean> beanList;
 
   @DbJson(length = 700)
   Map<String, PlainBean> beanMap = new LinkedHashMap<>();
 
-  @DbJson(length = 500)
+  @DbJson(length = 500, mutationDetection = SOURCE) // such that we can rebuild old values
   PlainBean plainBean;
 
   @DbJson(length = 50)

ebean-core/src/test/java/org/tests/model/json/EBasicPlain.java

@@ -0,0 +1,67 @@
+package org.tests.model.json;
+
+import io.ebean.annotation.DbJson;
+
+import javax.persistence.Entity;
+import javax.persistence.Id;
+import javax.persistence.Version;
+
+import static io.ebean.annotation.MutationDetection.NONE;
+
+@Entity
+public class EBasicPlain {
+
+  @Id
+  long id;
+
+  String attr;
+
+  @DbJson(length = 500)
+  PlainBean plainBean;
+
+  @DbJson(length = 500, mutationDetection = NONE) // only update when property set
+  PlainBean plainBean2;
+
+  @Version
+  long version;
+
+  public long getId() {
+    return id;
+  }
+
+  public void setId(long id) {
+    this.id = id;
+  }
+
+  public String getAttr() {
+    return attr;
+  }
+
+  public void setAttr(String attr) {
+    this.attr = attr;
+  }
+
+  public PlainBean getPlainBean() {
+    return plainBean;
+  }
+
+  public void setPlainBean(PlainBean plainBean) {
+    this.plainBean = plainBean;
+  }
+
+  public PlainBean getPlainBean2() {
+    return plainBean2;
+  }
+
+  public void setPlainBean2(PlainBean plainBean2) {
+    this.plainBean2 = plainBean2;
+  }
+
+  public long getVersion() {
+    return version;
+  }
+
+  public void setVersion(long version) {
+    this.version = version;
+  }
+}

ebean-core/src/test/java/org/tests/model/json/TestJacksonPlainBean.java

@@ -0,0 +1,95 @@
+package org.tests.model.json;
+
+import io.ebean.DB;
+import io.ebeantest.LoggedSql;
+import org.junit.Test;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+public class TestJacksonPlainBean {
+
+  @Test
+  public void insertNullStayNull() {
+
+    // insert with jackson beans as null
+    EBasicPlain bean = new EBasicPlain();
+    bean.setAttr("n0");
+    DB.save(bean);
+
+    LoggedSql.start();
+    bean.setAttr("n1");
+    DB.save(bean);
+    expectedSql(0, "update ebasic_plain set attr=?, version=? where id=? and version=?");
+
+    bean.setPlainBean(new PlainBean("x", 1));
+    DB.save(bean);
+    expectedSql(0, "update ebasic_plain set plain_bean=?, version=? where id=? and version=?");
+
+    final EBasicPlain found = DB.find(EBasicPlain.class, bean.getId());
+    found.setAttr("n2");
+    DB.save(found);
+    expectedSql(1, "update ebasic_plain set attr=?, version=? where id=? and version=?");
+
+    LoggedSql.stop();
+  }
+
+  @Test
+  public void insertUpdate() {
+
+    DB.getDefault();
+    LoggedSql.start();
+
+    PlainBean content = new PlainBean("foo", 42);
+    EBasicPlain bean = new EBasicPlain();
+    bean.setAttr("attr0");
+    bean.setPlainBean(content);
+    bean.setPlainBean2(new PlainBean("bar", 27));
+
+    DB.save(bean);
+    expectedSql(0, "insert into ebasic_plain (attr, plain_bean, plain_bean2, version) values (?,?,?,?)");
+
+    // inserted plainBean has not been mutated
+    bean.setAttr("attr1");
+    DB.save(bean);
+    expectedSql(0, "update ebasic_plain set attr=?, version=? where id=? and version=?");
+
+    // inserted plainBean has now been mutated
+    content.setName("notFoo");
+    bean.setAttr("attr2");
+    DB.save(bean);
+    expectedSql(0, "update ebasic_plain set attr=?, plain_bean=?, version=? where id=? and version=?");
+
+
+    final EBasicPlain found = DB.find(EBasicPlain.class, bean.getId());
+
+    // update mutating PlainBean only
+    final PlainBean plainBean = found.getPlainBean();
+    plainBean.setName("mod1");
+    DB.save(found);
+    expectedSql(1, "update ebasic_plain set plain_bean=?, version=? where id=? and version=?");
+
+    // dirtyDetection = false, so not included in update
+    found.getPlainBean2().setName("Modification Ignored");
+    // dirtyDetection = true, mutation detected
+    plainBean.setName("mod2");
+    DB.save(found);
+    expectedSql(0, "update ebasic_plain set plain_bean=?, version=? where id=? and version=?");
+
+    // update bean, not mutating PlainBean
+    found.setAttr("attr3");
+    DB.save(found);
+    expectedSql(0, "update ebasic_plain set attr=?, version=? where id=? and version=?");
+
+    // dirtyDetection = false, set a new plainBean2 instance, included in update
+    found.setPlainBean2(new PlainBean("bar", 27));
+    DB.save(found);
+    expectedSql(0, "update ebasic_plain set plain_bean2=?, version=? where id=? and version=?");
+
+    LoggedSql.stop();
+  }
+
+  private void expectedSql(int i, String s) {
+    assertThat(LoggedSql.collect().get(i)).contains(s);
+  }
+
+}