diff --git a/ebean-api/src/main/java/io/ebean/common/BeanList.java b/ebean-api/src/main/java/io/ebean/common/BeanList.java
index 12d0770d6..49b0e520b 100644
--- a/ebean-api/src/main/java/io/ebean/common/BeanList.java
+++ b/ebean-api/src/main/java/io/ebean/common/BeanList.java
@@ -141,14 +141,16 @@ public final class BeanList<E> extends AbstractBeanCollection<E> implements List
     }
   }
 
-  /**
-   * Set the actual underlying list.
-   * <p>
-   * This is primarily for the deferred fetching function.
-   */
-  @SuppressWarnings("unchecked")
-  public void setActualList(List<?> list) {
-    this.list = (List<E>) list;
+  public BeanCollectionAdd collectionAdd() {
+    if (list == null) {
+      list = new ArrayList<>();
+    }
+    return this;
+  }
+
+  public void refresh(ModifyListenMode modifyListenMode, BeanList<E> newList) {
+    setModifyListening(modifyListenMode);
+    this.list = newList.actualList();
   }
 
   /**
diff --git a/ebean-api/src/main/java/io/ebean/common/BeanMap.java b/ebean-api/src/main/java/io/ebean/common/BeanMap.java
index 796d469be..6f2de6e35 100644
--- a/ebean-api/src/main/java/io/ebean/common/BeanMap.java
+++ b/ebean-api/src/main/java/io/ebean/common/BeanMap.java
@@ -1,9 +1,6 @@
 package io.ebean.common;
 
-import io.ebean.bean.BeanCollection;
-import io.ebean.bean.BeanCollectionLoader;
-import io.ebean.bean.EntityBean;
-import io.ebean.bean.ToStringBuilder;
+import io.ebean.bean.*;
 
 import java.util.*;
 
@@ -160,12 +157,17 @@ public final class BeanMap<K, E> extends AbstractBeanCollection<E> implements Ma
     }
   }
 
-  /**
-   * Set the actual underlying map. Used for performing lazy fetch.
-   */
+  public LinkedHashMap<K, E> collectionAdd() {
+    if (map == null) {
+      map = new LinkedHashMap<>();
+    }
+    return map;
+  }
+
   @SuppressWarnings("unchecked")
-  public void setActualMap(LinkedHashMap<?, ?> map) {
-    this.map = (LinkedHashMap<K, E>) map;
+  public void refresh(ModifyListenMode modifyListenMode, BeanMap<?, ?> newMap) {
+    setModifyListening(modifyListenMode);
+    this.map = (LinkedHashMap<K, E>) newMap.actualMap();
   }
 
   /**
diff --git a/ebean-api/src/main/java/io/ebean/common/BeanSet.java b/ebean-api/src/main/java/io/ebean/common/BeanSet.java
index bb4eaf7bc..52f6da7dd 100644
--- a/ebean-api/src/main/java/io/ebean/common/BeanSet.java
+++ b/ebean-api/src/main/java/io/ebean/common/BeanSet.java
@@ -3,10 +3,7 @@ package io.ebean.common;
 import io.ebean.bean.*;
 
 import java.io.Serializable;
-import java.util.Collection;
-import java.util.Iterator;
-import java.util.LinkedHashSet;
-import java.util.Set;
+import java.util.*;
 
 /**
  * Set capable of lazy loading and modification aware.
@@ -144,12 +141,16 @@ public final class BeanSet<E> extends AbstractBeanCollection<E> implements Set<E
     }
   }
 
-  /**
-   * Set the underlying set (used for lazy fetch).
-   */
-  @SuppressWarnings("unchecked")
-  public void setActualSet(LinkedHashSet<?> set) {
-    this.set = (LinkedHashSet<E>) set;
+  public BeanCollectionAdd collectionAdd() {
+    if (set == null) {
+      set = new LinkedHashSet<>();
+    }
+    return this;
+  }
+
+  public void refresh(ModifyListenMode modifyListenMode, BeanSet<E> newSet) {
+    setModifyListening(modifyListenMode);
+    this.set = newSet.actualSet();
   }
 
   /**
diff --git a/ebean-api/src/main/java21/io/ebean/common/BeanMap.java b/ebean-api/src/main/java21/io/ebean/common/BeanMap.java
index e8fdc065f..69ce2797d 100644
--- a/ebean-api/src/main/java21/io/ebean/common/BeanMap.java
+++ b/ebean-api/src/main/java21/io/ebean/common/BeanMap.java
@@ -160,12 +160,17 @@ public final class BeanMap<K, E> extends AbstractBeanCollection<E> implements Se
     }
   }
 
-  /**
-   * Set the actual underlying map. Used for performing lazy fetch.
-   */
+  public LinkedHashMap<K, E> collectionAdd() {
+    if (map == null) {
+      map = new LinkedHashMap<>();
+    }
+    return map;
+  }
+
   @SuppressWarnings("unchecked")
-  public void setActualMap(LinkedHashMap<?, ?> map) {
-    this.map = (LinkedHashMap<K, E>) map;
+  public void refresh(ModifyListenMode modifyListenMode, BeanMap<?, ?> newMap) {
+    setModifyListening(modifyListenMode);
+    this.map = (LinkedHashMap<K, E>) newMap.actualMap();
   }
 
   /**
diff --git a/ebean-api/src/main/java21/io/ebean/common/BeanSet.java b/ebean-api/src/main/java21/io/ebean/common/BeanSet.java
index cf579c4eb..63dcd5755 100644
--- a/ebean-api/src/main/java21/io/ebean/common/BeanSet.java
+++ b/ebean-api/src/main/java21/io/ebean/common/BeanSet.java
@@ -144,12 +144,16 @@ public final class BeanSet<E> extends AbstractBeanCollection<E> implements Seque
     }
   }
 
-  /**
-   * Set the underlying set (used for lazy fetch).
-   */
-  @SuppressWarnings("unchecked")
-  public void setActualSet(LinkedHashSet<?> set) {
-    this.set = (LinkedHashSet<E>) set;
+  public BeanCollectionAdd collectionAdd() {
+    if (set == null) {
+      set = new LinkedHashSet<>();
+    }
+    return this;
+  }
+
+  public void refresh(ModifyListenMode modifyListenMode, BeanSet<E> newSet) {
+    setModifyListening(modifyListenMode);
+    this.set = newSet.actualSet();
   }
 
   /**
diff --git a/ebean-core/src/main/java/io/ebeaninternal/server/deploy/BeanListHelp.java b/ebean-core/src/main/java/io/ebeaninternal/server/deploy/BeanListHelp.java
index ed069c9c2..25d0f98a7 100644
--- a/ebean-core/src/main/java/io/ebeaninternal/server/deploy/BeanListHelp.java
+++ b/ebean-core/src/main/java/io/ebeaninternal/server/deploy/BeanListHelp.java
@@ -7,7 +7,6 @@ import io.ebean.common.BeanList;
 import io.ebeaninternal.api.json.SpiJsonWriter;
 
 import java.io.IOException;
-import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
 
@@ -24,11 +23,7 @@ public class BeanListHelp<T> extends BaseCollectionHelp<T> {
   public final BeanCollectionAdd getBeanCollectionAdd(Object bc, String mapKey) {
     if (bc instanceof BeanList<?>) {
       BeanList<?> bl = (BeanList<?>) bc;
-      if (bl.actualList() == null) {
-        bl.setActualList(new ArrayList<>());
-      }
-      return bl;
-
+      return bl.collectionAdd();
     } else {
       throw new RuntimeException("Unhandled type " + bc);
     }
@@ -60,20 +55,20 @@ public class BeanListHelp<T> extends BaseCollectionHelp<T> {
     return beanList;
   }
 
+  @SuppressWarnings("unchecked")
   @Override
   public final void refresh(BeanCollection<?> bc, EntityBean parentBean) {
-    BeanList<?> newBeanList = (BeanList<?>) bc;
+    BeanList<T> newBeanList = (BeanList<T>) bc;
     List<?> currentList = (List<?>) many.getValue(parentBean);
     newBeanList.setModifyListening(many.modifyListenMode());
     if (currentList == null) {
       // the currentList is null? Not really expecting this...
       many.setValue(parentBean, newBeanList);
 
-    } else if (currentList instanceof BeanList<?>) {
+    } else if (currentList instanceof BeanList) {
       // normally this case, replace just the underlying list
-      BeanList<?> currentBeanList = (BeanList<?>) currentList;
-      currentBeanList.setActualList(newBeanList.actualList());
-      currentBeanList.setModifyListening(many.modifyListenMode());
+      BeanList<T> currentBeanList = (BeanList<T>) currentList;
+      currentBeanList.refresh(many.modifyListenMode(), newBeanList);
 
     } else {
       // replace the entire list with the BeanList
diff --git a/ebean-core/src/main/java/io/ebeaninternal/server/deploy/BeanMapHelp.java b/ebean-core/src/main/java/io/ebeaninternal/server/deploy/BeanMapHelp.java
index 12748af70..abd78e492 100644
--- a/ebean-core/src/main/java/io/ebeaninternal/server/deploy/BeanMapHelp.java
+++ b/ebean-core/src/main/java/io/ebeaninternal/server/deploy/BeanMapHelp.java
@@ -8,7 +8,6 @@ import io.ebeaninternal.api.json.SpiJsonWriter;
 
 import java.io.IOException;
 import java.util.Collections;
-import java.util.LinkedHashMap;
 import java.util.Map;
 import java.util.Map.Entry;
 
@@ -41,13 +40,7 @@ public class BeanMapHelp<T> extends BaseCollectionHelp<T> {
     BeanProperty beanProp = targetDescriptor.beanProperty(mapKey);
     if (bc instanceof BeanMap<?, ?>) {
       BeanMap<Object, Object> bm = (BeanMap<Object, Object>) bc;
-      var actualMap = bm.actualMap();
-      if (actualMap == null) {
-        actualMap = new LinkedHashMap<>();
-        bm.setActualMap(actualMap);
-      }
-      return new Adder(beanProp, actualMap);
-
+      return new Adder(beanProp, bm.collectionAdd());
     } else {
       throw new RuntimeException("Unhandled type " + bc);
     }
@@ -123,8 +116,7 @@ public class BeanMapHelp<T> extends BaseCollectionHelp<T> {
     } else if (current instanceof BeanMap<?, ?>) {
       // normally this case, replace just the underlying list
       BeanMap<?, ?> currentBeanMap = (BeanMap<?, ?>) current;
-      currentBeanMap.setActualMap(newBeanMap.actualMap());
-      currentBeanMap.setModifyListening(many.modifyListenMode());
+      currentBeanMap.refresh(many.modifyListenMode(), newBeanMap);
 
     } else {
       // replace the entire set
diff --git a/ebean-core/src/main/java/io/ebeaninternal/server/deploy/BeanSetHelp.java b/ebean-core/src/main/java/io/ebeaninternal/server/deploy/BeanSetHelp.java
index 35d7797c4..e3801ff5e 100644
--- a/ebean-core/src/main/java/io/ebeaninternal/server/deploy/BeanSetHelp.java
+++ b/ebean-core/src/main/java/io/ebeaninternal/server/deploy/BeanSetHelp.java
@@ -8,7 +8,6 @@ import io.ebeaninternal.api.json.SpiJsonWriter;
 
 import java.io.IOException;
 import java.util.Collections;
-import java.util.LinkedHashSet;
 import java.util.Set;
 
 /**
@@ -27,10 +26,7 @@ public class BeanSetHelp<T> extends BaseCollectionHelp<T> {
   public final BeanCollectionAdd getBeanCollectionAdd(Object bc, String mapKey) {
     if (bc instanceof BeanSet<?>) {
       BeanSet<?> beanSet = (BeanSet<?>) bc;
-      if (beanSet.actualSet() == null) {
-        beanSet.setActualSet(new LinkedHashSet<>());
-      }
-      return beanSet;
+      return beanSet.collectionAdd();
     } else {
       throw new RuntimeException("Unhandled type " + bc);
     }
@@ -62,9 +58,10 @@ public class BeanSetHelp<T> extends BaseCollectionHelp<T> {
     return beanSet;
   }
 
+  @SuppressWarnings("unchecked")
   @Override
   public final void refresh(BeanCollection<?> bc, EntityBean parentBean) {
-    BeanSet<?> newBeanSet = (BeanSet<?>) bc;
+    BeanSet<T> newBeanSet = (BeanSet<T>) bc;
     Set<?> current = (Set<?>) many.getValue(parentBean);
     newBeanSet.setModifyListening(many.modifyListenMode());
     if (current == null) {
@@ -73,9 +70,8 @@ public class BeanSetHelp<T> extends BaseCollectionHelp<T> {
 
     } else if (current instanceof BeanSet<?>) {
       // normally this case, replace just the underlying list
-      BeanSet<?> currentBeanSet = (BeanSet<?>) current;
-      currentBeanSet.setActualSet(newBeanSet.actualSet());
-      currentBeanSet.setModifyListening(many.modifyListenMode());
+      BeanSet<T> currentBeanSet = (BeanSet<T>) current;
+      currentBeanSet.refresh(many.modifyListenMode(), newBeanSet);
 
     } else {
       // replace the entire set