NEW: TempFileProvider will clean up files automatically if the reference is gc'ed

ebean-api/src/main/java/io/ebean/config/DatabaseConfig.java

@@ -378,6 +378,8 @@ public class DatabaseConfig {
    */
   private Clock clock = Clock.systemUTC();
 
+  private TempFileProvider tempFileProvider = new WeakRefTempFileProvider();
+
   private List<IdGenerator> idGenerators = new ArrayList<>();
   private List<BeanFindController> findControllers = new ArrayList<>();
   private List<BeanPersistController> persistControllers = new ArrayList<>();
@@ -545,6 +547,14 @@ public class DatabaseConfig {
     this.clock = clock;
   }
 
+  public TempFileProvider getTempFileProvider() {
+    return tempFileProvider;
+  }
+
+  public void setTempFileProvider(final TempFileProvider tempFileProvider) {
+    this.tempFileProvider = tempFileProvider;
+  }
+
   /**
    * Return the slow query time in millis.
    */

ebean-api/src/main/java/io/ebean/config/DeleteOnShutdownTempFileProvider.java

@@ -0,0 +1,68 @@
+package io.ebean.config;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * TempFileProvider implementation, which deletes all temp files on shutdown.
+ *
+ * @author Roland Praml, FOCONIS AG
+ *
+ */
+public class DeleteOnShutdownTempFileProvider implements TempFileProvider {
+
+  private static final Logger logger = LoggerFactory.getLogger(DeleteOnShutdownTempFileProvider.class);
+
+  List<String> tempFiles = new ArrayList<>();
+  private final String prefix;
+  private final String suffix;
+  private final File directory;
+
+  /**
+   * Creates the TempFileProvider with default prefix "db-".
+   */
+  public DeleteOnShutdownTempFileProvider() {
+    this("db-", null, null);
+  }
+
+  /**
+   * Creates the TempFileProvider.
+   */
+  public DeleteOnShutdownTempFileProvider(String prefix, String suffix, File directory) {
+    this.prefix = prefix;
+    this.suffix = suffix;
+    this.directory = directory;
+  }
+
+  @Override
+  public File createTempFile() throws IOException {
+    File file = File.createTempFile(prefix, suffix, directory);
+    synchronized (tempFiles) {
+      tempFiles.add(file.getAbsolutePath());
+    }
+    return file;
+  }
+
+  /**
+   * Deletes all created files on shutdown.
+   */
+  @Override
+  public void shutdown() {
+    synchronized (tempFiles) {
+      for (String path : tempFiles) {
+        if (new File(path).delete()) {
+          logger.trace("deleted {}", path);
+        } else {
+          logger.warn("could not delete {}", path);
+        }
+      }
+      tempFiles.clear();
+    }
+  }
+
+}

ebean-api/src/main/java/io/ebean/config/TempFileProvider.java

@@ -0,0 +1,23 @@
+package io.ebean.config;
+
+import java.io.File;
+import java.io.IOException;
+
+/**
+ * Creates a temp file for the ScalarTypeFile datatype.
+ *
+ * @author Roland Praml, FOCONIS AG
+ *
+ */
+public interface TempFileProvider {
+
+  /**
+   * Creates a tempFile.
+   */
+  File createTempFile() throws IOException;
+
+  /**
+   * Shutdown the tempFileProvider.
+   */
+  void shutdown();
+}

ebean-api/src/main/java/io/ebean/config/WeakRefTempFileProvider.java

@@ -0,0 +1,141 @@
+package io.ebean.config;
+
+import java.io.File;
+import java.io.IOException;
+import java.lang.ref.ReferenceQueue;
+import java.lang.ref.WeakReference;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * WeakRefTempFileProvider will delete the tempFile if all references to the returned File
+ * object are collected by the garbage collection.
+ *
+ * @author Roland Praml, FOCONIS AG
+ *
+ */
+public class WeakRefTempFileProvider implements TempFileProvider {
+
+  private static final Logger logger = LoggerFactory.getLogger(WeakRefTempFileProvider.class);
+
+  private final ReferenceQueue<File> tempFiles = new ReferenceQueue<>();
+
+  private WeakFileReference root;
+
+  private final String prefix;
+  private final String suffix;
+  private final File directory;
+
+  /**
+   * We hold a linkedList of weak references. So we can remove stale files in O(1)
+   *
+   * @author Roland Praml, FOCONIS AG
+   */
+  private static class WeakFileReference extends WeakReference<File> {
+
+    String path;
+    WeakFileReference prev;
+    WeakFileReference next;
+
+    WeakFileReference(File referent, ReferenceQueue<? super File> q) {
+      super(referent, q);
+      path = referent.getAbsolutePath();
+    }
+
+    boolean delete(boolean shutdown) {
+      if (new File(path).delete()) {
+        logger.trace("deleted {}", path);
+        return true;
+      } else {
+        if (shutdown) {
+          logger.warn("could not delete {}", path);
+        } else {
+          logger.info("could not delete {} - will delete on shutdown", path);
+        }
+        return false;
+      }
+    }
+  }
+
+
+  /**
+   * Creates the TempFileProvider with default prefix "db-".
+   */
+  public WeakRefTempFileProvider() {
+    this("db-", null, null);
+  }
+
+  /**
+   * Creates the TempFileProvider.
+   */
+  public WeakRefTempFileProvider(String prefix, String suffix, File directory) {
+    this.prefix = prefix;
+    this.suffix = suffix;
+    this.directory = directory;
+  }
+
+  @Override
+  public File createTempFile() throws IOException {
+    File tempFile = File.createTempFile(prefix, suffix, directory);
+    logger.trace("createTempFile: {}", tempFile);
+    synchronized (this) {
+      add(new WeakFileReference(tempFile, tempFiles));
+    }
+    return tempFile;
+  }
+
+  /**
+   * Will delete stale files.
+   * This is public to use in tests.
+   */
+  public void deleteStaleTempFiles() {
+    synchronized (this) {
+      deleteStaleTempFilesInternal();
+    }
+  }
+
+  private void deleteStaleTempFilesInternal() {
+    WeakFileReference ref;
+    while ((ref = (WeakFileReference) tempFiles.poll()) != null) {
+      if (ref.delete(false)) {
+        remove(ref); // remove from linkedList only, if delete was successful.
+      }
+    }
+  }
+
+  private void add(WeakFileReference ref) {
+    deleteStaleTempFilesInternal();
+
+    if (root == null) {
+      root = ref;
+    } else {
+      ref.next = root;
+      root.prev = ref;
+      root = ref;
+    }
+  }
+
+  private void remove(WeakFileReference ref) {
+    if (ref.next != null) {
+      ref.next.prev = ref.prev;
+    }
+    if (ref.prev != null) {
+      ref.prev.next = ref.next;
+    } else {
+      root = ref.next;
+    }
+  }
+
+  /**
+   * Deletes all created files on shutdown.
+   */
+  @Override
+  public void shutdown() {
+    while (root != null) {
+      root.delete(true);
+      root = root.next;
+    }
+  }
+
+}

ebean-api/src/test/java/io/ebean/TestWeakRefTempFileProvider.java

@@ -0,0 +1,171 @@
+package io.ebean;
+
+
+import io.ebean.config.WeakRefTempFileProvider;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.Disabled;
+import org.junit.jupiter.api.Test;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.nio.channels.FileLock;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+/**
+ * Test for the WeakRefTempFileProvider. (Note: this test relies on an aggressive garbage collection.
+ * if GC implementation will change, the test may fail)
+ *
+ * @author Roland Praml, FOCONIS AG
+ */
+public class TestWeakRefTempFileProvider {
+
+  WeakRefTempFileProvider prov = new WeakRefTempFileProvider();
+
+  @AfterEach
+  public void shutdown() {
+    prov.shutdown();
+  }
+
+  /**
+   * Run the garbage collection and delete stale files.
+   */
+  private void gc() throws InterruptedException {
+    System.gc();
+    Thread.sleep(100);
+    prov.deleteStaleTempFiles();
+  }
+
+  @Test
+  public void testStaleEntries() throws Exception {
+    File tempFile = prov.createTempFile();
+    String fileName = tempFile.getAbsolutePath();
+
+    gc();
+
+    assertThat(new File(fileName)).exists();
+
+    tempFile = null; // give up reference
+
+    gc();
+
+    assertThat(new File(fileName)).doesNotExist();
+
+
+  }
+
+  @Test
+  public void testLinkedListForward() throws Exception {
+    File tempFile1 = prov.createTempFile();
+    String fileName1 = tempFile1.getAbsolutePath();
+    File tempFile2 = prov.createTempFile();
+    String fileName2 = tempFile2.getAbsolutePath();
+    File tempFile3 = prov.createTempFile();
+    String fileName3 = tempFile3.getAbsolutePath();
+
+    assertThat(new File(fileName1)).exists();
+    assertThat(new File(fileName2)).exists();
+    assertThat(new File(fileName3)).exists();
+
+    gc();
+
+    // give up first ref
+    tempFile1 = null;
+
+    gc();
+
+    assertThat(new File(fileName1)).doesNotExist();
+    assertThat(new File(fileName2)).exists();
+    assertThat(new File(fileName3)).exists();
+
+    // give up second ref
+    tempFile2 = null;
+
+    gc();
+
+    assertThat(new File(fileName1)).doesNotExist();
+    assertThat(new File(fileName2)).doesNotExist();
+    assertThat(new File(fileName3)).exists();
+
+    // give up third ref
+    tempFile3 = null;
+
+    gc();
+
+    assertThat(new File(fileName1)).doesNotExist();
+    assertThat(new File(fileName2)).doesNotExist();
+    assertThat(new File(fileName3)).doesNotExist();
+
+  }
+
+
+  @Test
+  public void testLinkedListReverse() throws Exception {
+    File tempFile1 = prov.createTempFile();
+    String fileName1 = tempFile1.getAbsolutePath();
+    File tempFile2 = prov.createTempFile();
+    String fileName2 = tempFile2.getAbsolutePath();
+    File tempFile3 = prov.createTempFile();
+    String fileName3 = tempFile3.getAbsolutePath();
+
+    assertThat(new File(fileName1)).exists();
+    assertThat(new File(fileName2)).exists();
+    assertThat(new File(fileName3)).exists();
+
+    gc();
+
+    // give up third ref
+    tempFile3 = null;
+
+    gc();
+
+    assertThat(new File(fileName1)).exists();
+    assertThat(new File(fileName2)).exists();
+    assertThat(new File(fileName3)).doesNotExist();
+
+    // give up second ref
+    tempFile2 = null;
+
+    gc();
+
+    assertThat(new File(fileName1)).exists();
+    assertThat(new File(fileName2)).doesNotExist();
+    assertThat(new File(fileName3)).doesNotExist();
+
+    // give up first ref
+    tempFile1 = null;
+
+    gc();
+
+    assertThat(new File(fileName1)).doesNotExist();
+    assertThat(new File(fileName2)).doesNotExist();
+    assertThat(new File(fileName3)).doesNotExist();
+
+  }
+
+  @Test
+  @Disabled("Runs on Windows only")
+  public void testFileLocked() throws Exception {
+    File tempFile = prov.createTempFile();
+    String fileName = tempFile.getAbsolutePath();
+
+    try (FileOutputStream os = new FileOutputStream(fileName)) {
+      FileLock lock = os.getChannel().lock();
+      try {
+        os.write(42);
+
+        tempFile = null;
+        gc();
+      } finally {
+        lock.release();
+      }
+
+    }
+
+    assertThat(new File(fileName)).exists();
+
+    prov.shutdown();
+
+    assertThat(new File(fileName)).doesNotExist();
+  }
+}

ebean-core/src/main/java/io/ebeaninternal/server/core/DefaultServer.java

@@ -54,6 +54,7 @@ import io.ebean.config.QueryPlanCapture;
 import io.ebean.config.QueryPlanListener;
 import io.ebean.config.SlowQueryEvent;
 import io.ebean.config.SlowQueryListener;
+import io.ebean.config.TempFileProvider;
 import io.ebean.config.TenantMode;
 import io.ebean.config.dbplatform.DatabasePlatform;
 import io.ebean.event.BeanPersistController;
@@ -143,6 +144,7 @@ public final class DefaultServer implements SpiServer, SpiEbeanServer {
   private final String serverName;
   private final DatabasePlatform databasePlatform;
   private final TransactionManager transactionManager;
+  private final TempFileProvider tempFileProvider;
   private final QueryPlanManager queryPlanManager;
   private final ExtraMetrics extraMetrics;
   private final DataTimeZone dataTimeZone;
@@ -223,6 +225,7 @@ public final class DefaultServer implements SpiServer, SpiEbeanServer {
     this.queryPlanManager = config.initQueryPlanManager(transactionManager);
     this.metaInfoManager = new DefaultMetaInfoManager(this);
     this.serverPlugins = config.getPlugins();
+    this.tempFileProvider = config.getConfig().getTempFileProvider();
     this.ddlGenerator = config.initDdlGenerator(this);
     this.scriptRunner = new DScriptRunner(this);
     this.initDatabase = !config.getConfig().skipInitDatabase();
@@ -460,6 +463,8 @@ public final class DefaultServer implements SpiServer, SpiEbeanServer {
     backgroundExecutor.shutdown();
     // shutdown DataSource (if its an Ebean one)
     transactionManager.shutdown(shutdownDataSource, deregisterDriver);
+
+    tempFileProvider.shutdown();
     dumpMetrics();
     shutdown = true;
     if (shutdownDataSource) {

ebean-core/src/main/java/io/ebeaninternal/server/core/InternalConfiguration.java

@@ -8,6 +8,7 @@ import io.ebean.config.DatabaseConfig;
 import io.ebean.config.ExternalTransactionManager;
 import io.ebean.config.ProfilingConfig;
 import io.ebean.config.SlowQueryListener;
+import io.ebean.config.TempFileProvider;
 import io.ebean.config.dbplatform.DatabasePlatform;
 import io.ebean.config.dbplatform.DbHistorySupport;
 import io.ebean.event.changelog.ChangeLogListener;
@@ -75,6 +76,7 @@ public final class InternalConfiguration {
   private final DatabasePlatform databasePlatform;
   private final DeployInherit deployInherit;
   private final TypeManager typeManager;
+  private final TempFileProvider tempFileProvider;
   private final DtoBeanManager dtoBeanManager;
   private final ClockService clockService;
   private final DataTimeZone dataTimeZone;
@@ -115,6 +117,7 @@ public final class InternalConfiguration {
     this.databasePlatform = config.getDatabasePlatform();
     this.expressionFactory = initExpressionFactory(config);
     this.typeManager = new DefaultTypeManager(config, bootupClasses);
+    this.tempFileProvider = config.getTempFileProvider();
     this.multiValueBind = createMultiValueBind(databasePlatform.getPlatform());
     this.deployInherit = new DeployInherit(bootupClasses);
     this.deployCreateProperties = new DeployCreateProperties(typeManager);
@@ -511,6 +514,10 @@ public final class InternalConfiguration {
     return logManager;
   }
 
+  public TempFileProvider getTempFileProvider() {
+    return tempFileProvider;
+  }
+
   private ServerCachePlugin initServerCachePlugin() {
     if (config.isLocalOnlyL2Cache()) {
       localL2Caching = true;

ebean-core/src/main/java/io/ebeaninternal/server/type/DefaultTypeManager.java

@@ -61,7 +61,7 @@ public final class DefaultTypeManager implements TypeManager {
   private final DefaultTypeFactory extraTypeFactory;
 
   private final ScalarType<?> hstoreType = new ScalarTypePostgresHstore();
-  private final ScalarTypeFile fileType = new ScalarTypeFile();
+  private final ScalarTypeFile fileType;
   private final ScalarType<?> charType = new ScalarTypeChar();
   private final ScalarType<?> charArrayType = new ScalarTypeCharArray();
   private final ScalarType<?> longVarcharType = new ScalarTypeLongVarchar();
@@ -141,6 +141,7 @@ public final class DefaultTypeManager implements TypeManager {
     this.arrayTypeSetFactory = arrayTypeSetFactory(config.getDatabasePlatform());
     this.offlineMigrationGeneration = DbOffline.isGenerateMigration();
     this.defaultEnumType = config.getDefaultEnumType();
+    this.fileType = new ScalarTypeFile(config.getTempFileProvider());
 
     initialiseStandard(config);
     initialiseJavaTimeTypes(config);

ebean-core/src/main/java/io/ebeaninternal/server/type/ScalarTypeFile.java

@@ -2,6 +2,7 @@ package io.ebeaninternal.server.type;
 
 import com.fasterxml.jackson.core.JsonGenerator;
 import com.fasterxml.jackson.core.JsonParser;
+import io.ebean.config.TempFileProvider;
 import io.ebean.core.type.DataBinder;
 import io.ebean.core.type.DataReader;
 import io.ebean.core.type.DocPropertyType;
@@ -17,26 +18,22 @@ import java.sql.Types;
  */
 final class ScalarTypeFile extends ScalarTypeBase<File> {
 
-  private final String prefix;
-  private final String suffix;
-  private final File directory;
+  private final TempFileProvider tempFileProvider;
   private final int bufferSize;
 
   /**
-   * Construct with reasonable defaults of Blob and 8096 buffer size.
+   * Construct with reasonable defaults of Blob and 8192 buffer size.
    */
-  ScalarTypeFile() {
-    this(Types.LONGVARBINARY, "db-", null, null, 8096);
+  ScalarTypeFile(TempFileProvider tempFileProvider) {
+    this(Types.LONGVARBINARY, tempFileProvider, 8192);
   }
 
   /**
    * Create the ScalarTypeFile.
    */
-  ScalarTypeFile(int jdbcType, String prefix, String suffix, File directory, int bufferSize) {
+  ScalarTypeFile(int jdbcType, TempFileProvider tempFileProvider, int bufferSize) {
     super(File.class, false, jdbcType);
-    this.prefix = prefix;
-    this.suffix = suffix;
-    this.directory = directory;
+    this.tempFileProvider = tempFileProvider;
     this.bufferSize = bufferSize;
   }
 
@@ -63,7 +60,7 @@ final class ScalarTypeFile extends ScalarTypeBase<File> {
     }
     try {
       // stream from db into our temp file
-      File tempFile = File.createTempFile(prefix, suffix, directory);
+      File tempFile = tempFileProvider.createTempFile();
       OutputStream os = getOutputStream(tempFile);
       pump(is, os);
       return tempFile;
@@ -106,7 +103,7 @@ final class ScalarTypeFile extends ScalarTypeBase<File> {
 
   @Override
   public File jsonRead(JsonParser parser) throws IOException {
-    File tempFile = File.createTempFile(prefix, suffix, directory);
+    File tempFile = tempFileProvider.createTempFile();
     try (OutputStream os = getOutputStream(tempFile)) {
       parser.readBinaryValue(os);
       os.flush();