diff --git a/ebean-api/src/main/java/io/ebean/config/DatabaseConfig.java b/ebean-api/src/main/java/io/ebean/config/DatabaseConfig.java index 2f174baaf..3e0085a12 100644 --- a/ebean-api/src/main/java/io/ebean/config/DatabaseConfig.java +++ b/ebean-api/src/main/java/io/ebean/config/DatabaseConfig.java @@ -378,6 +378,8 @@ public class DatabaseConfig { */ private Clock clock = Clock.systemUTC(); + private TempFileProvider tempFileProvider = new WeakRefTempFileProvider(); + private List idGenerators = new ArrayList<>(); private List findControllers = new ArrayList<>(); private List persistControllers = new ArrayList<>(); @@ -545,6 +547,14 @@ public class DatabaseConfig { this.clock = clock; } + public TempFileProvider getTempFileProvider() { + return tempFileProvider; + } + + public void setTempFileProvider(final TempFileProvider tempFileProvider) { + this.tempFileProvider = tempFileProvider; + } + /** * Return the slow query time in millis. */ diff --git a/ebean-api/src/main/java/io/ebean/config/DeleteOnShutdownTempFileProvider.java b/ebean-api/src/main/java/io/ebean/config/DeleteOnShutdownTempFileProvider.java new file mode 100644 index 000000000..ad257c98e --- /dev/null +++ b/ebean-api/src/main/java/io/ebean/config/DeleteOnShutdownTempFileProvider.java @@ -0,0 +1,68 @@ +package io.ebean.config; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * TempFileProvider implementation, which deletes all temp files on shutdown. + * + * @author Roland Praml, FOCONIS AG + * + */ +public class DeleteOnShutdownTempFileProvider implements TempFileProvider { + + private static final Logger logger = LoggerFactory.getLogger(DeleteOnShutdownTempFileProvider.class); + + List tempFiles = new ArrayList<>(); + private final String prefix; + private final String suffix; + private final File directory; + + /** + * Creates the TempFileProvider with default prefix "db-". + */ + public DeleteOnShutdownTempFileProvider() { + this("db-", null, null); + } + + /** + * Creates the TempFileProvider. + */ + public DeleteOnShutdownTempFileProvider(String prefix, String suffix, File directory) { + this.prefix = prefix; + this.suffix = suffix; + this.directory = directory; + } + + @Override + public File createTempFile() throws IOException { + File file = File.createTempFile(prefix, suffix, directory); + synchronized (tempFiles) { + tempFiles.add(file.getAbsolutePath()); + } + return file; + } + + /** + * Deletes all created files on shutdown. + */ + @Override + public void shutdown() { + synchronized (tempFiles) { + for (String path : tempFiles) { + if (new File(path).delete()) { + logger.trace("deleted {}", path); + } else { + logger.warn("could not delete {}", path); + } + } + tempFiles.clear(); + } + } + +} diff --git a/ebean-api/src/main/java/io/ebean/config/TempFileProvider.java b/ebean-api/src/main/java/io/ebean/config/TempFileProvider.java new file mode 100644 index 000000000..4658b46c2 --- /dev/null +++ b/ebean-api/src/main/java/io/ebean/config/TempFileProvider.java @@ -0,0 +1,23 @@ +package io.ebean.config; + +import java.io.File; +import java.io.IOException; + +/** + * Creates a temp file for the ScalarTypeFile datatype. + * + * @author Roland Praml, FOCONIS AG + * + */ +public interface TempFileProvider { + + /** + * Creates a tempFile. + */ + File createTempFile() throws IOException; + + /** + * Shutdown the tempFileProvider. + */ + void shutdown(); +} diff --git a/ebean-api/src/main/java/io/ebean/config/WeakRefTempFileProvider.java b/ebean-api/src/main/java/io/ebean/config/WeakRefTempFileProvider.java new file mode 100644 index 000000000..1cd09dd29 --- /dev/null +++ b/ebean-api/src/main/java/io/ebean/config/WeakRefTempFileProvider.java @@ -0,0 +1,141 @@ +package io.ebean.config; + +import java.io.File; +import java.io.IOException; +import java.lang.ref.ReferenceQueue; +import java.lang.ref.WeakReference; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * WeakRefTempFileProvider will delete the tempFile if all references to the returned File + * object are collected by the garbage collection. + * + * @author Roland Praml, FOCONIS AG + * + */ +public class WeakRefTempFileProvider implements TempFileProvider { + + private static final Logger logger = LoggerFactory.getLogger(WeakRefTempFileProvider.class); + + private final ReferenceQueue tempFiles = new ReferenceQueue<>(); + + private WeakFileReference root; + + private final String prefix; + private final String suffix; + private final File directory; + + /** + * We hold a linkedList of weak references. So we can remove stale files in O(1) + * + * @author Roland Praml, FOCONIS AG + */ + private static class WeakFileReference extends WeakReference { + + String path; + WeakFileReference prev; + WeakFileReference next; + + WeakFileReference(File referent, ReferenceQueue q) { + super(referent, q); + path = referent.getAbsolutePath(); + } + + boolean delete(boolean shutdown) { + if (new File(path).delete()) { + logger.trace("deleted {}", path); + return true; + } else { + if (shutdown) { + logger.warn("could not delete {}", path); + } else { + logger.info("could not delete {} - will delete on shutdown", path); + } + return false; + } + } + } + + + /** + * Creates the TempFileProvider with default prefix "db-". + */ + public WeakRefTempFileProvider() { + this("db-", null, null); + } + + /** + * Creates the TempFileProvider. + */ + public WeakRefTempFileProvider(String prefix, String suffix, File directory) { + this.prefix = prefix; + this.suffix = suffix; + this.directory = directory; + } + + @Override + public File createTempFile() throws IOException { + File tempFile = File.createTempFile(prefix, suffix, directory); + logger.trace("createTempFile: {}", tempFile); + synchronized (this) { + add(new WeakFileReference(tempFile, tempFiles)); + } + return tempFile; + } + + /** + * Will delete stale files. + * This is public to use in tests. + */ + public void deleteStaleTempFiles() { + synchronized (this) { + deleteStaleTempFilesInternal(); + } + } + + private void deleteStaleTempFilesInternal() { + WeakFileReference ref; + while ((ref = (WeakFileReference) tempFiles.poll()) != null) { + if (ref.delete(false)) { + remove(ref); // remove from linkedList only, if delete was successful. + } + } + } + + private void add(WeakFileReference ref) { + deleteStaleTempFilesInternal(); + + if (root == null) { + root = ref; + } else { + ref.next = root; + root.prev = ref; + root = ref; + } + } + + private void remove(WeakFileReference ref) { + if (ref.next != null) { + ref.next.prev = ref.prev; + } + if (ref.prev != null) { + ref.prev.next = ref.next; + } else { + root = ref.next; + } + } + + /** + * Deletes all created files on shutdown. + */ + @Override + public void shutdown() { + while (root != null) { + root.delete(true); + root = root.next; + } + } + +} diff --git a/ebean-api/src/test/java/io/ebean/TestWeakRefTempFileProvider.java b/ebean-api/src/test/java/io/ebean/TestWeakRefTempFileProvider.java new file mode 100644 index 000000000..dc69cc687 --- /dev/null +++ b/ebean-api/src/test/java/io/ebean/TestWeakRefTempFileProvider.java @@ -0,0 +1,171 @@ +package io.ebean; + + +import io.ebean.config.WeakRefTempFileProvider; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; + +import java.io.File; +import java.io.FileOutputStream; +import java.nio.channels.FileLock; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * Test for the WeakRefTempFileProvider. (Note: this test relies on an aggressive garbage collection. + * if GC implementation will change, the test may fail) + * + * @author Roland Praml, FOCONIS AG + */ +public class TestWeakRefTempFileProvider { + + WeakRefTempFileProvider prov = new WeakRefTempFileProvider(); + + @AfterEach + public void shutdown() { + prov.shutdown(); + } + + /** + * Run the garbage collection and delete stale files. + */ + private void gc() throws InterruptedException { + System.gc(); + Thread.sleep(100); + prov.deleteStaleTempFiles(); + } + + @Test + public void testStaleEntries() throws Exception { + File tempFile = prov.createTempFile(); + String fileName = tempFile.getAbsolutePath(); + + gc(); + + assertThat(new File(fileName)).exists(); + + tempFile = null; // give up reference + + gc(); + + assertThat(new File(fileName)).doesNotExist(); + + + } + + @Test + public void testLinkedListForward() throws Exception { + File tempFile1 = prov.createTempFile(); + String fileName1 = tempFile1.getAbsolutePath(); + File tempFile2 = prov.createTempFile(); + String fileName2 = tempFile2.getAbsolutePath(); + File tempFile3 = prov.createTempFile(); + String fileName3 = tempFile3.getAbsolutePath(); + + assertThat(new File(fileName1)).exists(); + assertThat(new File(fileName2)).exists(); + assertThat(new File(fileName3)).exists(); + + gc(); + + // give up first ref + tempFile1 = null; + + gc(); + + assertThat(new File(fileName1)).doesNotExist(); + assertThat(new File(fileName2)).exists(); + assertThat(new File(fileName3)).exists(); + + // give up second ref + tempFile2 = null; + + gc(); + + assertThat(new File(fileName1)).doesNotExist(); + assertThat(new File(fileName2)).doesNotExist(); + assertThat(new File(fileName3)).exists(); + + // give up third ref + tempFile3 = null; + + gc(); + + assertThat(new File(fileName1)).doesNotExist(); + assertThat(new File(fileName2)).doesNotExist(); + assertThat(new File(fileName3)).doesNotExist(); + + } + + + @Test + public void testLinkedListReverse() throws Exception { + File tempFile1 = prov.createTempFile(); + String fileName1 = tempFile1.getAbsolutePath(); + File tempFile2 = prov.createTempFile(); + String fileName2 = tempFile2.getAbsolutePath(); + File tempFile3 = prov.createTempFile(); + String fileName3 = tempFile3.getAbsolutePath(); + + assertThat(new File(fileName1)).exists(); + assertThat(new File(fileName2)).exists(); + assertThat(new File(fileName3)).exists(); + + gc(); + + // give up third ref + tempFile3 = null; + + gc(); + + assertThat(new File(fileName1)).exists(); + assertThat(new File(fileName2)).exists(); + assertThat(new File(fileName3)).doesNotExist(); + + // give up second ref + tempFile2 = null; + + gc(); + + assertThat(new File(fileName1)).exists(); + assertThat(new File(fileName2)).doesNotExist(); + assertThat(new File(fileName3)).doesNotExist(); + + // give up first ref + tempFile1 = null; + + gc(); + + assertThat(new File(fileName1)).doesNotExist(); + assertThat(new File(fileName2)).doesNotExist(); + assertThat(new File(fileName3)).doesNotExist(); + + } + + @Test + @Disabled("Runs on Windows only") + public void testFileLocked() throws Exception { + File tempFile = prov.createTempFile(); + String fileName = tempFile.getAbsolutePath(); + + try (FileOutputStream os = new FileOutputStream(fileName)) { + FileLock lock = os.getChannel().lock(); + try { + os.write(42); + + tempFile = null; + gc(); + } finally { + lock.release(); + } + + } + + assertThat(new File(fileName)).exists(); + + prov.shutdown(); + + assertThat(new File(fileName)).doesNotExist(); + } +} diff --git a/ebean-core/src/main/java/io/ebeaninternal/server/core/DefaultServer.java b/ebean-core/src/main/java/io/ebeaninternal/server/core/DefaultServer.java index e677f8e51..a927d7c88 100644 --- a/ebean-core/src/main/java/io/ebeaninternal/server/core/DefaultServer.java +++ b/ebean-core/src/main/java/io/ebeaninternal/server/core/DefaultServer.java @@ -54,6 +54,7 @@ import io.ebean.config.QueryPlanCapture; import io.ebean.config.QueryPlanListener; import io.ebean.config.SlowQueryEvent; import io.ebean.config.SlowQueryListener; +import io.ebean.config.TempFileProvider; import io.ebean.config.TenantMode; import io.ebean.config.dbplatform.DatabasePlatform; import io.ebean.event.BeanPersistController; @@ -143,6 +144,7 @@ public final class DefaultServer implements SpiServer, SpiEbeanServer { private final String serverName; private final DatabasePlatform databasePlatform; private final TransactionManager transactionManager; + private final TempFileProvider tempFileProvider; private final QueryPlanManager queryPlanManager; private final ExtraMetrics extraMetrics; private final DataTimeZone dataTimeZone; @@ -223,6 +225,7 @@ public final class DefaultServer implements SpiServer, SpiEbeanServer { this.queryPlanManager = config.initQueryPlanManager(transactionManager); this.metaInfoManager = new DefaultMetaInfoManager(this); this.serverPlugins = config.getPlugins(); + this.tempFileProvider = config.getConfig().getTempFileProvider(); this.ddlGenerator = config.initDdlGenerator(this); this.scriptRunner = new DScriptRunner(this); this.initDatabase = !config.getConfig().skipInitDatabase(); @@ -460,6 +463,8 @@ public final class DefaultServer implements SpiServer, SpiEbeanServer { backgroundExecutor.shutdown(); // shutdown DataSource (if its an Ebean one) transactionManager.shutdown(shutdownDataSource, deregisterDriver); + + tempFileProvider.shutdown(); dumpMetrics(); shutdown = true; if (shutdownDataSource) { diff --git a/ebean-core/src/main/java/io/ebeaninternal/server/core/InternalConfiguration.java b/ebean-core/src/main/java/io/ebeaninternal/server/core/InternalConfiguration.java index 151a7031f..b1e79b803 100644 --- a/ebean-core/src/main/java/io/ebeaninternal/server/core/InternalConfiguration.java +++ b/ebean-core/src/main/java/io/ebeaninternal/server/core/InternalConfiguration.java @@ -8,6 +8,7 @@ import io.ebean.config.DatabaseConfig; import io.ebean.config.ExternalTransactionManager; import io.ebean.config.ProfilingConfig; import io.ebean.config.SlowQueryListener; +import io.ebean.config.TempFileProvider; import io.ebean.config.dbplatform.DatabasePlatform; import io.ebean.config.dbplatform.DbHistorySupport; import io.ebean.event.changelog.ChangeLogListener; @@ -75,6 +76,7 @@ public final class InternalConfiguration { private final DatabasePlatform databasePlatform; private final DeployInherit deployInherit; private final TypeManager typeManager; + private final TempFileProvider tempFileProvider; private final DtoBeanManager dtoBeanManager; private final ClockService clockService; private final DataTimeZone dataTimeZone; @@ -115,6 +117,7 @@ public final class InternalConfiguration { this.databasePlatform = config.getDatabasePlatform(); this.expressionFactory = initExpressionFactory(config); this.typeManager = new DefaultTypeManager(config, bootupClasses); + this.tempFileProvider = config.getTempFileProvider(); this.multiValueBind = createMultiValueBind(databasePlatform.getPlatform()); this.deployInherit = new DeployInherit(bootupClasses); this.deployCreateProperties = new DeployCreateProperties(typeManager); @@ -511,6 +514,10 @@ public final class InternalConfiguration { return logManager; } + public TempFileProvider getTempFileProvider() { + return tempFileProvider; + } + private ServerCachePlugin initServerCachePlugin() { if (config.isLocalOnlyL2Cache()) { localL2Caching = true; diff --git a/ebean-core/src/main/java/io/ebeaninternal/server/type/DefaultTypeManager.java b/ebean-core/src/main/java/io/ebeaninternal/server/type/DefaultTypeManager.java index 5bc51bbb3..969e0ef64 100644 --- a/ebean-core/src/main/java/io/ebeaninternal/server/type/DefaultTypeManager.java +++ b/ebean-core/src/main/java/io/ebeaninternal/server/type/DefaultTypeManager.java @@ -61,7 +61,7 @@ public final class DefaultTypeManager implements TypeManager { private final DefaultTypeFactory extraTypeFactory; private final ScalarType hstoreType = new ScalarTypePostgresHstore(); - private final ScalarTypeFile fileType = new ScalarTypeFile(); + private final ScalarTypeFile fileType; private final ScalarType charType = new ScalarTypeChar(); private final ScalarType charArrayType = new ScalarTypeCharArray(); private final ScalarType longVarcharType = new ScalarTypeLongVarchar(); @@ -141,6 +141,7 @@ public final class DefaultTypeManager implements TypeManager { this.arrayTypeSetFactory = arrayTypeSetFactory(config.getDatabasePlatform()); this.offlineMigrationGeneration = DbOffline.isGenerateMigration(); this.defaultEnumType = config.getDefaultEnumType(); + this.fileType = new ScalarTypeFile(config.getTempFileProvider()); initialiseStandard(config); initialiseJavaTimeTypes(config); diff --git a/ebean-core/src/main/java/io/ebeaninternal/server/type/ScalarTypeFile.java b/ebean-core/src/main/java/io/ebeaninternal/server/type/ScalarTypeFile.java index d5f500d78..6430e2d1c 100644 --- a/ebean-core/src/main/java/io/ebeaninternal/server/type/ScalarTypeFile.java +++ b/ebean-core/src/main/java/io/ebeaninternal/server/type/ScalarTypeFile.java @@ -2,6 +2,7 @@ package io.ebeaninternal.server.type; import com.fasterxml.jackson.core.JsonGenerator; import com.fasterxml.jackson.core.JsonParser; +import io.ebean.config.TempFileProvider; import io.ebean.core.type.DataBinder; import io.ebean.core.type.DataReader; import io.ebean.core.type.DocPropertyType; @@ -17,26 +18,22 @@ import java.sql.Types; */ final class ScalarTypeFile extends ScalarTypeBase { - private final String prefix; - private final String suffix; - private final File directory; + private final TempFileProvider tempFileProvider; private final int bufferSize; /** - * Construct with reasonable defaults of Blob and 8096 buffer size. + * Construct with reasonable defaults of Blob and 8192 buffer size. */ - ScalarTypeFile() { - this(Types.LONGVARBINARY, "db-", null, null, 8096); + ScalarTypeFile(TempFileProvider tempFileProvider) { + this(Types.LONGVARBINARY, tempFileProvider, 8192); } /** * Create the ScalarTypeFile. */ - ScalarTypeFile(int jdbcType, String prefix, String suffix, File directory, int bufferSize) { + ScalarTypeFile(int jdbcType, TempFileProvider tempFileProvider, int bufferSize) { super(File.class, false, jdbcType); - this.prefix = prefix; - this.suffix = suffix; - this.directory = directory; + this.tempFileProvider = tempFileProvider; this.bufferSize = bufferSize; } @@ -63,7 +60,7 @@ final class ScalarTypeFile extends ScalarTypeBase { } try { // stream from db into our temp file - File tempFile = File.createTempFile(prefix, suffix, directory); + File tempFile = tempFileProvider.createTempFile(); OutputStream os = getOutputStream(tempFile); pump(is, os); return tempFile; @@ -106,7 +103,7 @@ final class ScalarTypeFile extends ScalarTypeBase { @Override public File jsonRead(JsonParser parser) throws IOException { - File tempFile = File.createTempFile(prefix, suffix, directory); + File tempFile = tempFileProvider.createTempFile(); try (OutputStream os = getOutputStream(tempFile)) { parser.readBinaryValue(os); os.flush();