From ad3292b92d44b700d32c9ea6d961e8bd6cccb133 Mon Sep 17 00:00:00 2001 From: Robin Bygrave Date: Sat, 6 Feb 2016 00:17:08 +1300 Subject: [PATCH 1/3] WIP - DB Migration runner --- pom.xml | 6 + .../avaje/ebean/config/ClassLoadConfig.java | 25 ++++ .../config/dbplatform/DatabasePlatform.java | 26 +++- .../avaje/ebean/dbmigration/DdlGenerator.java | 30 ++-- .../avaje/ebean/dbmigration/DdlRunner.java | 24 ++-- .../ebean/dbmigration/MigrationRunner.java | 68 +++++++++ .../dbmigration/model/MigrationVersion.java | 7 + .../runner/LocalMigrationResource.java | 35 +++++ .../runner/LocalMigrationResources.java | 85 +++++++++++ .../runner/MigrationScriptRunner.java | 26 ++++ .../dbmigration/runner/MigrationTable.java | 133 ++++++++++++++++++ .../server/core/DefaultServer.java | 4 + .../avaje/ebeaninternal/util/EncodeUtil.java | 99 +++++++++++++ .../com/avaje/ebeaninternal/util/IOUtils.java | 95 +++++++++++++ .../avaje/ebeaninternal/util/JdbcClose.java | 26 ++++ 15 files changed, 665 insertions(+), 24 deletions(-) create mode 100644 src/main/java/com/avaje/ebean/dbmigration/MigrationRunner.java create mode 100644 src/main/java/com/avaje/ebean/dbmigration/runner/LocalMigrationResource.java create mode 100644 src/main/java/com/avaje/ebean/dbmigration/runner/LocalMigrationResources.java create mode 100644 src/main/java/com/avaje/ebean/dbmigration/runner/MigrationScriptRunner.java create mode 100644 src/main/java/com/avaje/ebean/dbmigration/runner/MigrationTable.java create mode 100644 src/main/java/com/avaje/ebeaninternal/util/EncodeUtil.java create mode 100644 src/main/java/com/avaje/ebeaninternal/util/IOUtils.java create mode 100644 src/main/java/com/avaje/ebeaninternal/util/JdbcClose.java diff --git a/pom.xml b/pom.xml index 47ab2e1eb..4aa0107f4 100644 --- a/pom.xml +++ b/pom.xml @@ -61,6 +61,12 @@ 1.7.7 + + org.avaje.util + avaje-classpath-scanner + 1.1.1-SNAPSHOT + + org.springframework.boot diff --git a/src/main/java/com/avaje/ebean/config/ClassLoadConfig.java b/src/main/java/com/avaje/ebean/config/ClassLoadConfig.java index ca0a75c74..ac7c7674e 100644 --- a/src/main/java/com/avaje/ebean/config/ClassLoadConfig.java +++ b/src/main/java/com/avaje/ebean/config/ClassLoadConfig.java @@ -1,6 +1,10 @@ package com.avaje.ebean.config; +import java.io.IOException; +import java.net.URL; +import java.util.Enumeration; + /** * Helper to find classes taking into account the context class loader. */ @@ -70,6 +74,10 @@ public class ClassLoadConfig { } } + public Enumeration getResources(String name) throws IOException { + return context.getResources(name); + } + /** * Return true if the given class is present. */ @@ -90,6 +98,13 @@ public class ClassLoadConfig { return context.forName(name); } + /** + * Return the preferred classLoader. + */ + public ClassLoader getClassLoader() { + return context.getClassLoader(); + } + /** * Wraps the preferred, caller and context class loaders. */ @@ -115,6 +130,13 @@ public class ClassLoadConfig { return (loader != null) ? loader: callerLoader; } + Enumeration getResources(String name) throws IOException { + if (preferredLoader != null) { + return preferredLoader.getResources(name); + } + return contextLoader().getResources(name); + } + Class forName(String name) throws ClassNotFoundException { if (preferredLoader != null) { @@ -138,6 +160,9 @@ public class ClassLoadConfig { return Class.forName(name, true, classLoader); } + ClassLoader getClassLoader() { + return preferredLoader != null ? preferredLoader : contextLoader; + } } } diff --git a/src/main/java/com/avaje/ebean/config/dbplatform/DatabasePlatform.java b/src/main/java/com/avaje/ebean/config/dbplatform/DatabasePlatform.java index a4c8f73e9..cffc75973 100644 --- a/src/main/java/com/avaje/ebean/config/dbplatform/DatabasePlatform.java +++ b/src/main/java/com/avaje/ebean/config/dbplatform/DatabasePlatform.java @@ -9,6 +9,10 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import javax.sql.DataSource; +import java.sql.Connection; +import java.sql.DatabaseMetaData; +import java.sql.ResultSet; +import java.sql.SQLException; import java.sql.Types; /** @@ -18,7 +22,6 @@ public class DatabasePlatform { private static final Logger logger = LoggerFactory.getLogger(DatabasePlatform.class); - /** * Behavior used when ending a query only transaction (at read committed isolation level). */ @@ -497,4 +500,25 @@ public class DatabasePlatform { return disallowBatchOnCascade; } + /** + * Return true if the table exists. + */ + public boolean tableExists(Connection connection, String catalog, String schema, String table) throws SQLException { + + DatabaseMetaData metaData = connection.getMetaData(); + ResultSet tables = metaData.getTables(catalog, schema, table, null); + try { + return tables.next(); + } finally { + close(tables); + } + } + + protected void close(ResultSet resultSet) { + try { + resultSet.close(); + } catch (SQLException e) { + logger.error("Error closing resultSet", e); + } + } } diff --git a/src/main/java/com/avaje/ebean/dbmigration/DdlGenerator.java b/src/main/java/com/avaje/ebean/dbmigration/DdlGenerator.java index 8a6449670..0ce387167 100644 --- a/src/main/java/com/avaje/ebean/dbmigration/DdlGenerator.java +++ b/src/main/java/com/avaje/ebean/dbmigration/DdlGenerator.java @@ -2,8 +2,8 @@ package com.avaje.ebean.dbmigration; import com.avaje.ebean.config.ServerConfig; import com.avaje.ebean.dbmigration.model.CurrentModel; -import com.avaje.ebeaninternal.api.SpiEbeanPlugin; import com.avaje.ebeaninternal.api.SpiEbeanServer; +import com.avaje.ebeaninternal.util.JdbcClose; import javax.persistence.PersistenceException; import java.io.File; @@ -14,6 +14,7 @@ import java.io.InputStream; import java.io.InputStreamReader; import java.io.LineNumberReader; import java.io.Reader; +import java.sql.Connection; /** * Controls the generation and execution of "Create All" and "Drop All" DDL scripts. @@ -82,6 +83,21 @@ public class DdlGenerator { } } + /** + * Execute all the DDL statements in the script. + */ + public int runScript(boolean expectErrors, String content, String scriptName) { + + DdlRunner runner = new DdlRunner(expectErrors, scriptName); + // get a connection without threadLocal + Connection connection = server.createTransaction().getConnection(); + try { + return runner.runAll(content, connection); + } finally { + JdbcClose.close(connection); + } + } + protected void runDropSql() throws IOException { if (!createOnly) { if (dropContent == null) { @@ -111,9 +127,8 @@ public class DdlGenerator { if (sqlScript != null) { InputStream is = getClassLoader().getResourceAsStream(sqlScript); if (is != null) { - DdlRunner runner = new DdlRunner(false, sqlScript); String content = readContent(new InputStreamReader(is)); - runner.runAll(content, server); + runScript(false, content, sqlScript); } } } @@ -224,13 +239,4 @@ public class DdlGenerator { } } - /** - * Execute all the DDL statements in the script. - */ - public int runScript(boolean expectErrors, String content, String scriptName) { - - DdlRunner runner = new DdlRunner(expectErrors, scriptName); - return runner.runAll(content, server); - } - } diff --git a/src/main/java/com/avaje/ebean/dbmigration/DdlRunner.java b/src/main/java/com/avaje/ebean/dbmigration/DdlRunner.java index f1b1d2b58..9b8e3124c 100644 --- a/src/main/java/com/avaje/ebean/dbmigration/DdlRunner.java +++ b/src/main/java/com/avaje/ebean/dbmigration/DdlRunner.java @@ -1,7 +1,5 @@ package com.avaje.ebean.dbmigration; -import com.avaje.ebean.Transaction; -import com.avaje.ebeaninternal.api.SpiEbeanServer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -37,29 +35,33 @@ public class DdlRunner { /** * Parse the content into sql statements and execute them in a transaction. */ - public int runAll(String content, SpiEbeanServer server) { + public int runAll(String content, Connection connection) { List statements = ddlParser.parse(new StringReader(content)); - return runStatements(statements, server); + return runStatements(statements, connection); } /** * Execute all the statements in a single transaction. */ - public int runStatements(List statements, SpiEbeanServer server) { + public int runStatements(List statements, Connection connection) { - Transaction t = server.createTransaction(); try { - int statementCount = runStatements(expectErrors, statements, t.getConnection()); - t.commit(); - + int statementCount = runStatements(expectErrors, statements, connection); + connection.commit(); return statementCount; } catch (Exception e) { + rollback(connection); throw new PersistenceException("Error: " + e.getMessage(), e); + } + } - } finally { - t.end(); + private void rollback(Connection connection) { + try { + connection.rollback(); + } catch (SQLException e) { + logger.error("Error trying to rollback connection", e); } } diff --git a/src/main/java/com/avaje/ebean/dbmigration/MigrationRunner.java b/src/main/java/com/avaje/ebean/dbmigration/MigrationRunner.java new file mode 100644 index 000000000..7af720bc2 --- /dev/null +++ b/src/main/java/com/avaje/ebean/dbmigration/MigrationRunner.java @@ -0,0 +1,68 @@ +package com.avaje.ebean.dbmigration; + +import com.avaje.ebean.EbeanServer; +import com.avaje.ebean.Transaction; +import com.avaje.ebean.config.DbMigrationConfig; +import com.avaje.ebean.config.ServerConfig; +import com.avaje.ebean.dbmigration.runner.LocalMigrationResource; +import com.avaje.ebean.dbmigration.runner.LocalMigrationResources; +import com.avaje.ebean.dbmigration.runner.MigrationTable; +import com.avaje.ebeaninternal.util.JdbcClose; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.sql.Connection; +import java.util.List; + +/** + * + */ +public class MigrationRunner { + + private static final Logger logger = LoggerFactory.getLogger(MigrationRunner.class); + + final EbeanServer server; + final ServerConfig config; + + private final DbMigrationConfig migrationConfig; + + public MigrationRunner(EbeanServer server) { + this.server = server; + this.config = server.getPluginApi().getServerConfig(); + migrationConfig = this.config.getMigrationConfig(); + } + + public void run() { + + LocalMigrationResources resources = new LocalMigrationResources(config); + if (!resources.readResources()) { + logger.debug("no migrations to check"); + return; + } + + Transaction transaction = server.createTransaction(); + Connection connection = transaction.getConnection(); + try { + MigrationTable table = new MigrationTable(server, migrationConfig, connection); + table.createIfNeeded(); + + List localVersions = resources.getVersions(); + for (int i = 0; i < localVersions.size(); i++) { + LocalMigrationResource localVersion = localVersions.get(i); + if (!table.shouldRun(i, localVersion)) { + break; + } + } + + table.commit(); + + } catch (Exception e) { + throw new RuntimeException(e); + + } finally { + JdbcClose.close(connection); + } + + } + +} diff --git a/src/main/java/com/avaje/ebean/dbmigration/model/MigrationVersion.java b/src/main/java/com/avaje/ebean/dbmigration/model/MigrationVersion.java index 8bb5f5ed9..a60a24e40 100644 --- a/src/main/java/com/avaje/ebean/dbmigration/model/MigrationVersion.java +++ b/src/main/java/com/avaje/ebean/dbmigration/model/MigrationVersion.java @@ -20,6 +20,13 @@ public class MigrationVersion implements Comparable { this.ordering = ordering; } + /** + * Return the full version. + */ + public String getFull() { + return raw; + } + public String toString() { return raw; } diff --git a/src/main/java/com/avaje/ebean/dbmigration/runner/LocalMigrationResource.java b/src/main/java/com/avaje/ebean/dbmigration/runner/LocalMigrationResource.java new file mode 100644 index 000000000..5345dd6d8 --- /dev/null +++ b/src/main/java/com/avaje/ebean/dbmigration/runner/LocalMigrationResource.java @@ -0,0 +1,35 @@ +package com.avaje.ebean.dbmigration.runner; + +import com.avaje.ebean.dbmigration.model.MigrationVersion; +import org.avaje.classpath.scanner.Resource; + +/** + * + */ +public class LocalMigrationResource implements Comparable { + + private final MigrationVersion version; + + private final String location; + + private final Resource resource; + + public LocalMigrationResource(MigrationVersion v0, String location, Resource resource) { + this.version = v0; + this.location = location; + this.resource = resource; + } + + public MigrationVersion getVersion() { + return version; + } + + public String getLocation() { + return location; + } + + @Override + public int compareTo(LocalMigrationResource o) { + return version.compareTo(o.version); + } +} diff --git a/src/main/java/com/avaje/ebean/dbmigration/runner/LocalMigrationResources.java b/src/main/java/com/avaje/ebean/dbmigration/runner/LocalMigrationResources.java new file mode 100644 index 000000000..f0ac4c42f --- /dev/null +++ b/src/main/java/com/avaje/ebean/dbmigration/runner/LocalMigrationResources.java @@ -0,0 +1,85 @@ +package com.avaje.ebean.dbmigration.runner; + +import com.avaje.ebean.config.DbMigrationConfig; +import com.avaje.ebean.config.ServerConfig; +import com.avaje.ebean.dbmigration.model.MigrationVersion; +import org.avaje.classpath.scanner.Location; +import org.avaje.classpath.scanner.MatchResource; +import org.avaje.classpath.scanner.Resource; +import org.avaje.classpath.scanner.Scanner; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +/** + * + */ +public class LocalMigrationResources { + + private static final Logger logger = LoggerFactory.getLogger(LocalMigrationResources.class); + + private final ServerConfig serverConfig; + + private final DbMigrationConfig migrationConfig; + + private final List versions = new ArrayList(); + + public LocalMigrationResources(ServerConfig serverConfig) { + this.serverConfig = serverConfig; + this.migrationConfig = serverConfig.getMigrationConfig(); + } + + public boolean readResources() { + + String migrationPath = migrationConfig.getMigrationPath(); + + ClassLoader classLoader = serverConfig.getClassLoadConfig().getClassLoader(); + + Scanner scanner = new Scanner(classLoader); + List resourceList = scanner.scanForResources(new Location(migrationPath), new Match(migrationConfig)); + + logger.debug("resources: {}", resourceList); + + for (Resource resource : resourceList) { + String filename = resource.getFilename(); + if (filename.endsWith(migrationConfig.getApplySuffix())) { + int pos = filename.lastIndexOf(migrationConfig.getApplySuffix()); + String mainName = filename.substring(0, pos); + + MigrationVersion v0 = MigrationVersion.parse(mainName); + LocalMigrationResource res = new LocalMigrationResource(v0, resource.getLocation(), resource); + versions.add(res); + } + } + + Collections.sort(versions); + return !versions.isEmpty(); + } + + public List getVersions() { + return versions; + } + + + static class Match implements MatchResource { + + final DbMigrationConfig migrationConfig; + + Match(DbMigrationConfig migrationConfig) { + this.migrationConfig = migrationConfig; + } + + @Override + public boolean isMatch(String name) { + + return name.endsWith(migrationConfig.getApplySuffix()) + || name.endsWith(migrationConfig.getModelSuffix()) + || name.endsWith(migrationConfig.getDropSuffix()) + || name.endsWith(migrationConfig.getRollbackSuffix()); + + } + } +} diff --git a/src/main/java/com/avaje/ebean/dbmigration/runner/MigrationScriptRunner.java b/src/main/java/com/avaje/ebean/dbmigration/runner/MigrationScriptRunner.java new file mode 100644 index 000000000..f52ae6caf --- /dev/null +++ b/src/main/java/com/avaje/ebean/dbmigration/runner/MigrationScriptRunner.java @@ -0,0 +1,26 @@ +package com.avaje.ebean.dbmigration.runner; + +import com.avaje.ebean.dbmigration.DdlRunner; + +import java.sql.Connection; + +/** + * Created by rob on 5/02/16. + */ +public class MigrationScriptRunner { + + final Connection connection; + + public MigrationScriptRunner(Connection connection) { + this.connection = connection; + } + + /** + * Execute all the DDL statements in the script. + */ + public int runScript(boolean expectErrors, String content, String scriptName) { + + DdlRunner runner = new DdlRunner(expectErrors, scriptName); + return runner.runAll(content, connection); + } +} diff --git a/src/main/java/com/avaje/ebean/dbmigration/runner/MigrationTable.java b/src/main/java/com/avaje/ebean/dbmigration/runner/MigrationTable.java new file mode 100644 index 000000000..809b128fe --- /dev/null +++ b/src/main/java/com/avaje/ebean/dbmigration/runner/MigrationTable.java @@ -0,0 +1,133 @@ +package com.avaje.ebean.dbmigration.runner; + +import com.avaje.ebean.EbeanServer; +import com.avaje.ebean.SqlQuery; +import com.avaje.ebean.SqlRow; +import com.avaje.ebean.SqlUpdate; +import com.avaje.ebean.config.DbMigrationConfig; +import com.avaje.ebean.config.ServerConfig; +import com.avaje.ebean.config.dbplatform.DatabasePlatform; +import com.avaje.ebean.plugin.SpiServer; +import com.avaje.ebeaninternal.server.transaction.ExternalJdbcTransaction; +import com.avaje.ebeaninternal.util.IOUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.net.URL; +import java.sql.Connection; +import java.sql.SQLException; +import java.sql.Timestamp; +import java.util.Enumeration; +import java.util.List; + +/** + * Created by rob on 5/02/16. + */ +public class MigrationTable { + + private static final Logger logger = LoggerFactory.getLogger(MigrationTable.class); + + private final Connection connection; + + private final EbeanServer server; + + private final DatabasePlatform databasePlatform; + + private final DbMigrationConfig migrationConfig; + + private final String catalog; + private final String schema; + private final String table; + private final ServerConfig serverConfig; + + int metaRowPosition; + + List metaRows; + + public MigrationTable(EbeanServer server, DbMigrationConfig migrationConfig, Connection connection) { + this.connection = connection; + this.server = server; + + SpiServer pluginApi = server.getPluginApi(); + this.serverConfig = pluginApi.getServerConfig(); + this.databasePlatform = pluginApi.getDatabasePlatform(); + this.migrationConfig = migrationConfig; + this.catalog = null; + this.schema = null; + this.table = "ebean_migration"; + } + + + public void createIfNeeded() throws SQLException, IOException { + + if (!tableExists(connection)) { + createTable(connection); + } + + ExternalJdbcTransaction t = new ExternalJdbcTransaction(connection); + SqlQuery sqlQuery = server.createSqlQuery("select * from ebean_migration order by id for update"); + metaRows = server.findList(sqlQuery, t); + } + + private void createTable(Connection connection) throws IOException { + + String script = getCreateTableScript(); + MigrationScriptRunner run = new MigrationScriptRunner(connection); + run.runScript(false, script, "create migration tables"); + } + + private String getCreateTableScript() throws IOException { + + Enumeration resources = serverConfig.getClassLoadConfig().getResources("migration-support/create.sql"); + if (resources.hasMoreElements()) { + URL url = resources.nextElement(); + return IOUtils.readUtf8(url.openStream()); + } + return null; + } + + public boolean tableExists(Connection connection) throws SQLException { + + return databasePlatform.tableExists(connection, catalog, schema, table); + } + + + public boolean shouldRun(int runPosition, LocalMigrationResource localVersion) { + + if (runPosition >= metaRows.size()) { + logger.debug("No matching row"); + runMigration(runPosition, localVersion); + return true; + } + + return false; + } + + public void commit() throws SQLException { + connection.commit(); + } + + private void runMigration(int runPosition, LocalMigrationResource localVersion) { + + logger.debug("run migration "+localVersion.getLocation()); + + String sql = "insert into ebean_migration (id,status,run_version,dep_version,comment,checksum,run_on,run_by,run_ip) "+ + "values (?,?,?,?,?,?,?,?,?)"; + + SqlUpdate sqlUpdate = server.createSqlUpdate(sql); + sqlUpdate.setParameter(1, runPosition+1); + sqlUpdate.setParameter(2, "success"); + sqlUpdate.setParameter(3, localVersion.getVersion().getFull()); + sqlUpdate.setParameter(4, "na"); + sqlUpdate.setParameter(5, "comm"); + sqlUpdate.setParameter(6, 0); + sqlUpdate.setParameter(7, new Timestamp(System.currentTimeMillis())); + sqlUpdate.setParameter(8, "user"); + sqlUpdate.setParameter(9, "userIp"); + + ExternalJdbcTransaction t = new ExternalJdbcTransaction(connection); + server.execute(sqlUpdate, t); + + } +} diff --git a/src/main/java/com/avaje/ebeaninternal/server/core/DefaultServer.java b/src/main/java/com/avaje/ebeaninternal/server/core/DefaultServer.java index 95dc3b21d..5061185b4 100644 --- a/src/main/java/com/avaje/ebeaninternal/server/core/DefaultServer.java +++ b/src/main/java/com/avaje/ebeaninternal/server/core/DefaultServer.java @@ -14,6 +14,7 @@ import com.avaje.ebean.config.EncryptKeyManager; import com.avaje.ebean.config.ServerConfig; import com.avaje.ebean.config.dbplatform.DatabasePlatform; import com.avaje.ebean.dbmigration.DdlGenerator; +import com.avaje.ebean.dbmigration.MigrationRunner; import com.avaje.ebean.event.BeanPersistController; import com.avaje.ebean.event.BeanQueryAdapter; import com.avaje.ebean.event.readaudit.ReadAuditLogger; @@ -366,6 +367,9 @@ public final class DefaultServer implements SpiServer, SpiEbeanServer { if (migrationConfig != null) { migrationConfig.generateOnStart(this); } + + MigrationRunner migrationRunner = new MigrationRunner(this); + migrationRunner.run(); } /** diff --git a/src/main/java/com/avaje/ebeaninternal/util/EncodeUtil.java b/src/main/java/com/avaje/ebeaninternal/util/EncodeUtil.java new file mode 100644 index 000000000..e5c45aff9 --- /dev/null +++ b/src/main/java/com/avaje/ebeaninternal/util/EncodeUtil.java @@ -0,0 +1,99 @@ +package com.avaje.ebeaninternal.util; + + +import java.io.UnsupportedEncodingException; +import java.net.URLDecoder; +import java.net.URLEncoder; + + +/** + * Utilities for encoding and decoding strings. + */ +public final class EncodeUtil { + + private EncodeUtil() { + /* no instances */ + } + + + /** + * URL-encodes the specified UTF-8 string. + */ + public static String urlEncode(String string) { + if (string == null) return null; + try { + return URLEncoder.encode(string, "UTF-8"); + } catch (UnsupportedEncodingException e) { + throw new IllegalStateException("Support for UTF-8 is mandated by the Java spec", e); + } + } + + /** + * URL-decodes the specified string as a UTF-8 string. + */ + public static String urlDecode(String string) { + if (string == null) return null; + try { + return URLDecoder.decode(string, "UTF-8"); + } catch (UnsupportedEncodingException e) { + throw new IllegalStateException("Support for UTF-8 is mandated by the Java spec", e); + } + } + + /** + * Returns the bytes corresponding to the specified ASCII string. + */ + public static byte[] asciiToBytes(String string) { + if (string == null) return null; + try { + return string.getBytes("US-ASCII"); + } catch (UnsupportedEncodingException e) { + throw new IllegalStateException("Support for US-ASCII is mandated by the Java spec", e); + } + } + + /** + * Returns the ASCII string corresponding to the specified bytes. + */ + public static String bytesToAscii(byte[] data) { + try { + return new String(data, "US-ASCII"); + } catch (UnsupportedEncodingException e) { + throw new IllegalStateException("Support for US-ASCII is mandated by the Java spec", e); + } + } + + /** + * Returns the bytes corresponding to the specified UTF-8 string. + */ + public static byte[] utf8ToBytes(String string) { + if (string == null) return null; + try { + return string.getBytes("UTF-8"); + } catch (UnsupportedEncodingException e) { + throw new IllegalStateException("Support for UTF-8 is mandated by the Java spec", e); + } + } + + /** + * Returns the UTF-8 string corresponding to the specified bytes. + */ + public static String bytesToUtf8(byte[] data) { + try { + return new String(data, "UTF-8"); + } catch (UnsupportedEncodingException e) { + throw new IllegalStateException("Support for UTF-8 is mandated by the Java spec", e); + } + } + + /** + * Returns the UTF-8 string corresponding to the specified bytes. + */ + public static String decodeBytes(byte[] data, String encoding) { + try { + return new String(data, encoding); + } catch (UnsupportedEncodingException e) { + throw new RuntimeException("Error decoding bytes with "+encoding, e); + } + } +} \ No newline at end of file diff --git a/src/main/java/com/avaje/ebeaninternal/util/IOUtils.java b/src/main/java/com/avaje/ebeaninternal/util/IOUtils.java new file mode 100644 index 000000000..4d89e28b3 --- /dev/null +++ b/src/main/java/com/avaje/ebeaninternal/util/IOUtils.java @@ -0,0 +1,95 @@ +package com.avaje.ebeaninternal.util; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.io.Reader; + +/** + * Created by rob on 5/02/16. + */ +public class IOUtils { + /** + * Reads the entire contents of the specified input stream and returns them + * as a byte array. + */ + public static byte[] read(InputStream in) throws IOException { + ByteArrayOutputStream buffer = new ByteArrayOutputStream(); + pump(in, buffer); + return buffer.toByteArray(); + } + + /** + * Reads the entire contents of the specified input stream and returns them + * as an ASCII string. + */ + public static String readAscii(InputStream in) throws IOException { + return EncodeUtil.bytesToAscii(read(in)); + } + + /** + * Reads the entire contents of the specified input stream and returns them + * as UTF-8 string. + */ + public static String readUtf8(InputStream in) throws IOException { + return EncodeUtil.bytesToUtf8(read(in)); + } + + /** + * Reads the entire contents of the specified input stream and returns them + * as a string using the encoding supplied. + */ + public static String readEncoded(InputStream in, String encoding) throws IOException { + return EncodeUtil.decodeBytes(read(in), encoding); + } + + public static String read(Reader reader) throws IOException { + + StringBuilder sb = new StringBuilder(); + try { + char[] buffer = new char[4096]; + for (; ; ) { + int len = reader.read(buffer); + if (len < 0) { + break; + } + sb.append(buffer, 0, len); + } + } finally { + reader.close(); + } + + return sb.toString(); + } + + /** + * Reads data from the specified input stream and copies it to the specified + * output stream, until the input stream is at EOF. Both streams are then + * closed. + * + * @throws IOException if the input or output stream is null + */ + public static void pump(InputStream in, OutputStream out) throws IOException { + + if (in == null) throw new IOException("Input stream is null"); + if (out == null) throw new IOException("Output stream is null"); + + try { + try { + byte[] buffer = new byte[4096]; + for (; ; ) { + int bytes = in.read(buffer); + if (bytes < 0) { + break; + } + out.write(buffer, 0, bytes); + } + } finally { + in.close(); + } + } finally { + out.close(); + } + } +} diff --git a/src/main/java/com/avaje/ebeaninternal/util/JdbcClose.java b/src/main/java/com/avaje/ebeaninternal/util/JdbcClose.java new file mode 100644 index 000000000..bfdb83532 --- /dev/null +++ b/src/main/java/com/avaje/ebeaninternal/util/JdbcClose.java @@ -0,0 +1,26 @@ +package com.avaje.ebeaninternal.util; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.sql.Connection; +import java.sql.SQLException; + +/** + * Utility for closing raw Jdbc resources. + */ +public class JdbcClose { + + private static final Logger logger = LoggerFactory.getLogger(JdbcClose.class); + + /** + * Close the connection logging if an error occurs. + */ + public static void close(Connection connection) { + try { + connection.close(); + } catch (SQLException e) { + logger.warn("Error closing connection", e); + } + } +} From 6f702098bc8a2935e8099642369a3a2c7a1ea0d1 Mon Sep 17 00:00:00 2001 From: Robin Bygrave Date: Tue, 9 Feb 2016 12:55:41 +1300 Subject: [PATCH 2/3] Update DB Migration runner --- pom.xml | 4 +- .../ebean/dbmigration/MigrationRunner.java | 6 ++- .../runner/LocalMigrationResource.java | 38 ++++++++++++++----- .../runner/LocalMigrationResources.java | 6 +-- .../dbmigration/runner/MigrationTable.java | 21 +++++++++- 5 files changed, 58 insertions(+), 17 deletions(-) diff --git a/pom.xml b/pom.xml index 4aa0107f4..80ec96e15 100644 --- a/pom.xml +++ b/pom.xml @@ -62,9 +62,9 @@ - org.avaje.util + org.avaje avaje-classpath-scanner - 1.1.1-SNAPSHOT + 1.1.1 diff --git a/src/main/java/com/avaje/ebean/dbmigration/MigrationRunner.java b/src/main/java/com/avaje/ebean/dbmigration/MigrationRunner.java index 7af720bc2..e0d3b319f 100644 --- a/src/main/java/com/avaje/ebean/dbmigration/MigrationRunner.java +++ b/src/main/java/com/avaje/ebean/dbmigration/MigrationRunner.java @@ -46,10 +46,14 @@ public class MigrationRunner { MigrationTable table = new MigrationTable(server, migrationConfig, connection); table.createIfNeeded(); + LocalMigrationResource priorVersion = null; List localVersions = resources.getVersions(); for (int i = 0; i < localVersions.size(); i++) { LocalMigrationResource localVersion = localVersions.get(i); - if (!table.shouldRun(i, localVersion)) { + if (i > 0) { + priorVersion = localVersions.get(i-1); + } + if (!table.shouldRun(i, localVersion, priorVersion)) { break; } } diff --git a/src/main/java/com/avaje/ebean/dbmigration/runner/LocalMigrationResource.java b/src/main/java/com/avaje/ebean/dbmigration/runner/LocalMigrationResource.java index 5345dd6d8..979552e07 100644 --- a/src/main/java/com/avaje/ebean/dbmigration/runner/LocalMigrationResource.java +++ b/src/main/java/com/avaje/ebean/dbmigration/runner/LocalMigrationResource.java @@ -4,7 +4,7 @@ import com.avaje.ebean.dbmigration.model.MigrationVersion; import org.avaje.classpath.scanner.Resource; /** - * + * A DB migration resource. */ public class LocalMigrationResource implements Comparable { @@ -14,22 +14,42 @@ public class LocalMigrationResource implements Comparable resourceList = scanner.scanForResources(new Location(migrationPath), new Match(migrationConfig)); + List resourceList = scanner.scanForResources(migrationPath, new Match(migrationConfig)); logger.debug("resources: {}", resourceList); @@ -49,8 +49,8 @@ public class LocalMigrationResources { int pos = filename.lastIndexOf(migrationConfig.getApplySuffix()); String mainName = filename.substring(0, pos); - MigrationVersion v0 = MigrationVersion.parse(mainName); - LocalMigrationResource res = new LocalMigrationResource(v0, resource.getLocation(), resource); + MigrationVersion migrationVersion = MigrationVersion.parse(mainName); + LocalMigrationResource res = new LocalMigrationResource(migrationVersion, resource.getLocation(), resource); versions.add(res); } } diff --git a/src/main/java/com/avaje/ebean/dbmigration/runner/MigrationTable.java b/src/main/java/com/avaje/ebean/dbmigration/runner/MigrationTable.java index 809b128fe..73fb01754 100644 --- a/src/main/java/com/avaje/ebean/dbmigration/runner/MigrationTable.java +++ b/src/main/java/com/avaje/ebean/dbmigration/runner/MigrationTable.java @@ -40,6 +40,7 @@ public class MigrationTable { private final String schema; private final String table; private final ServerConfig serverConfig; + private final String envUserName; int metaRowPosition; @@ -56,6 +57,8 @@ public class MigrationTable { this.catalog = null; this.schema = null; this.table = "ebean_migration"; + + this.envUserName = System.getProperty("user.name"); } @@ -93,7 +96,15 @@ public class MigrationTable { } - public boolean shouldRun(int runPosition, LocalMigrationResource localVersion) { + public boolean shouldRun(int runPosition, LocalMigrationResource localVersion, LocalMigrationResource priorVersion) { + + // if prior != null check previous version installed + // if previous version not installed ... error - missing version (prior version) + + // if localVersion installed + // check checksum and return ok, or re-installable? + // else + // install version and continue if (runPosition >= metaRows.size()) { logger.debug("No matching row"); @@ -112,6 +123,12 @@ public class MigrationTable { logger.debug("run migration "+localVersion.getLocation()); + + String script = localVersion.getContent(); + + MigrationScriptRunner run = new MigrationScriptRunner(connection); + run.runScript(false, script, "run migration version: "+localVersion.getVersion()); + String sql = "insert into ebean_migration (id,status,run_version,dep_version,comment,checksum,run_on,run_by,run_ip) "+ "values (?,?,?,?,?,?,?,?,?)"; @@ -123,7 +140,7 @@ public class MigrationTable { sqlUpdate.setParameter(5, "comm"); sqlUpdate.setParameter(6, 0); sqlUpdate.setParameter(7, new Timestamp(System.currentTimeMillis())); - sqlUpdate.setParameter(8, "user"); + sqlUpdate.setParameter(8, envUserName); sqlUpdate.setParameter(9, "userIp"); ExternalJdbcTransaction t = new ExternalJdbcTransaction(connection); From 84fa3737283a99d4748f913ae5c8275f73fed7ae Mon Sep 17 00:00:00 2001 From: Robin Bygrave Date: Wed, 10 Feb 2016 20:51:55 +1300 Subject: [PATCH 3/3] WIP update for migration runner --- .../ebean/dbmigration/MigrationRunner.java | 2 +- .../ebean/dbmigration/runner/Checksum.java | 32 ++++++++++++++++ .../dbmigration/runner/MigrationMetaRow.java | 27 +++++++++++++ .../dbmigration/runner/MigrationTable.java | 38 +++++++++++++------ .../dbmigration/runner/ChecksumTest.java | 18 +++++++++ 5 files changed, 104 insertions(+), 13 deletions(-) create mode 100644 src/main/java/com/avaje/ebean/dbmigration/runner/Checksum.java create mode 100644 src/main/java/com/avaje/ebean/dbmigration/runner/MigrationMetaRow.java create mode 100644 src/test/java/com/avaje/ebean/dbmigration/runner/ChecksumTest.java diff --git a/src/main/java/com/avaje/ebean/dbmigration/MigrationRunner.java b/src/main/java/com/avaje/ebean/dbmigration/MigrationRunner.java index e0d3b319f..c00b4c77f 100644 --- a/src/main/java/com/avaje/ebean/dbmigration/MigrationRunner.java +++ b/src/main/java/com/avaje/ebean/dbmigration/MigrationRunner.java @@ -53,7 +53,7 @@ public class MigrationRunner { if (i > 0) { priorVersion = localVersions.get(i-1); } - if (!table.shouldRun(i, localVersion, priorVersion)) { + if (!table.shouldRun(localVersion, priorVersion)) { break; } } diff --git a/src/main/java/com/avaje/ebean/dbmigration/runner/Checksum.java b/src/main/java/com/avaje/ebean/dbmigration/runner/Checksum.java new file mode 100644 index 000000000..515bf3606 --- /dev/null +++ b/src/main/java/com/avaje/ebean/dbmigration/runner/Checksum.java @@ -0,0 +1,32 @@ +package com.avaje.ebean.dbmigration.runner; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.StringReader; +import java.util.zip.CRC32; + +/** + * Calculates the checksum for the given string content. + */ +public class Checksum { + + /** + * Returns the checksum of this string. + */ + public static int calculate(String str) { + + final CRC32 crc32 = new CRC32(); + + BufferedReader bufferedReader = new BufferedReader(new StringReader(str)); + try { + String line; + while ((line = bufferedReader.readLine()) != null) { + crc32.update(line.getBytes("UTF-8")); + } + } catch (IOException e) { + throw new RuntimeException("Failed to calculate checksum", e); + } + + return (int) crc32.getValue(); + } +} diff --git a/src/main/java/com/avaje/ebean/dbmigration/runner/MigrationMetaRow.java b/src/main/java/com/avaje/ebean/dbmigration/runner/MigrationMetaRow.java new file mode 100644 index 000000000..5a066ac87 --- /dev/null +++ b/src/main/java/com/avaje/ebean/dbmigration/runner/MigrationMetaRow.java @@ -0,0 +1,27 @@ +package com.avaje.ebean.dbmigration.runner; + +import java.sql.Timestamp; + +/** + * + */ +public class MigrationMetaRow implements Comparable { + + int id; + String status; + String runVersion; + String depVersion; + String comment; + int checksum; + Timestamp runOn; + String runBy; + + @Override + public int compareTo(MigrationMetaRow o) { + return (id < o.id) ? -1 : ((id == o.id) ? 0 : 1); + } + + + //String sql = "insert into ebean_migration (id,status,run_version,dep_version,comment,checksum,run_on,run_by,run_ip) "+ + // "values (?,?,?,?,?,?,?,?,?)"; +} diff --git a/src/main/java/com/avaje/ebean/dbmigration/runner/MigrationTable.java b/src/main/java/com/avaje/ebean/dbmigration/runner/MigrationTable.java index 73fb01754..c5d15a6ad 100644 --- a/src/main/java/com/avaje/ebean/dbmigration/runner/MigrationTable.java +++ b/src/main/java/com/avaje/ebean/dbmigration/runner/MigrationTable.java @@ -42,7 +42,7 @@ public class MigrationTable { private final ServerConfig serverConfig; private final String envUserName; - int metaRowPosition; + int currentSearchPosition; List metaRows; @@ -96,25 +96,39 @@ public class MigrationTable { } - public boolean shouldRun(int runPosition, LocalMigrationResource localVersion, LocalMigrationResource priorVersion) { + public boolean shouldRun(LocalMigrationResource localVersion, LocalMigrationResource priorVersion) { // if prior != null check previous version installed // if previous version not installed ... error - missing version (prior version) - // if localVersion installed - // check checksum and return ok, or re-installable? - // else - // install version and continue - - if (runPosition >= metaRows.size()) { - logger.debug("No matching row"); - runMigration(runPosition, localVersion); - return true; - } +// if (!priorVersionNotInstalled(priorVersion)) { +// +// } +// +// // if localVersion installed +// // check checksum and return ok, or re-installable? +// // else +// // install version and continue +// +// if (runPosition >= metaRows.size()) { +// logger.debug("No matching row"); +// runMigration(runPosition, localVersion); +// return true; +// } return false; } + private void checkInstalled(LocalMigrationResource priorVersion) { + if (priorVersion != null) { + searchFor(priorVersion); + } + } + + private void searchFor(LocalMigrationResource priorVersion) { + + } + public void commit() throws SQLException { connection.commit(); } diff --git a/src/test/java/com/avaje/ebean/dbmigration/runner/ChecksumTest.java b/src/test/java/com/avaje/ebean/dbmigration/runner/ChecksumTest.java new file mode 100644 index 000000000..b268c4444 --- /dev/null +++ b/src/test/java/com/avaje/ebean/dbmigration/runner/ChecksumTest.java @@ -0,0 +1,18 @@ +package com.avaje.ebean.dbmigration.runner; + +import org.junit.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +public class ChecksumTest { + + @Test + public void test_calculate() throws Exception { + + int checkFoo = Checksum.calculate("foo"); + + assertThat(Checksum.calculate("foo")).isEqualTo(checkFoo); + assertThat(Checksum.calculate("Foo")).isNotEqualTo(checkFoo); + } + +} \ No newline at end of file