#506 - Changes to DB Migration, support run on startup and external version numbering

This commit is contained in:
Robin Bygrave
2015-12-24 16:41:06 +13:00
parent 03a9aa3dec
commit be91e3f3f7
10 changed files with 566 additions and 156 deletions
@@ -1,15 +1,75 @@
package com.avaje.ebean.config;
import com.avaje.ebean.EbeanServer;
import com.avaje.ebean.config.dbplatform.DbPlatformName;
import com.avaje.ebean.dbmigration.DbMigration;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* Configuration for the DB migration processing.
*/
public class DbMigrationConfig {
protected static final Logger logger = LoggerFactory.getLogger(DbMigrationConfig.class);
/**
* The database platform to generate migration DDL for.
*/
protected DbPlatformName platform;
protected boolean useSubdirectories;
/**
* Set to true if the DB migration should be generated on server start.
*/
protected boolean generate;
/**
* The migration version name (typically FlywayDb compatible).
* <p>
* Example: 1.1.1_2
* <p>
* The version is expected to be the combination of the current pom version plus
* a 'feature' id. The combined version must be unique and ordered to work with
* FlywayDb so each developer sets a unique version so that the migration script
* generated is unique (typically just prior to being submitted as a merge request).
*/
protected String version;
/**
* Description text that can be appended to the version to become the ddl script file name.
* <p>
* So if the name is "a foo table" then the ddl script file could be:
* "1.1.1_2__a-foo-table.sql"
* <p>
* When the DB migration relates to a git feature (merge request) then this description text
* is a short description of the feature.
*/
protected String name;
/**
* Resource path for the migration xml and sql.
* Typically you would change 'app' to be a better/more unique.
*/
private String resourcePath = "dbmigration/app";
protected String resourcePath = "dbmigration/app";
/**
* Return the DB platform to generate migration DDL for.
*
* We typically need to explicitly specify this as migration can often be generated
* when running against H2.
*/
public DbPlatformName getPlatform() {
return platform;
}
/**
* Set the DB platform to generate migration DDL for.
*/
public void setPlatform(DbPlatformName platform) {
this.platform = platform;
}
/**
* Return the resource path for db migrations.
@@ -18,6 +78,20 @@ public class DbMigrationConfig {
return resourcePath;
}
/**
* Return true if the 'rollback' and 'drop' scripts should be put into subdirectories.
*/
public boolean isUseSubdirectories() {
return useSubdirectories;
}
/**
* Set to true if the 'rollback' and 'drop' scripts should be put into subdirectories.
*/
public void setUseSubdirectories(boolean useSubdirectories) {
this.useSubdirectories = useSubdirectories;
}
/**
* Set the resource path for db migrations.
* <p>
@@ -30,10 +104,134 @@ public class DbMigrationConfig {
this.resourcePath = resourcePath;
}
/**
* Set the migration version.
* <p>
* Note that version set via System property or environment variable <code>ddl.migration.version</code> takes precedence.
*/
public void setVersion(String version) {
this.version = version;
}
/**
* Set the migration name.
* <p>
* Note that name set via System property or environment variable <code>ddl.migration.name</code> takes precedence.
*/
public void setName(String name) {
this.name = name;
}
/**
* Load the settings from the PropertiesWrapper.
*/
public void loadSettings(PropertiesWrapper properties) {
resourcePath = properties.get("migration.resourcePath", resourcePath);
platform = properties.getEnum(DbPlatformName.class, "migration.platform", platform);
generate = properties.getBoolean("migration.generate", generate);
version = properties.get("migration.version", version);
name = properties.get("migration.name", name);
useSubdirectories = properties.getBoolean("migration.useSubdirectories", useSubdirectories);
}
/**
* Return true if the migration should be generated.
* <p>
* It is expected that when an environment variable <code>ddl.migration.enabled</code>
* is set to <code>true</code> then the DB migration will generate the migration DDL.
* </p>
*/
public boolean isGenerateOnStart() {
// environment properties take precedence
String envGenerate = readEnvironment("ddl.migration.generate");
if (envGenerate != null) {
return "true".equalsIgnoreCase(envGenerate.trim());
}
return generate;
}
/**
* Called by EbeanServer on start.
*
* <p>
* If enabled this generates the migration xml and DDL scripts.
* </p>
*/
public void generateOnStart(EbeanServer server) {
if (isGenerateOnStart()) {
if (platform == null) {
logger.warn("No platform set for migration DDL generation");
} else {
// generate the migration xml and platform specific DDL
DbMigration migration = new DbMigration(server);
migration.setPlatform(platform);
try {
migration.generateMigration();
} catch (Exception e) {
throw new RuntimeException("Error generating DB migration", e);
}
}
}
}
/**
* Return the migration version (typically FlywayDb compatible).
* <p>
* Example: 1.1.1_2
* <p>
* The version is expected to be the combination of the current pom version plus
* a 'feature' id. The combined version must be unique and ordered to work with
* FlywayDb so each developer sets a unique version so that the migration script
* generated is unique (typically just prior to being submitted as a merge request).
*/
public String getVersion() {
String envVersion = readEnvironment("ddl.migration.version");
if (!isEmpty(envVersion)) {
return envVersion.trim();
}
return version;
}
/**
* Return the migration name which is short description text that can be appended to
* the migration version to become the ddl script file name.
* <p>
* So if the name is "a foo table" then the ddl script file could be:
* "1.1.1_2__a-foo-table.sql"
* </p>
* <p>
* When the DB migration relates to a git feature (merge request) then this description text
* is a short description of the feature.
* </p>
*/
public String getName() {
String envName = readEnvironment("ddl.migration.name");
if (!isEmpty(envName)) {
return envName.trim();
}
return name;
}
/**
* Return the system or environment property.
*/
protected String readEnvironment(String key) {
String val = System.getProperty(key);
if (val == null) {
val = System.getenv(key);
}
return val;
}
/**
* Return true if the string is null or empty.
*/
protected boolean isEmpty(String val) {
return val == null || val.trim().isEmpty();
}
}
@@ -21,8 +21,8 @@ import com.avaje.ebean.dbmigration.model.CurrentModel;
import com.avaje.ebean.dbmigration.model.MConfiguration;
import com.avaje.ebean.dbmigration.model.MigrationModel;
import com.avaje.ebean.dbmigration.model.ModelContainer;
import com.avaje.ebean.dbmigration.model.PlatformDdlWriter;
import com.avaje.ebean.dbmigration.model.ModelDiff;
import com.avaje.ebean.dbmigration.model.PlatformDdlWriter;
import com.avaje.ebeaninternal.api.SpiEbeanServer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -57,6 +57,13 @@ public class DbMigration {
protected static final Logger logger = LoggerFactory.getLogger(DbMigration.class);
private static final String initialVersion = "1.0";
/**
* Set to true if DbMigration run with online EbeanServer instance.
*/
protected final boolean online;
protected SpiEbeanServer server;
protected DbMigrationConfig migrationConfig;
@@ -71,7 +78,19 @@ public class DbMigration {
protected DbConstraintNaming constraintNaming;
/**
* Create for offline migration generation.
*/
public DbMigration() {
this.online = false;
}
/**
* Create using online EbeanServer.
*/
public DbMigration(EbeanServer server) {
this.online = true;
setServer(server);
}
/**
@@ -125,7 +144,9 @@ public class DbMigration {
*/
public void setPlatform(DatabasePlatform databasePlatform) {
this.databasePlatform = databasePlatform;
DbOffline.setPlatform(databasePlatform.getName());
if (!online) {
DbOffline.setPlatform(databasePlatform.getName());
}
}
/**
@@ -175,16 +196,18 @@ public class DbMigration {
public void generateMigration() throws IOException {
// use this flag to stop other plugins like full DDL generation
DbOffline.setRunningMigration();
if (!online) {
DbOffline.setRunningMigration();
}
setDefaults();
try {
MigrationModel migrationModel = new MigrationModel(migrationConfig.getResourcePath());
ModelContainer migrated = migrationModel.read();
int nextMajorVersion = migrationModel.getNextMajorVersion();
logger.info("next migration version {}", nextMajorVersion);
File migrationDirectory = getMigrationDirectory();
MigrationModel migrationModel = new MigrationModel(migrationDirectory);
ModelContainer migrated = migrationModel.read();
CurrentModel currentModel = new CurrentModel(server, constraintNaming);
ModelContainer current = currentModel.read();
@@ -200,46 +223,88 @@ public class DbMigration {
// there were actually changes to write
Migration dbMigration = diff.getMigration();
File writePath = getWritePath();
logger.info("migration writing version {} to {}", nextMajorVersion, writePath.getAbsolutePath());
writeMigrationXml(dbMigration, writePath, nextMajorVersion);
String fullVersion = getFullVersion(migrationModel);
if (databasePlatform != null) {
// writer needs the current model to provide table/column details for
// history ddl generation (triggers, history tables etc)
DdlWrite write = new DdlWrite(new MConfiguration(), currentModel.read());
PlatformDdlWriter writer = new PlatformDdlWriter(databasePlatform, serverConfig);
writer.processMigration(dbMigration, write, writePath, nextMajorVersion);
logger.info("generating migration:{}", fullVersion);
if (!writeMigrationXml(dbMigration, migrationDirectory, fullVersion)) {
logger.warn("migration already exists, not generating DDL");
} else {
if (databasePlatform != null) {
// writer needs the current model to provide table/column details for
// history ddl generation (triggers, history tables etc)
DdlWrite write = new DdlWrite(new MConfiguration(), currentModel.read());
PlatformDdlWriter writer = createDdlWriter(databasePlatform, "");
writer.processMigration(dbMigration, write, migrationDirectory, fullVersion);
}
writeExtraPlatformDdl(fullVersion, currentModel, dbMigration, migrationDirectory);
}
writeExtraPlatformDdl(nextMajorVersion, currentModel, dbMigration, writePath);
} finally {
DbOffline.reset();
if (!online) {
DbOffline.reset();
}
}
}
/**
* Return the full version for the migration being generated.
*/
private String getFullVersion(MigrationModel migrationModel) {
String version = migrationConfig.getVersion();
if (version == null) {
version = migrationModel.getNextVersion(initialVersion);
}
String fullVersion = version;
String name = migrationConfig.getName();
if (name != null) {
fullVersion += "__" + toUnderScore(name);
}
return fullVersion;
}
/**
* Replace spaces with underscores.
*/
private String toUnderScore(String name) {
return name.replace(' ','_');
}
/**
* Write any extra platform ddl.
*/
protected void writeExtraPlatformDdl(int nextMajorVersion, CurrentModel currentModel, Migration dbMigration, File writePath) throws IOException {
protected void writeExtraPlatformDdl(String fullVersion, CurrentModel currentModel, Migration dbMigration, File writePath) throws IOException {
for (Pair pair : platforms) {
DdlWrite platformBuffer = new DdlWrite(new MConfiguration(), currentModel.read());
PlatformDdlWriter platformWriter = new PlatformDdlWriter(pair.platform, serverConfig, pair.prefix);
platformWriter.processMigration(dbMigration, platformBuffer, writePath, nextMajorVersion);
PlatformDdlWriter platformWriter = createDdlWriter(pair);
platformWriter.processMigration(dbMigration, platformBuffer, writePath, fullVersion);
}
}
private PlatformDdlWriter createDdlWriter(Pair pair) {
return createDdlWriter(pair.platform, pair.prefix);
}
private PlatformDdlWriter createDdlWriter(DatabasePlatform platform, String prefix) {
return new PlatformDdlWriter(platform, serverConfig, prefix, migrationConfig.isUseSubdirectories());
}
/**
* Write the migration xml.
*/
protected void writeMigrationXml(Migration dbMigration, File resourcePath, int migrationVersion) {
protected boolean writeMigrationXml(Migration dbMigration, File resourcePath, String fullVersion) {
File file = new File(resourcePath, "v"+migrationVersion+".0.xml");
File file = new File(resourcePath, fullVersion+".xml");
if (file.exists()) {
return false;
}
MigrationXmlWriter xmlWriter = new MigrationXmlWriter();
xmlWriter.write(dbMigration, file);
return true;
}
/**
@@ -260,7 +325,7 @@ public class DbMigration {
/**
* Return the file path to write the xml and sql to.
*/
protected File getWritePath() {
protected File getMigrationDirectory() {
// path to src/main/resources in typical maven project
File resourceRootDir = new File(pathToResources);
@@ -6,6 +6,9 @@ import com.avaje.ebean.dbmigration.migration.Migration;
import javax.xml.bind.JAXBContext;
import javax.xml.bind.JAXBException;
import javax.xml.bind.Unmarshaller;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
/**
@@ -15,19 +18,6 @@ public class MigrationXmlReader {
private static final MigrationXmlReader INSTANCE = new MigrationXmlReader();
/**
* Read and return a Migration from an xml document at the given resource path.
*/
public static Migration readMaybe(String resourcePath) {
InputStream is = MigrationXmlReader.class.getResourceAsStream(resourcePath);
if (is == null) {
return null;
}
return INSTANCE.read(is);
}
/**
* Read and return a Migration from an xml document at the given resource path.
*/
@@ -41,10 +31,27 @@ public class MigrationXmlReader {
return INSTANCE.read(is);
}
/**
* Read and return a Migration from a migration xml file.
*/
public static Migration read(File migrationFile) {
try {
FileInputStream is = new FileInputStream(migrationFile);
try {
return read(is);
} finally {
is.close();
}
} catch (IOException e) {
throw new RuntimeException(e);
}
}
/**
* Read and return a Migration from an xml document.
*/
public Migration read(InputStream is) {
public static Migration read(InputStream is) {
try {
JAXBContext jaxbContext = JAXBContext.newInstance(Migration.class);
@@ -1,12 +1,13 @@
package com.avaje.ebean.dbmigration.model;
import com.avaje.ebean.dbmigration.migration.Migration;
import com.avaje.ebean.dbmigration.migrationreader.MigrationXmlReader;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.LinkedHashSet;
import java.util.Set;
import java.io.File;
import java.io.FileFilter;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
/**
* Build the model from the series of migrations.
@@ -17,26 +18,12 @@ public class MigrationModel {
private final ModelContainer model = new ModelContainer();
private final Set<String> readVersions = new LinkedHashSet<String>();
private final File migrationDirectory;
private final String resourcePath;
private MigrationVersion lastVersion;
int nextMajorVersion;
public MigrationModel(String resourcePath) {
this.resourcePath = normaliseResourcePath(resourcePath);
}
private String normaliseResourcePath(String resourcePath) {
if (resourcePath.endsWith("/")) {
// trim trailing slash
resourcePath = resourcePath.substring(0, resourcePath.length()-1);
}
if (resourcePath.startsWith("/")) {
// trim leading slash
resourcePath = resourcePath.substring(1);
}
return resourcePath;
public MigrationModel(File migrationDirectory) {
this.migrationDirectory = migrationDirectory;
}
/**
@@ -46,57 +33,41 @@ public class MigrationModel {
public ModelContainer read() {
readMigrations();
logger.info("read versions {}", readVersions);
return model;
}
/**
* Return the set of versions that were read.
*/
public Set<String> getReadVersions() {
return readVersions;
}
public int getNextMajorVersion() {
return nextMajorVersion;
}
private void readMigrations() {
for (int majorVersion = 1; majorVersion < 100; majorVersion++) {
if (!readMinorVersions(majorVersion)){
// no major.0 version so stopping
nextMajorVersion = majorVersion;
return;
// find all the migration xml files
File[] xmlFiles = migrationDirectory.listFiles(new FileFilter() {
@Override
public boolean accept(File pathname) {
return pathname.getName().toLowerCase().endsWith(".xml");
}
});
List<MigrationResource> resources = new ArrayList<MigrationResource>();
for (File xmlFile: xmlFiles) {
resources.add(new MigrationResource(xmlFile));
}
// sort into version order before applying
Collections.sort(resources);
for (MigrationResource migrationResource: resources) {
logger.debug("read {}", migrationResource);
model.apply(migrationResource.read());
}
// remember the last version
if (!resources.isEmpty()) {
lastVersion = resources.get(resources.size() - 1).getVersion();
}
}
private boolean readMinorVersions(int majorVersion) {
public String getNextVersion(String initialVersion) {
for (int minorVersion = 0; minorVersion < 100; minorVersion++) {
if (!readMigration(majorVersion, minorVersion)) {
// continue reading next major if minorVersion 0 was read
return (minorVersion > 0);
}
}
return true;
return lastVersion == null ? initialVersion : lastVersion.nextVersion();
}
private boolean readMigration(int majorVersion, int minorVersion) {
String version = majorVersion+"."+minorVersion;
String path = "/"+resourcePath+"/v"+version+".xml";
Migration migration = MigrationXmlReader.readMaybe(path);
if (migration == null) {
logger.debug("... no migration at path:{}", path);
return false;
}
readVersions.add(version);
logger.trace("... read migration v{}", version);
model.apply(migration);
return true;
}
}
@@ -0,0 +1,51 @@
package com.avaje.ebean.dbmigration.model;
import com.avaje.ebean.dbmigration.migration.Migration;
import com.avaje.ebean.dbmigration.migrationreader.MigrationXmlReader;
import java.io.File;
/**
* Migration XML resource that holds the changes to be applied.
*/
public class MigrationResource implements Comparable<MigrationResource> {
private final File migrationFile;
private final MigrationVersion version;
/**
* Construct with a migration xml file.
*/
public MigrationResource(File migrationFile) {
this.migrationFile = migrationFile;
this.version = MigrationVersion.parse(migrationFile.getName());
}
public String toString() {
return migrationFile.getName();
}
/**
* Return the version associated with this resource.
*/
public MigrationVersion getVersion() {
return version;
}
/**
* Read and return the migration from the resource.
*/
public Migration read() {
return MigrationXmlReader.read(migrationFile);
}
/**
* Compare by underlying version.
*/
@Override
public int compareTo(MigrationResource other) {
return version.compareTo(other.version);
}
}
@@ -0,0 +1,86 @@
package com.avaje.ebean.dbmigration.model;
/**
* The version of a migration used so that migrations are processed in order.
*/
public class MigrationVersion implements Comparable<MigrationVersion> {
/**
* The raw version text.
*/
private final String raw;
/**
* The ordering parts.
*/
private final int[] ordering;
private MigrationVersion(String raw, int[] ordering) {
this.raw = raw;
this.ordering = ordering;
}
public String toString() {
return raw;
}
public String nextVersion() {
StringBuilder sb = new StringBuilder();
for (int i = 0; i < ordering.length; i++) {
if (i < ordering.length -1 ) {
sb.append(ordering[i]).append(".");
} else {
sb.append(ordering[i]+1);
}
}
return sb.toString();
}
@Override
public int compareTo(MigrationVersion other) {
int otherLength = other.ordering.length;
for (int i = 0; i < ordering.length; i++) {
if (i >= otherLength) {
// considered greater
return 1;
}
if (ordering[i] != other.ordering[i]) {
return (ordering[i] > other.ordering[i]) ? 1 : -1;
}
}
// considered the same
return 0;
}
/**
* Parse the raw version string into a MigrationVersion.
*/
public static MigrationVersion parse(String raw) {
String value = raw.replace("__",".");
value = value.replace('_','.');
String[] sections = value.split("\\.");
int[] ordering = new int[sections.length];
int stopIndex = 0;
for (int i = 0; i < sections.length; i++) {
try {
ordering[i] = Integer.parseInt(sections[i]);
stopIndex++;
} catch (NumberFormatException e) {
// stop parsing
break;
}
}
int[] actualOrder = new int[stopIndex];
System.arraycopy(ordering, 0, actualOrder, 0, stopIndex);
return new MigrationVersion(raw, actualOrder);
}
}
@@ -24,20 +24,19 @@ public class PlatformDdlWriter {
private final String platformPrefix;
public PlatformDdlWriter(DatabasePlatform platform, ServerConfig serverConfig) {
this(platform, serverConfig, "");
}
private final boolean useSubdirectories;
public PlatformDdlWriter(DatabasePlatform platform, ServerConfig serverConfig, String platformPrefix) {
public PlatformDdlWriter(DatabasePlatform platform, ServerConfig serverConfig, String platformPrefix, boolean useSubdirectories) {
this.platform = platform;
this.serverConfig = serverConfig;
this.platformPrefix = platformPrefix;
this.useSubdirectories = useSubdirectories;
}
/**
* Write the migration as platform specific ddl.
*/
public void processMigration(Migration dbMigration, DdlWrite write, File writePath, int nextMajorVersion) throws IOException {
public void processMigration(Migration dbMigration, DdlWrite write, File writePath, String fullVersion) throws IOException {
DdlHandler handler = handler();
@@ -49,16 +48,16 @@ public class PlatformDdlWriter {
}
handler.generateExtra(write);
writePlatformDdl(write, writePath, nextMajorVersion);
writePlatformDdl(write, writePath, fullVersion);
}
/**
* Write the ddl files.
*/
protected void writePlatformDdl(DdlWrite write, File resourcePath, int migrationVersion) throws IOException {
protected void writePlatformDdl(DdlWrite write, File resourcePath, String fullVersion) throws IOException {
if (!write.isApplyEmpty()) {
FileWriter applyWriter = createWriter(resourcePath, migrationVersion, "apply.sql");
FileWriter applyWriter = createWriter(resourcePath, fullVersion, "");
try {
writeApplyDdl(applyWriter, write);
applyWriter.flush();
@@ -67,7 +66,7 @@ public class PlatformDdlWriter {
}
if (!write.isApplyRollbackEmpty()) {
FileWriter applyRollbackWriter = createWriter(resourcePath, migrationVersion, "applyRollback.sql");
FileWriter applyRollbackWriter = createWriter(resourcePath, fullVersion, "rollback");
try {
writeApplyRollbackDdl(applyRollbackWriter, write);
applyRollbackWriter.flush();
@@ -78,7 +77,7 @@ public class PlatformDdlWriter {
}
if (!write.isDropEmpty()) {
FileWriter dropWriter = createWriter(resourcePath, migrationVersion, "drop.sql");
FileWriter dropWriter = createWriter(resourcePath, fullVersion, "drop");
try {
writeDropDdl(dropWriter, write);
dropWriter.flush();
@@ -88,12 +87,32 @@ public class PlatformDdlWriter {
}
}
protected FileWriter createWriter(File resourcePath, int migrationVersion, String suffix) throws IOException {
protected FileWriter createWriter(File path, String fullVersion, String suffix) throws IOException {
File applyFile = new File(resourcePath, "v" + migrationVersion + ".0-" + platformPrefix + suffix);
String fileName = fullVersion;
if (!platformPrefix.isEmpty()) {
fileName += "-"+platformPrefix;
}
if (!suffix.isEmpty()) {
fileName += "-"+suffix;
path = subPath(path, suffix);
}
fileName += ".sql";
File applyFile = new File(path, fileName);
return new FileWriter(applyFile);
}
protected File subPath(File path, String suffix) {
if (!useSubdirectories) {
return path;
}
File subPath = new File(path, suffix);
if (!subPath.exists()) {
subPath.mkdirs();
}
return subPath;
}
/**
* Write the 'Apply' DDL buffers to the writer.
*/
@@ -9,6 +9,7 @@ import com.avaje.ebean.bean.ObjectGraphNode;
import com.avaje.ebean.bean.PersistenceContext;
import com.avaje.ebean.bean.PersistenceContext.WithOption;
import com.avaje.ebean.cache.ServerCacheManager;
import com.avaje.ebean.config.DbMigrationConfig;
import com.avaje.ebean.config.EncryptKeyManager;
import com.avaje.ebean.config.ServerConfig;
import com.avaje.ebean.config.dbplatform.DatabasePlatform;
@@ -373,6 +374,10 @@ public final class DefaultServer implements SpiServer, SpiEbeanServer {
* Start any services after registering with the ClusterManager.
*/
public void start() {
DbMigrationConfig migrationConfig = serverConfig.getMigrationConfig();
if (migrationConfig != null) {
migrationConfig.generateOnStart(this);
}
}
/**