#556 - DDL - DB Migration for History - change treatment of dropped columns such that they are still included in apply script

src/main/java/com/avaje/ebean/dbmigration/DbMigration.java

@@ -236,7 +236,7 @@ public class DbMigration {
         if (databasePlatform != null) {
           // writer needs the current model to provide table/column details for
           // history ddl generation (triggers, history tables etc)
-          DdlWrite write = new DdlWrite(new MConfiguration(), currentModel.read());
+          DdlWrite write = new DdlWrite(new MConfiguration(), current);
           PlatformDdlWriter writer = createDdlWriter(databasePlatform, "");
           writer.processMigration(dbMigration, write, migrationDir , fullVersion);
         }

src/main/java/com/avaje/ebean/dbmigration/ddlgeneration/DdlWrite.java

@@ -10,6 +10,12 @@ import com.avaje.ebean.dbmigration.model.ModelContainer;
  */
 public class DdlWrite {
 
+  public enum Mode {
+    APPLY,
+    ROLLBACK,
+    DROP
+  }
+
   private final ModelContainer currentModel;
 
   private final DdlBuffer applyDropDependencies;
@@ -43,6 +49,11 @@ public class DdlWrite {
    */
   private final DdlBuffer dropHistory;
 
+  /**
+   * Buffer used to drop dependencies early in the 'drop script'.
+   */
+  private final DdlBuffer dropDropDependencies;
+
   /**
    * Create without any configuration or current model (no history support).
    */
@@ -64,6 +75,7 @@ public class DdlWrite {
     this.rollback = new BaseDdlBuffer(configuration);
     this.drop = new BaseDdlBuffer(configuration);
     this.dropHistory = new BaseDdlBuffer(configuration);
+    this.dropDropDependencies = new BaseDdlBuffer(configuration);
   }
 
   /**
@@ -99,15 +111,41 @@ public class DdlWrite {
   /**
    * Return the apply or rollback buffer.
    */
-  public DdlBuffer buffer(boolean apply) {
-    return (apply) ? apply() : rollback();
+  public DdlBuffer buffer(Mode mode) {
+    switch (mode) {
+      case APPLY: return apply();
+      case ROLLBACK: return rollback();
+      case DROP: return drop();
+      default:
+        throw new IllegalStateException("Invalid mode" + mode);
+    }
+  }
+
+  /**
+   * Return the apply or rollback buffer.
+   */
+  public DdlBuffer historyBuffer(Mode mode) {
+    switch (mode) {
+      case APPLY: return applyHistory();
+      case ROLLBACK: return rollback();
+      case DROP: return dropHistory();
+      default:
+        throw new IllegalStateException("Invalid mode" + mode);
+    }
   }
 
   /**
    * Return the apply or rollback drop dependencies buffer.
    */
-  public DdlBuffer dropDependencies(boolean apply) {
-    return (apply) ? applyDropDependencies() : rollbackDropDependencies();
+  public DdlBuffer dropDependencies(Mode mode) {
+
+    switch (mode) {
+      case APPLY: return applyDropDependencies();
+      case ROLLBACK: return rollbackDropDependencies();
+      case DROP: return dropDropDependencies();
+      default:
+        throw new IllegalStateException("Invalid mode" + mode);
+    }
   }
 
 
@@ -193,5 +231,11 @@ public class DdlWrite {
     return dropHistory;
   }
 
+  /**
+   * Return the buffer that executes early for 'drop' script.
+   */
+  public DdlBuffer dropDropDependencies() {
+    return dropDropDependencies;
+  }
 
 }

src/main/java/com/avaje/ebean/dbmigration/ddlgeneration/platform/DbTriggerBasedHistoryDdl.java

@@ -10,7 +10,6 @@ import com.avaje.ebean.dbmigration.model.MColumn;
 import com.avaje.ebean.dbmigration.model.MTable;
 
 import java.io.IOException;
-import java.util.ArrayList;
 import java.util.Collection;
 import java.util.List;
 
@@ -45,21 +44,71 @@ public abstract class DbTriggerBasedHistoryDdl implements PlatformHistoryDdl {
     this.historySuffix = serverConfig.getHistoryTableSuffix();
     this.constraintNaming = serverConfig.getConstraintNaming();
 
-    this.sysPeriodStart = sysPeriod+"_start";
-    this.sysPeriodEnd = sysPeriod+"_end";
+    this.sysPeriodStart = sysPeriod + "_start";
+    this.sysPeriodEnd = sysPeriod + "_end";
   }
 
   @Override
-  public void regenerateHistoryTriggers(DdlWrite writer, HistoryTableUpdate update) throws IOException {
+  public void updateTriggers(DdlWrite writer, HistoryTableUpdate update) throws IOException {
 
     MTable table = writer.getTable(update.getBaseTable());
     if (table == null) {
-      throw new IllegalStateException("MTable "+update.getBaseTable()+" not found in writer? (required for history DDL)");
+      throw new IllegalStateException("MTable " + update.getBaseTable() + " not found in writer? (required for history DDL)");
     }
-    regenerateHistoryTriggers(writer, table, update);
+    updateTriggers(writer, table, update);
   }
 
-  protected abstract void regenerateHistoryTriggers(DdlWrite writer, MTable table, HistoryTableUpdate update) throws IOException;
+  /**
+   * Replace the existing triggers/stored procedures/views for history table support given the included columns.
+   */
+  protected abstract void updateHistoryTriggers(DbTriggerUpdate triggerUpdate) throws IOException;
+
+  /**
+   * Process the HistoryTableUpdate which can result in changes to the apply, rollback
+   * and drop scripts.
+   */
+  protected void updateTriggers(DdlWrite writer, MTable table, HistoryTableUpdate update) throws IOException {
+
+    DbTriggerUpdate triggerUpdate = createDbTriggerUpdate(writer, table);
+
+    if (update.hasApplyChanges()) {
+      // includes add, include and exclude column changes
+
+      String applyChangeDescription = update.descriptionForApply();
+      List<String> includedColumns = columnNamesForApply(table);
+
+      DdlBuffer apply = writer.applyHistory();
+      apply.append("-- changes: ").append(applyChangeDescription).newLine();
+
+      triggerUpdate.prepare(DdlWrite.Mode.APPLY, includedColumns);
+      updateHistoryTriggers(triggerUpdate);
+
+      // put a reverted version into the rollback buffer
+      update.toRevertedColumns(includedColumns);
+
+      DdlBuffer rollback = writer.rollback();
+      rollback.append("-- revert changes: ").append(applyChangeDescription).newLine();
+
+      triggerUpdate.prepare(DdlWrite.Mode.ROLLBACK, includedColumns);
+      updateHistoryTriggers(triggerUpdate);//writer, DdlWrite.Mode.ROLLBACK, baseTableName, historyTableName, includedColumns);
+    }
+
+    if (update.hasDropChanges()) {
+      // effectively applies the dropped columns changes to history triggers
+
+      DdlBuffer drop = writer.dropHistory();
+      drop.append("-- changes: ").append(update.descriptionForDrop()).newLine();
+
+      triggerUpdate.prepare(DdlWrite.Mode.DROP, columnNamesForDrop(table));
+      updateHistoryTriggers(triggerUpdate);//writer, DdlWrite.Mode.DROP, baseTableName, historyTableName, columnNamesForDrop(table));
+    }
+  }
+
+  protected DbTriggerUpdate createDbTriggerUpdate(DdlWrite writer, MTable table) {
+    String baseTableName = table.getName();
+    String historyTableName = historyTableName(baseTableName);
+    return new DbTriggerUpdate(baseTableName, historyTableName, writer);
+  }
 
   @Override
   public void dropHistoryTable(DdlWrite writer, DropHistoryTable dropHistoryTable) throws IOException {
@@ -78,7 +127,7 @@ public abstract class DbTriggerBasedHistoryDdl implements PlatformHistoryDdl {
     String baseTable = addHistoryTable.getBaseTable();
     MTable table = writer.getTable(baseTable);
     if (table == null) {
-      throw new IllegalStateException("MTable "+baseTable+" not found in writer? (required for history DDL)");
+      throw new IllegalStateException("MTable " + baseTable + " not found in writer? (required for history DDL)");
     }
 
     createWithHistory(writer, table);
@@ -95,7 +144,7 @@ public abstract class DbTriggerBasedHistoryDdl implements PlatformHistoryDdl {
     dropHistoryTableEtc(writer.rollback(), baseTable);
 
     addHistoryTable(writer, table, whenCreatedColumn);
-    addStoredFunction(writer, table, null);
+    createStoredFunction(writer, table);
     createTriggers(writer, table);
   }
 
@@ -103,7 +152,7 @@ public abstract class DbTriggerBasedHistoryDdl implements PlatformHistoryDdl {
 
   protected abstract void dropTriggers(DdlBuffer buffer, String baseTable) throws IOException;
 
-  protected void addStoredFunction(DdlWrite writer, MTable table, HistoryTableUpdate update) throws IOException {
+  protected void createStoredFunction(DdlWrite writer, MTable table) throws IOException {
     // do nothing
   }
 
@@ -158,7 +207,7 @@ public abstract class DbTriggerBasedHistoryDdl implements PlatformHistoryDdl {
 
     apply.append("create table ").append(table.getName()).append(historySuffix).append("(").newLine();
 
-    Collection<MColumn> cols = table.getColumns().values();
+    Collection<MColumn> cols = table.allColumns();
     for (MColumn column : cols) {
       if (!column.isDraftOnly()) {
         writeColumnDefinition(apply, column.getName(), column.getType());
@@ -194,15 +243,16 @@ public abstract class DbTriggerBasedHistoryDdl implements PlatformHistoryDdl {
   /**
    * Create or replace the with_history view with explicit columns.
    */
-  protected void createWithHistoryView(DdlBuffer apply, String baseTableName, List<String> columns) throws IOException {
+  protected void createWithHistoryView(DbTriggerUpdate update) throws IOException {
 
-    apply.append("create or replace view ").append(baseTableName).append(viewSuffix).append(" as select ");
-    appendColumnNames(apply, columns, "");
+    DdlBuffer apply = update.historyBuffer();
+    apply.append("create or replace view ").append(update.getBaseTable()).append(viewSuffix).append(" as select ");
+    appendColumnNames(apply, update.getColumns(), "");
     appendSysPeriodColumns(apply, ", ");
-    apply.append(" from ").append(baseTableName).append(" union all select ");
-    appendColumnNames(apply, columns, "");
+    apply.append(" from ").append(update.getBaseTable()).append(" union all select ");
+    appendColumnNames(apply, update.getColumns(), "");
     appendSysPeriodColumns(apply, ", ");
-    apply.append(" from ").append(baseTableName).append(historySuffix).endOfStatement().end();
+    apply.append(" from ").append(update.getHistoryTable()).endOfStatement().end();
   }
 
   protected void appendSysPeriodColumns(DdlBuffer apply, String prefix) throws IOException {
@@ -233,7 +283,7 @@ public abstract class DbTriggerBasedHistoryDdl implements PlatformHistoryDdl {
 
   protected void appendColumnNames(DdlBuffer buffer, List<String> columns, String columnPrefix) throws IOException {
 
-    for (int i=0; i< columns.size(); i++) {
+    for (int i = 0; i < columns.size(); i++) {
       if (i > 0) {
         buffer.append(", ");
       }
@@ -253,18 +303,23 @@ public abstract class DbTriggerBasedHistoryDdl implements PlatformHistoryDdl {
   }
 
   /**
-   * Return the list of included columns in order.
+   * Return the column names included in history for the apply script.
+   * <p>
+   * Note that dropped columns are actually still included at this point as they are going
+   * to be removed from the history handling when the drop script runs that also deletes
+   * the column.
+   * </p>
    */
-  protected List<String> includedColumnNames(MTable table) throws IOException {
+  protected List<String> columnNamesForApply(MTable table) throws IOException {
 
-    Collection<MColumn> columns = table.getColumns().values();
-    List<String> includedColumns = new ArrayList<String>(columns.size());
+    return table.allHistoryColumns(true);
+  }
 
-    for (MColumn column : columns) {
-      if (column.isIncludeInHistory()) {
-        includedColumns.add(column.getName());
-      }
-    }
-    return includedColumns;
+  /**
+   * Return the column names included in history for the drop script.
+   */
+  protected List<String> columnNamesForDrop(MTable table) throws IOException {
+
+    return table.allHistoryColumns(false);
   }
 }

src/main/java/com/avaje/ebean/dbmigration/ddlgeneration/platform/DbTriggerUpdate.java

@@ -0,0 +1,74 @@
+package com.avaje.ebean.dbmigration.ddlgeneration.platform;
+
+import com.avaje.ebean.dbmigration.ddlgeneration.DdlBuffer;
+import com.avaje.ebean.dbmigration.ddlgeneration.DdlWrite;
+
+import java.util.List;
+
+/**
+ * DB trigger update when a change occurs on a table with history.
+ */
+public class DbTriggerUpdate {
+
+  private final String baseTableName;
+
+  private final String historyTableName;
+
+  private final DdlWrite writer;
+
+  private DdlWrite.Mode mode;
+
+  private List<String> includedColumns;
+
+  public DbTriggerUpdate(String baseTableName, String historyTableName, DdlWrite writer) {
+    this.baseTableName = baseTableName;
+    this.historyTableName = historyTableName;
+    this.writer = writer;
+  }
+
+  /**
+   * Prepare for use given the mode and columns included in history.
+   * @param mode
+   * @param includedColumns
+   */
+  public void prepare(DdlWrite.Mode mode, List<String> includedColumns) {
+    this.mode = mode;
+    this.includedColumns = includedColumns;
+  }
+
+  /**
+   * Return the appropriate buffer for the current mode.
+   */
+  public DdlBuffer historyBuffer() {
+    return writer.historyBuffer(mode);
+  }
+
+  /**
+   * Return the appropriate drop dependency buffer for the current mode.
+   */
+  public DdlBuffer dropDependencyBuffer() {
+    return writer.dropDependencies(mode);
+  }
+
+  /**
+   * Return the base table name.
+   */
+  public String getBaseTable() {
+    return baseTableName;
+  }
+
+  /**
+   * Return the history table name.
+   */
+  public String getHistoryTable() {
+    return historyTableName;
+  }
+
+  /**
+   * Return the included columns.
+   */
+  public List<String> getColumns() {
+    return includedColumns;
+  }
+
+}

src/main/java/com/avaje/ebean/dbmigration/ddlgeneration/platform/H2HistoryDdl.java

@@ -6,7 +6,6 @@ import com.avaje.ebean.dbmigration.ddlgeneration.DdlWrite;
 import com.avaje.ebean.dbmigration.model.MTable;
 
 import java.io.IOException;
-import java.util.List;
 
 /**
  * H2 history support using DB triggers to maintain a history table.
@@ -28,40 +27,20 @@ public class H2HistoryDdl extends DbTriggerBasedHistoryDdl {
   protected void createTriggers(DdlWrite writer, MTable table) throws IOException {
 
     String baseTableName = table.getName();
-    String historyTableName = historyTableName(baseTableName);
-    List<String> includedColumns = includedColumnNames(table);
-
     DdlBuffer apply = writer.applyHistory();
 
-    addCreateTrigger(apply, updateTriggerName(baseTableName), baseTableName, historyTableName, includedColumns);
+    addCreateTrigger(apply, updateTriggerName(baseTableName), baseTableName);
   }
 
   @Override
-  protected void regenerateHistoryTriggers(DdlWrite writer, MTable table, HistoryTableUpdate update) throws IOException {
+  protected void updateHistoryTriggers(DbTriggerUpdate update) throws IOException {
 
-    String baseTableName = table.getName();
-    String historyTableName = historyTableName(baseTableName);
-    List<String> includedColumns = includedColumnNames(table);
-
-    DdlBuffer apply = writer.applyHistory();
-
-    apply.append("-- Regenerated ").newLine();
-    apply.append("-- changes: ").append(update.description()).newLine();
-
-    dropTriggers(apply, baseTableName);
-    addCreateTrigger(apply, updateTriggerName(baseTableName), baseTableName, historyTableName, includedColumns);
-
-    // put a reverted version into the rollback buffer
-    update.toRevertedColumns(includedColumns);
-
-    DdlBuffer rollback = writer.rollback();
-    rollback.append("-- Revert regenerated ").newLine();
-    rollback.append("-- revert changes: ").append(update.description()).newLine();
-    dropTriggers(rollback, baseTableName);
-    addCreateTrigger(rollback, updateTriggerName(baseTableName), baseTableName, historyTableName, includedColumns);
+    DdlBuffer buffer = update.historyBuffer();
+    dropTriggers(buffer, update.getBaseTable());
+    addCreateTrigger(buffer, updateTriggerName(update.getBaseTable()), update.getBaseTable());
   }
 
-  private void addCreateTrigger(DdlBuffer apply, String triggerName, String baseTable, String historyTable, List<String> includedColumns) throws IOException {
+  private void addCreateTrigger(DdlBuffer apply, String triggerName, String baseTable) throws IOException {
 
     // Note that this does not take into account the historyTable name (excepts _history suffix) and
     // does not take into account excluded columns (all columns included in history)

src/main/java/com/avaje/ebean/dbmigration/ddlgeneration/platform/HistoryTableUpdate.java

@@ -21,10 +21,12 @@ public class HistoryTableUpdate {
     EXCLUDE
   }
 
-  public static class Column {
+  private static class Column {
+
+    final Change change;
+
+    final String column;
 
-    public final Change change;
-    public final String column;
     public Column(Change change, String column) {
       this.change = change;
       this.column = column;
@@ -34,18 +36,23 @@ public class HistoryTableUpdate {
       return change.name().toLowerCase()+" "+column;
     }
 
-    public void apply(List<String> includedColumns) {
+    private boolean isChangeFor(boolean apply) {
+      return apply ? change != Change.DROP : change == Change.DROP;
+    }
+
+    private void revert(List<String> includedColumns) {
       switch (change) {
         case ADD:
         case INCLUDE: {
           includedColumns.remove(column);
           break;
         }
-        case DROP:
         case EXCLUDE: {
           includedColumns.add(column);
           break;
         }
+        case DROP:
+          break;
         default:
           throw new IllegalStateException("Unexpected change "+change);
       }
@@ -63,29 +70,73 @@ public class HistoryTableUpdate {
     this.baseTable = baseTable;
   }
 
+  private boolean isChangeFor(boolean apply) {
+    for (Column columnChange : columnChanges) {
+      if (columnChange.isChangeFor(apply)) {
+        return true;
+      }
+    }
+    return false;
+  }
+
+  /**
+   * Return true if the change includes apply changes (ADD, INCLUDE, EXCLUDE).
+   */
+  public boolean hasApplyChanges() {
+    return isChangeFor(true);
+  }
+
+  /**
+   * Return true if the change includes DROP column.
+   */
+  public boolean hasDropChanges() {
+    return isChangeFor(false);
+  }
+
   /**
    * Return a description of the changes that cause the history trigger/function
-   * to be regenerated (added, dropped, included or excluded columns).
+   * to be regenerated (added, included or excluded columns).
    */
-  public String description() {
+  public String descriptionForApply() {
+    return descriptionFor(true);
+  }
+
+  /**
+   * Return a description of the changes that cause the history trigger/function
+   * to be regenerated in the drop script (dropped columns only).
+   */
+  public String descriptionForDrop() {
+    return descriptionFor(false);
+  }
+
+  private String descriptionFor(boolean apply) {
+
     StringBuilder sb = new StringBuilder(90);
-    for (int i = 0; i < columnChanges.size(); i++) {
-      if (i > 0) {
-        sb.append(", ");
+    boolean first = true;
+    for (Column column : columnChanges) {
+      if (column.isChangeFor(apply)) {
+        if (first) {
+          first = false;
+        } else {
+          sb.append(", ");
+        }
+        sb.append(column.description());
       }
-      sb.append(columnChanges.get(i).description());
     }
     return sb.toString();
   }
 
+  /**
+   * Reverse the apply changes which equates to removing any newly added or
+   * included columns.
+   */
   public void toRevertedColumns(List<String> includedColumns) {
 
     for (Column columnChange : columnChanges) {
-      columnChange.apply(includedColumns);
+      columnChange.revert(includedColumns);
     }
   }
 
-
   /**
    * Add a comment for column added, dropped, included or excluded.
    */
@@ -100,10 +151,4 @@ public class HistoryTableUpdate {
     return baseTable;
   }
 
-  /**
-   * Return the comments.
-   */
-  public List<Column> getColumnChanges() {
-    return columnChanges;
-  }
 }

src/main/java/com/avaje/ebean/dbmigration/ddlgeneration/platform/MySqlHistoryDdl.java

@@ -5,7 +5,6 @@ import com.avaje.ebean.dbmigration.ddlgeneration.DdlWrite;
 import com.avaje.ebean.dbmigration.model.MTable;
 
 import java.io.IOException;
-import java.util.List;
 
 /**
  * MySql history support using DB triggers to maintain a history table.
@@ -26,67 +25,48 @@ public class MySqlHistoryDdl extends DbTriggerBasedHistoryDdl {
   @Override
   protected void createTriggers(DdlWrite writer, MTable table) throws IOException {
 
-    String baseTableName = table.getName();
-    String historyTableName = historyTableName(baseTableName);
-    List<String> includedColumns = includedColumnNames(table);
+    DbTriggerUpdate update = createDbTriggerUpdate(writer, table);
 
-    DdlBuffer apply = writer.applyHistory();
+    update.prepare(DdlWrite.Mode.APPLY, columnNamesForApply(table));
 
-    addBeforeUpdate(apply, updateTriggerName(baseTableName), baseTableName, historyTableName, includedColumns);
-    addBeforeDelete(apply, deleteTriggerName(baseTableName), baseTableName, historyTableName, includedColumns);
+    addBeforeUpdate(updateTriggerName(update.getBaseTable()), update);
+    addBeforeDelete(deleteTriggerName(update.getBaseTable()), update);
   }
 
   @Override
-  protected void regenerateHistoryTriggers(DdlWrite writer, MTable table, HistoryTableUpdate update) throws IOException {
+  protected void updateHistoryTriggers(DbTriggerUpdate update) throws IOException {
 
-    String baseTableName = table.getName();
-    String historyTableName = historyTableName(baseTableName);
-    List<String> includedColumns = includedColumnNames(table);
+    DdlBuffer buffer = update.historyBuffer();
+    String baseTable = update.getBaseTable();
 
-    DdlBuffer apply = writer.applyHistory();
-
-    apply.append("-- Regenerated ").newLine();
-    apply.append("-- changes: ").append(update.description()).newLine();
-    // lock the base table while we drop and recreate the triggers
-    apply.append("lock tables ").append(baseTableName).append(" write").endOfStatement();
-    dropTriggers(apply, baseTableName);
-    addBeforeUpdate(apply, updateTriggerName(baseTableName), baseTableName, historyTableName, includedColumns);
-    addBeforeDelete(apply, deleteTriggerName(baseTableName), baseTableName, historyTableName, includedColumns);
-    apply.append("unlock tables").endOfStatement();
-
-    // put a reverted version into the rollback buffer
-    update.toRevertedColumns(includedColumns);
-
-    DdlBuffer rollback = writer.rollback();
-    rollback.append("-- Revert regenerated ").newLine();
-    rollback.append("-- revert changes: ").append(update.description()).newLine();
-    // lock the base table while we drop and recreate the triggers
-    rollback.append("lock tables ").append(baseTableName).append(" write").endOfStatement();
-    dropTriggers(rollback, baseTableName);
-    addBeforeUpdate(rollback, updateTriggerName(baseTableName), baseTableName, historyTableName, includedColumns);
-    addBeforeDelete(rollback, deleteTriggerName(baseTableName), baseTableName, historyTableName, includedColumns);
-    rollback.append("unlock tables").endOfStatement();
+    buffer.append("lock tables ").append(baseTable).append(" write").endOfStatement();
+    dropTriggers(buffer, baseTable);
+    addBeforeUpdate(updateTriggerName(baseTable), update);
+    addBeforeDelete(deleteTriggerName(baseTable), update);
+    buffer.append("unlock tables").endOfStatement();
   }
 
-  private void addBeforeUpdate(DdlBuffer apply, String triggerName, String baseTable, String historyTable, List<String> includedColumns) throws IOException {
+  private void addBeforeUpdate(String triggerName, DbTriggerUpdate update) throws IOException {
 
+    DdlBuffer apply = update.historyBuffer();
     apply
         .append("delimiter $$").newLine()
-        .append("create trigger ").append(triggerName).append(" before update on ").append(baseTable)
+        .append("create trigger ").append(triggerName).append(" before update on ").append(update.getBaseTable())
         .append(" for each row begin").newLine();
-    appendInsertIntoHistory(apply, historyTable, includedColumns);
+    appendInsertIntoHistory(apply, update.getHistoryTable(), update.getColumns());
     apply
         .append("    set NEW.").append(sysPeriod).append("_start = now(6)").endOfStatement()
         .append("end$$").newLine();
   }
 
-  private void addBeforeDelete(DdlBuffer apply, String triggerName, String baseTable, String historyTable, List<String> includedColumns) throws IOException {
+  private void addBeforeDelete(String triggerName, DbTriggerUpdate update) throws IOException {
 
+    DdlBuffer apply = update.historyBuffer();
     apply
         .append("delimiter $$").newLine()
-        .append("create trigger ").append(triggerName).append(" before delete on ").append(baseTable)
+        .append("create trigger ").append(triggerName).append(" before delete on ").append(update.getBaseTable())
         .append(" for each row begin").newLine();
-    appendInsertIntoHistory(apply, historyTable, includedColumns);
+    appendInsertIntoHistory(apply, update.getHistoryTable(), update.getColumns());
     apply.append("end$$").newLine();
   }
 

src/main/java/com/avaje/ebean/dbmigration/ddlgeneration/platform/NoHistorySupportDdl.java

@@ -35,7 +35,7 @@ public class NoHistorySupportDdl implements PlatformHistoryDdl {
   }
 
   @Override
-  public void regenerateHistoryTriggers(DdlWrite write, HistoryTableUpdate update) {
+  public void updateTriggers(DdlWrite write, HistoryTableUpdate update) {
     // does nothing
   }
 }

src/main/java/com/avaje/ebean/dbmigration/ddlgeneration/platform/PlatformDdl.java

@@ -200,7 +200,7 @@ public class PlatformDdl {
    * Regenerate the history triggers (or function) due to a column being added/dropped/excluded or included.
    */
   public void regenerateHistoryTriggers(DdlWrite write, HistoryTableUpdate update) throws IOException {
-    historyDdl.regenerateHistoryTriggers(write, update);
+    historyDdl.updateTriggers(write, update);
   }
 
   /**

src/main/java/com/avaje/ebean/dbmigration/ddlgeneration/platform/PlatformHistoryDdl.java

@@ -36,5 +36,5 @@ public interface PlatformHistoryDdl {
   /**
    * Regenerate the history triggers/stored function due to column added/dropped/included or excluded.
    */
-  void regenerateHistoryTriggers(DdlWrite write, HistoryTableUpdate baseTable) throws IOException;
+  void updateTriggers(DdlWrite write, HistoryTableUpdate baseTable) throws IOException;
 }

src/main/java/com/avaje/ebean/dbmigration/ddlgeneration/platform/PostgresHistoryDdl.java

@@ -77,7 +77,7 @@ public class PostgresHistoryDdl extends DbTriggerBasedHistoryDdl {
     buffer.end();
   }
 
-  protected void addFunction(DdlBuffer apply, String procedureName, String historyTable, List<String> includedColumns) throws IOException {
+  protected void createOrReplaceFunction(DdlBuffer apply, String procedureName, String historyTable, List<String> includedColumns) throws IOException {
     apply
         .append("create or replace function ").append(procedureName).append("() returns trigger as $$").newLine()
         .append("begin").newLine();
@@ -101,55 +101,36 @@ public class PostgresHistoryDdl extends DbTriggerBasedHistoryDdl {
   }
 
   @Override
-  protected void regenerateHistoryTriggers(DdlWrite writer, MTable table, HistoryTableUpdate update) throws IOException {
-
-    // just replace the stored function with 'create or replace'
-    addStoredFunction(writer, table, update);
-  }
-
-  @Override
-  protected void addStoredFunction(DdlWrite writer, MTable table, HistoryTableUpdate update) throws IOException {
+  protected void createStoredFunction(DdlWrite writer, MTable table) throws IOException {
 
     String procedureName = procedureName(table.getName());
     String historyTable = historyTableName(table.getName());
 
-    List<String> includedColumns = includedColumnNames(table);
+    List<String> columnNames = columnNamesForApply(table);
+    createOrReplaceFunction(writer.applyHistory(), procedureName, historyTable, columnNames);
+  }
 
-    DdlBuffer apply = writer.applyHistory();
+  @Override
+  protected void updateHistoryTriggers(DbTriggerUpdate update) throws IOException {
 
-    if (update != null) {
-      apply.append("-- regenerated ").append(procedureName).newLine();
-      apply.append("-- changes: ").append(update.description()).newLine();
+    String procedureName = procedureName(update.getBaseTable());
 
-      recreateHistoryView(writer, true, table.getName(), includedColumns);
-    }
-
-    addFunction(apply, procedureName, historyTable, includedColumns);
-
-    if (update != null) {
-      // put a reverted version into the rollback buffer
-      update.toRevertedColumns(includedColumns);
-
-      DdlBuffer rollback = writer.rollback();
-      rollback.append("-- reverse regenerated ").append(procedureName).newLine();
-      rollback.append("-- changes: ").append(update.description()).newLine();
-
-      recreateHistoryView(writer, false, table.getName(), includedColumns);
-      addFunction(rollback, procedureName, historyTable, includedColumns);
-    }
+    recreateHistoryView(update);
+    createOrReplaceFunction(update.historyBuffer(), procedureName, update.getHistoryTable(), update.getColumns());
   }
 
   /**
    * For postgres we need to drop and recreate the view. Well, we could add columns to the end of the view
    * but otherwise we need to drop and create it.
    */
-  private void recreateHistoryView(DdlWrite writer, boolean apply, String baseTableName, List<String> includedColumns) throws IOException {
+  private void recreateHistoryView(DbTriggerUpdate update) throws IOException {
 
+    DdlBuffer buffer = update.dropDependencyBuffer();
     // we need to drop the view early/first before any changes to the tables etc
-    writer.dropDependencies(apply).append("drop view if exists ").append(baseTableName).append(viewSuffix).endOfStatement();
+    buffer.append("drop view if exists ").append(update.getBaseTable()).append(viewSuffix).endOfStatement();
 
     // recreate the view with specific columns specified (the columns generally are not dropped until later)
-    createWithHistoryView(writer.buffer(apply), baseTableName, includedColumns);
+    createWithHistoryView(update);
   }
 
   @Override

src/main/java/com/avaje/ebean/dbmigration/model/MColumn.java

@@ -283,7 +283,6 @@ public class MColumn {
   /**
    * Compare the column meta data and return true if there is a change that means
    * the history table column needs
-
    */
   public void compare(ModelDiff modelDiff, MTable table, MColumn newColumn) {
 

src/main/java/com/avaje/ebean/dbmigration/model/MTable.java

@@ -16,6 +16,7 @@ import org.slf4j.LoggerFactory;
 import java.math.BigInteger;
 import java.util.ArrayList;
 import java.util.Collection;
+import java.util.Collections;
 import java.util.HashSet;
 import java.util.LinkedHashMap;
 import java.util.List;
@@ -121,6 +122,8 @@ public class MTable {
    */
   private AddColumn addColumn;
 
+  private List<DroppedColumn> droppedColumns = new ArrayList<DroppedColumn>();
+
   /**
    * Create a copy of this table structure as a 'draft' table.
    *
@@ -136,7 +139,7 @@ public class MTable {
     // compoundUniqueConstraints
     draftTable.identityType = identityType;
 
-    for (MColumn col: columns.values()) {
+    for (MColumn col: allColumns()) {
       draftTable.addColumn(col.copyForDraft());
     }
 
@@ -202,7 +205,7 @@ public class MTable {
       createTable.setDraft(Boolean.TRUE);
     }
 
-    for (MColumn column : this.columns.values()) {
+    for (MColumn column : allColumns()) {
       // filter out draftOnly columns from the base table
       if (draft || !column.isDraftOnly()) {
         createTable.getColumn().add(column.createColumn());
@@ -255,7 +258,7 @@ public class MTable {
 
     // compare newColumns to existing columns (look for new and diff columns)
     for (MColumn newColumn : newColumnMap.values()) {
-      MColumn localColumn = this.columns.get(newColumn.getName());
+      MColumn localColumn = getColumn(newColumn.getName());
       if (localColumn == null) {
         // can ignore if draftOnly column and non-draft table
         if (!newColumn.isDraftOnly() || draft) {
@@ -267,15 +270,17 @@ public class MTable {
     }
 
     // compare existing columns (look for dropped columns)
-    for (MColumn existingColumn : columns.values()) {
+    int columnPosition = 0;
+    for (MColumn existingColumn : allColumns()) {
       MColumn newColumn = newColumnMap.get(existingColumn.getName());
       if (newColumn == null) {
-        diffDropColumn(modelDiff, existingColumn);
+        diffDropColumn(modelDiff, existingColumn, columnPosition, newTable);
       } else if (newColumn.isDraftOnly() && !draft) {
         // effectively a drop column (draft only column on a non-draft table)
         logger.trace("... drop column {} from table {} as now draftOnly", newColumn.getName(), name);
-        diffDropColumn(modelDiff, existingColumn);
+        diffDropColumn(modelDiff, existingColumn, columnPosition, newTable);
       }
+      columnPosition++;
     }
 
     if (addColumn != null) {
@@ -299,7 +304,7 @@ public class MTable {
   public void apply(AlterColumn alterColumn) {
     checkTableName(alterColumn.getTableName());
     String columnName = alterColumn.getColumnName();
-    MColumn existingColumn = columns.get(columnName);
+    MColumn existingColumn = getColumn(columnName);
     if (existingColumn == null) {
       throw new IllegalStateException("Column [" + columnName + "] does not exist for AlterColumn change?");
     }
@@ -311,7 +316,10 @@ public class MTable {
    */
   public void apply(DropColumn dropColumn) {
     checkTableName(dropColumn.getTableName());
-    columns.remove(dropColumn.getColumnName());
+    MColumn removed = columns.remove(dropColumn.getColumnName());
+    if (removed == null) {
+      throw new IllegalStateException("Column [" + dropColumn.getColumnName() + "] does not exist for DropColumn change on table [" + dropColumn.getTableName() + "]?");
+    }
   }
 
   public String getName() {
@@ -325,10 +333,6 @@ public class MTable {
     return draft;
   }
 
-  public String getPkName() {
-    return pkName;
-  }
-
   public void setPkName(String pkName) {
     this.pkName = pkName;
   }
@@ -357,7 +361,43 @@ public class MTable {
     this.withHistory = withHistory;
   }
 
-  public Map<String, MColumn> getColumns() {
+  public List<String> allHistoryColumns(boolean includeDropped) {
+
+    List<String> columnNames = new ArrayList<String>(columns.size());
+    for (MColumn column : columns.values()) {
+      if (column.isIncludeInHistory()) {
+        columnNames.add(column.getName());
+      }
+    }
+    if (includeDropped && !droppedColumns.isEmpty()) {
+      Collections.sort(droppedColumns);
+      for (DroppedColumn droppedColumn : droppedColumns) {
+        if (droppedColumn.columnPosition >= columnNames.size()) {
+          columnNames.add(droppedColumn.name);
+        } else {
+          columnNames.add(droppedColumn.columnPosition, droppedColumn.name);
+        }
+      }
+    }
+    return columnNames;
+  }
+
+  /**
+   * Return all the columns (excluding columns marked as dropped).
+   */
+  public Collection<MColumn> allColumns() {
+
+    return columns.values();
+  }
+
+  /**
+   * Return the column by name.
+   */
+  public MColumn getColumn(String name) {
+    return columns.get(name);
+  }
+
+  private Map<String, MColumn> getColumns() {
     return columns;
   }
 
@@ -369,26 +409,14 @@ public class MTable {
     return compoundKeys;
   }
 
-  public String getSequenceName() {
-    return sequenceName;
-  }
-
   public void setSequenceName(String sequenceName) {
     this.sequenceName = sequenceName;
   }
 
-  public int getSequenceInitial() {
-    return sequenceInitial;
-  }
-
   public void setSequenceInitial(int sequenceInitial) {
     this.sequenceInitial = sequenceInitial;
   }
 
-  public int getSequenceAllocate() {
-    return sequenceAllocate;
-  }
-
   public void setSequenceAllocate(int sequenceAllocate) {
     this.sequenceAllocate = sequenceAllocate;
   }
@@ -411,22 +439,12 @@ public class MTable {
     this.identityType = identityType;
   }
 
-  /**
-   * Returns the identity type to use for this table.
-   * <p>
-   * If set then this overrides the platform default so for UUID generated values
-   * or DB's supporting both sequences and autoincrement.
-   */
-  public IdentityType getIdentityType() {
-    return identityType;
-  }
-
   /**
    * Return the list of columns that make the primary key.
    */
   public List<MColumn> primaryKeyColumns() {
     List<MColumn> pk = new ArrayList<MColumn>(3);
-    for (MColumn column : columns.values()) {
+    for (MColumn column : allColumns()) {
       if (column.isPrimaryKey()) {
         pk.add(column);
       }
@@ -485,7 +503,7 @@ public class MTable {
    */
   public MColumn addColumn(String dbCol, String columnDefn, boolean notnull) {
 
-    MColumn existingColumn = columns.get(dbCol);
+    MColumn existingColumn = getColumn(dbCol);
     if (existingColumn != null) {
       if (notnull) {
         existingColumn.setNotnull(true);
@@ -519,20 +537,46 @@ public class MTable {
   /**
    * Add a 'drop column' to the diff.
    */
-  private void diffDropColumn(ModelDiff modelDiff, MColumn existingColumn) {
+  private void diffDropColumn(ModelDiff modelDiff, MColumn existingColumn, int columnPosition, MTable newTable) {
 
     DropColumn dropColumn = new DropColumn();
     dropColumn.setTableName(name);
     dropColumn.setColumnName(existingColumn.getName());
-    if (withHistory) {
+    if (withHistory && !existingColumn.isHistoryExclude()) {
       // These dropColumns should occur on the history
       // table as well as the base table
       dropColumn.setWithHistory(Boolean.TRUE);
+      newTable.registerDroppedColumn(existingColumn.getName(), columnPosition);
     }
 
     modelDiff.addDropColumn(dropColumn);
   }
 
+  /**
+   * Register a dropped column with it's previous column position.
+   * We need this for history triggers and views as we don't actually drop the
+   * column until the 'drop script' is run so the 'apply script' for history changes
+   * still needs to include the columns that are going to be dropped.
+   */
+  protected void registerDroppedColumn(String name, int columnPosition) {
+    droppedColumns.add(new DroppedColumn(name, columnPosition));
+  }
+
+  private static class DroppedColumn implements Comparable<DroppedColumn> {
+    final String name;
+    final int columnPosition;
+
+    DroppedColumn(String name, int columnPosition) {
+      this.name = name;
+      this.columnPosition = columnPosition;
+    }
+
+    @Override
+    public int compareTo(DroppedColumn o) {
+      return Integer.compare(o.columnPosition, columnPosition);
+    }
+  }
+
   private int toInt(BigInteger value) {
     return (value == null) ? 0 : value.intValue();
   }
@@ -575,7 +619,7 @@ public class MTable {
    */
   public void adjustReferences(ModelContainer modelContainer) {
 
-    Collection<MColumn> cols = columns.values();
+    Collection<MColumn> cols = allColumns();
     for (MColumn col : cols) {
       String references = col.getReferences();
       if (references != null) {

src/main/java/com/avaje/ebean/dbmigration/model/PlatformDdlWriter.java

@@ -3,6 +3,7 @@ package com.avaje.ebean.dbmigration.model;
 import com.avaje.ebean.config.DbMigrationConfig;
 import com.avaje.ebean.config.ServerConfig;
 import com.avaje.ebean.config.dbplatform.DatabasePlatform;
+import com.avaje.ebean.dbmigration.ddlgeneration.DdlBuffer;
 import com.avaje.ebean.dbmigration.ddlgeneration.DdlHandler;
 import com.avaje.ebean.dbmigration.ddlgeneration.DdlWrite;
 import com.avaje.ebean.dbmigration.migration.ChangeSet;
@@ -116,11 +117,7 @@ public class PlatformDdlWriter {
   protected void writeApplyDdl(Writer writer, DdlWrite write) throws IOException {
 
     // merge the apply buffers in the appropriate order
-    if (!write.applyDropDependencies().isEmpty()) {
-      writer.append("-- drop dependencies\n");
-      writer.append(write.applyDropDependencies().getBuffer());
-      writer.append("\n");
-    }
+    prependDropDependencies(writer, write.applyDropDependencies());
     writer.append("-- apply changes\n");
     writer.append(write.apply().getBuffer());
     writer.append(write.applyForeignKeys().getBuffer());
@@ -133,11 +130,7 @@ public class PlatformDdlWriter {
   protected void writeApplyRollbackDdl(Writer writer, DdlWrite write) throws IOException {
 
     // merge the rollback buffers in the appropriate order
-    if (!write.rollbackDropDependencies().isEmpty()) {
-      writer.append("-- drop dependencies\n");
-      writer.append(write.rollbackDropDependencies().getBuffer());
-      writer.append("\n");
-    }
+    prependDropDependencies(writer, write.rollbackDropDependencies());
     writer.append("-- reverse changes\n");
     writer.append(write.rollbackForeignKeys().getBuffer());
     writer.append(write.rollback().getBuffer());
@@ -149,10 +142,19 @@ public class PlatformDdlWriter {
   protected void writeDropDdl(Writer writer, DdlWrite write) throws IOException {
 
     // merge the rollback buffers in the appropriate order
+    prependDropDependencies(writer, write.dropDropDependencies());
     writer.append(write.dropHistory().getBuffer());
     writer.append(write.drop().getBuffer());
   }
 
+  private void prependDropDependencies(Writer writer, DdlBuffer buffer) throws IOException {
+    if (!buffer.isEmpty()) {
+      writer.append("-- drop dependencies\n");
+      writer.append(buffer.getBuffer());
+      writer.append("\n");
+    }
+  }
+
   /**
    * Return the platform specific DdlHandler (to generate DDL).
    */

src/main/java/com/avaje/ebean/dbmigration/model/build/ModelBuildContext.java

@@ -158,7 +158,7 @@ public class ModelBuildContext {
     int fkCount = 0;
     int ixCount = 0;
     int uqCount = 0;
-    Collection<MColumn> cols = draftTable.getColumns().values();
+    Collection<MColumn> cols = draftTable.allColumns();
     for (MColumn col: cols) {
       if (col.getForeignKeyName() != null) {
         // Note that we adjust the 'references' table later in a second pass

src/main/java/com/avaje/ebean/dbmigration/model/build/ModelBuildPropertyVisitor.java

@@ -84,7 +84,7 @@ public class ModelBuildPropertyVisitor extends BaseTablePropertyVisitor {
     table.setPkName(determinePrimaryKeyName());
 
     // check if indexes on foreign keys should be suppressed
-    for (MColumn column : table.getColumns().values()) {
+    for (MColumn column : table.allColumns()) {
       if (hasValue(column.getForeignKeyIndex())) {
         if (indexSet.contains(column.getName())) {
           // suppress index on foreign key as there is already