diff --git a/src/main/java/io/ebean/config/dbplatform/DatabasePlatform.java b/src/main/java/io/ebean/config/dbplatform/DatabasePlatform.java index ecc211289..ff887d038 100644 --- a/src/main/java/io/ebean/config/dbplatform/DatabasePlatform.java +++ b/src/main/java/io/ebean/config/dbplatform/DatabasePlatform.java @@ -2,6 +2,7 @@ package io.ebean.config.dbplatform; import io.ebean.BackgroundExecutor; import io.ebean.Query; +import io.ebean.annotation.PartitionMode; import io.ebean.annotation.PersistBatch; import io.ebean.annotation.Platform; import io.ebean.config.CustomDbTypeMapping; @@ -706,6 +707,20 @@ public class DatabasePlatform { } } + /** + * Return true if partitions exist for the given table. + */ + public boolean tablePartitionsExist(Connection connection, String table) throws SQLException { + return true; + } + + /** + * Return the SQL to create an initial partition for the given table. + */ + public String tablePartitionInit(String tableName, PartitionMode mode, String property, String singlePrimaryKey) { + return null; + } + /** * Escapes the like string for this DB-Platform */ diff --git a/src/main/java/io/ebean/config/dbplatform/postgres/PostgresPlatform.java b/src/main/java/io/ebean/config/dbplatform/postgres/PostgresPlatform.java index 75a395568..fde6f7af3 100644 --- a/src/main/java/io/ebean/config/dbplatform/postgres/PostgresPlatform.java +++ b/src/main/java/io/ebean/config/dbplatform/postgres/PostgresPlatform.java @@ -2,6 +2,7 @@ package io.ebean.config.dbplatform.postgres; import io.ebean.BackgroundExecutor; import io.ebean.Query; +import io.ebean.annotation.PartitionMode; import io.ebean.annotation.Platform; import io.ebean.config.dbplatform.DatabasePlatform; import io.ebean.config.dbplatform.DbPlatformType; @@ -11,6 +12,10 @@ import io.ebean.config.dbplatform.PlatformIdGenerator; import io.ebean.config.dbplatform.SqlErrorCodes; import javax.sql.DataSource; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; import java.sql.Types; /** @@ -116,4 +121,31 @@ public class PostgresPlatform extends DatabasePlatform { return sql + " for update"; } } + + @Override + public boolean tablePartitionsExist(Connection connection, String table) throws SQLException { + + try (PreparedStatement statement = connection.prepareStatement("select count(*) from pg_inherits i WHERE i.inhparent = ?::regclass")) { + statement.setString(1, table); + try (ResultSet resultSet = statement.executeQuery()) { + return resultSet.next() && resultSet.getInt(1) > 0; + } + } + } + + /** + * Return SQL using built in partition helper functions to create some initial partitions. + * + * Only use this if extra-dll doesn't have some initial partitions defined (which it should). + */ + public String tablePartitionInit(String tableName, PartitionMode mode, String property, String pkey) { + if (property == null) { + property = ""; + } + if (pkey == null) { + pkey = ""; + } + return "select partition('" + mode.name().toLowerCase() + "','" + tableName + "','" + pkey + "','" + property + "',1);"; + } + } diff --git a/src/main/java/io/ebean/dbmigration/DbMigration.java b/src/main/java/io/ebean/dbmigration/DbMigration.java index 582838e79..1a1e0e9f6 100644 --- a/src/main/java/io/ebean/dbmigration/DbMigration.java +++ b/src/main/java/io/ebean/dbmigration/DbMigration.java @@ -98,6 +98,11 @@ public interface DbMigration { */ void setIncludeGeneratedFileComment(boolean includeGeneratedFileComment); + /** + * Set this to false to exclude the builtin support for table partitioning (with @DbPartition). + */ + void setIncludeBuiltInPartitioning(boolean includeBuiltInPartitioning); + /** * Set the header that is included in the generated DDL script. */ diff --git a/src/main/java/io/ebeaninternal/dbmigration/DdlGenerator.java b/src/main/java/io/ebeaninternal/dbmigration/DdlGenerator.java index e8edf79dc..b8271273b 100644 --- a/src/main/java/io/ebeaninternal/dbmigration/DdlGenerator.java +++ b/src/main/java/io/ebeaninternal/dbmigration/DdlGenerator.java @@ -1,11 +1,14 @@ package io.ebeaninternal.dbmigration; import io.ebean.config.ServerConfig; +import io.ebean.config.dbplatform.DatabasePlatform; import io.ebean.migration.ddl.DdlRunner; import io.ebean.util.JdbcClose; import io.ebeaninternal.api.SpiEbeanServer; import io.ebeaninternal.dbmigration.model.CurrentModel; +import io.ebeaninternal.dbmigration.model.MTable; import io.ebeaninternal.extraddl.model.ExtraDdlXmlReader; +import io.ebeaninternal.server.deploy.PartitionMeta; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -180,10 +183,50 @@ public class DdlGenerator { String ignoreExtraDdl = System.getProperty("ebean.ignoreExtraDdl"); if (!"true".equalsIgnoreCase(ignoreExtraDdl) && jaxbPresent) { + if (currentModel.isTablePartitioning()) { + String extraPartitioning = ExtraDdlXmlReader.buildPartitioning(server.getDatabasePlatform().getName()); + if (extraPartitioning != null && !extraPartitioning.isEmpty()) { + runScript(connection, false, extraPartitioning, "builtin-partitioning-dll"); + } + } + String extraApply = ExtraDdlXmlReader.buildExtra(server.getDatabasePlatform().getName(), false); if (extraApply != null) { runScript(connection, false, extraApply, "extra-dll"); } + + if (currentModel.isTablePartitioning()) { + checkInitialTablePartitions(connection); + } + } + } + + /** + * Check if table partitions exist and if not create some. The expectation is that + * extra-dll.xml should have some partition initialisation but this helps people get going. + */ + private void checkInitialTablePartitions(Connection connection) { + + DatabasePlatform databasePlatform = server.getDatabasePlatform(); + try { + StringBuilder sb = new StringBuilder(); + for (MTable table : currentModel.getPartitionedTables()) { + String tableName = table.getName(); + if (!databasePlatform.tablePartitionsExist(connection, tableName)) { + log.info("No table partitions for table {}", tableName); + PartitionMeta meta = table.getPartitionMeta(); + String initPart = databasePlatform.tablePartitionInit(tableName, meta.getMode(), meta.getProperty(), table.singlePrimaryKey()); + sb.append(initPart).append("\n"); + } + } + + String initialPartitionSql = sb.toString(); + if (!initialPartitionSql.isEmpty()) { + runScript(connection, false, initialPartitionSql, "initial table partitions"); + } + + } catch (SQLException e) { + log.error("Error checking initial table partitions", e); } } diff --git a/src/main/java/io/ebeaninternal/dbmigration/DefaultDbMigration.java b/src/main/java/io/ebeaninternal/dbmigration/DefaultDbMigration.java index 68872221a..b73812955 100644 --- a/src/main/java/io/ebeaninternal/dbmigration/DefaultDbMigration.java +++ b/src/main/java/io/ebeaninternal/dbmigration/DefaultDbMigration.java @@ -100,6 +100,8 @@ public class DefaultDbMigration implements DbMigration { protected String name; protected String generatePendingDrop; + protected boolean includeBuiltInPartitioning = true; + /** * Create for offline migration generation. */ @@ -181,6 +183,11 @@ public class DefaultDbMigration implements DbMigration { this.includeGeneratedFileComment = includeGeneratedFileComment; } + @Override + public void setIncludeBuiltInPartitioning(boolean includeBuiltInPartitioning) { + this.includeBuiltInPartitioning = includeBuiltInPartitioning; + } + @Override public void setHeader(String header) { this.header = header; @@ -279,7 +286,7 @@ public class DefaultDbMigration implements DbMigration { try { Request request = createRequest(); if (platforms.isEmpty()) { - generateExtraDdl(request.migrationDir, databasePlatform); + generateExtraDdl(request.migrationDir, databasePlatform, request.isTablePartitioning()); } String pendingVersion = generatePendingDrop(); @@ -314,24 +321,27 @@ public class DefaultDbMigration implements DbMigration { * migration runner. *

*/ - private void generateExtraDdl(File migrationDir, DatabasePlatform dbPlatform) throws IOException { + private void generateExtraDdl(File migrationDir, DatabasePlatform dbPlatform, boolean tablePartitioning) throws IOException { if (dbPlatform != null) { - generateExtraDdl(migrationDir, dbPlatform, ExtraDdlXmlReader.readBuiltin()); - generateExtraDdl(migrationDir, dbPlatform, ExtraDdlXmlReader.read()); + if (tablePartitioning && includeBuiltInPartitioning) { + generateExtraDdlFor(migrationDir, dbPlatform, ExtraDdlXmlReader.readBuiltinTablePartitioning()); + } + generateExtraDdlFor(migrationDir, dbPlatform, ExtraDdlXmlReader.readBuiltin()); + generateExtraDdlFor(migrationDir, dbPlatform, ExtraDdlXmlReader.read()); } } - private void generateExtraDdl(File migrationDir, DatabasePlatform dbPlatform, ExtraDdl extraDdl) throws IOException { + private void generateExtraDdlFor(File migrationDir, DatabasePlatform dbPlatform, ExtraDdl extraDdl) throws IOException { if (extraDdl != null) { List ddlScript = extraDdl.getDdlScript(); for (DdlScript script : ddlScript) { if (!script.isDrop() && ExtraDdlXmlReader.matchPlatform(dbPlatform.getName(), script.getPlatforms())) { - writeExtraDdl(migrationDir, script); - } + writeExtraDdl(migrationDir, script); } } } + } /** * Write (or override) the "repeatable" migration script. @@ -349,8 +359,6 @@ public class DefaultDbMigration implements DbMigration { } } - - private String repeatableMigrationName(boolean init, String scriptName) { StringBuilder sb = new StringBuilder(); if (init) { @@ -421,6 +429,10 @@ public class DefaultDbMigration implements DbMigration { this.current = currentModel.read(); } + boolean isTablePartitioning() { + return current.isTablePartitioning(); + } + /** * Return the migration for the pending drops for a given version. */ @@ -528,7 +540,7 @@ public class DefaultDbMigration implements DbMigration { File subPath = platformWriter.subPath(writePath, pair.prefix); platformWriter.processMigration(dbMigration, platformBuffer, subPath, fullVersion); - generateExtraDdl(subPath, pair.platform); + generateExtraDdl(subPath, pair.platform, currentModel.isTablePartitioning()); } } diff --git a/src/main/java/io/ebeaninternal/dbmigration/model/CurrentModel.java b/src/main/java/io/ebeaninternal/dbmigration/model/CurrentModel.java index 8418cc8d0..1b83beb4c 100644 --- a/src/main/java/io/ebeaninternal/dbmigration/model/CurrentModel.java +++ b/src/main/java/io/ebeaninternal/dbmigration/model/CurrentModel.java @@ -60,6 +60,20 @@ public class CurrentModel { this.platformTypes = platformTypes; } + /** + * Return true if the model contains tables that are partitioned. + */ + public boolean isTablePartitioning() { + return model.isTablePartitioning(); + } + + /** + * Return the tables that have partitioning. + */ + public List getPartitionedTables() { + return model.getPartitionedTables(); + } + private static DbConstraintNaming.MaxLength maxLength(SpiEbeanServer server, DbConstraintNaming naming) { if (naming.getMaxLength() != null) { @@ -116,17 +130,7 @@ public class CurrentModel { ddl.append(header).append('\n'); } - ExtraDdl extraDdl = ExtraDdlXmlReader.readBuiltin(); - if (extraDdl != null) { - List ddlScript = extraDdl.getDdlScript(); - for (DdlScript script : ddlScript) { - if (script.isInit() && ExtraDdlXmlReader.matchPlatform(server.getDatabasePlatform().getName(), script.getPlatforms())) { - ddl.append("-- init script " + script.getName()).append('\n'); - ddl.append(script.getValue()); - } - } - } - + addExtraDdl(ddl, ExtraDdlXmlReader.readBuiltin(), "-- init script "); ddl.append(write.apply().getBuffer()); ddl.append(write.applyForeignKeys().getBuffer()); @@ -136,6 +140,18 @@ public class CurrentModel { return ddl.toString(); } + private void addExtraDdl(StringBuilder ddl, ExtraDdl extraDdl, String prefix) { + if (extraDdl != null) { + List ddlScript = extraDdl.getDdlScript(); + for (DdlScript script : ddlScript) { + if (script.isInit() && ExtraDdlXmlReader.matchPlatform(server.getDatabasePlatform().getName(), script.getPlatforms())) { + ddl.append(prefix + script.getName()).append('\n'); + ddl.append(script.getValue()); + } + } + } + } + /** * Return the 'Drop' DDL. */ diff --git a/src/main/java/io/ebeaninternal/dbmigration/model/MTable.java b/src/main/java/io/ebeaninternal/dbmigration/model/MTable.java index 024071aea..3e3c5fa1a 100644 --- a/src/main/java/io/ebeaninternal/dbmigration/model/MTable.java +++ b/src/main/java/io/ebeaninternal/dbmigration/model/MTable.java @@ -434,6 +434,13 @@ public class MTable { return partitionMeta != null; } + /** + * Return the partition meta for this table. + */ + public PartitionMeta getPartitionMeta() { + return partitionMeta; + } + public void setPkName(String pkName) { this.pkName = pkName; } @@ -546,6 +553,17 @@ public class MTable { return pk; } + /** + * Return the primary key column if it is a simple primary key. + */ + public String singlePrimaryKey() { + List columns = primaryKeyColumns(); + if (columns.size() == 1) { + return columns.get(0).getName(); + } + return null; + } + private void checkTableName(String tableName) { if (!name.equals(tableName)) { throw new IllegalArgumentException("addColumn tableName [" + tableName + "] does not match [" + name + "]"); diff --git a/src/main/java/io/ebeaninternal/dbmigration/model/ModelContainer.java b/src/main/java/io/ebeaninternal/dbmigration/model/ModelContainer.java index 15009f5b1..f2daca21b 100644 --- a/src/main/java/io/ebeaninternal/dbmigration/model/ModelContainer.java +++ b/src/main/java/io/ebeaninternal/dbmigration/model/ModelContainer.java @@ -19,6 +19,7 @@ import io.ebeaninternal.dbmigration.migration.DropTable; import io.ebeaninternal.dbmigration.migration.Migration; import io.ebeaninternal.dbmigration.migration.Sql; +import java.util.ArrayList; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -43,9 +44,25 @@ public class ModelContainer { private final PendingDrops pendingDrops = new PendingDrops(); + private final List partitionedTables = new ArrayList<>(); + public ModelContainer() { } + /** + * Return true if the model contains tables that are partitioned. + */ + public boolean isTablePartitioning() { + return !partitionedTables.isEmpty(); + } + + /** + * Return the list of partitioned tables. + */ + public List getPartitionedTables() { + return partitionedTables; + } + /** * Adjust the FK references on all the draft tables. */ @@ -300,6 +317,9 @@ public class ModelContainer { * Add a table (typically from reading EbeanServer meta data). */ public MTable addTable(MTable table) { + if (table.isPartitioned()) { + partitionedTables.add(table); + } return tables.put(table.getName(), table); } diff --git a/src/main/java/io/ebeaninternal/extraddl/model/ExtraDdlXmlReader.java b/src/main/java/io/ebeaninternal/extraddl/model/ExtraDdlXmlReader.java index 541b725a3..34b7381a5 100644 --- a/src/main/java/io/ebeaninternal/extraddl/model/ExtraDdlXmlReader.java +++ b/src/main/java/io/ebeaninternal/extraddl/model/ExtraDdlXmlReader.java @@ -26,9 +26,22 @@ public class ExtraDdlXmlReader { public static String buildExtra(String platformName, boolean drops) { ExtraDdl read = ExtraDdlXmlReader.read("/extra-ddl.xml"); + return buildExtra(platformName, drops, read); + } + + /** + * Return any extra DDL for supporting partitioning given the database platform. + */ + public static String buildPartitioning(String platformName) { + return buildExtra(platformName, false, readBuiltinTablePartitioning()); + } + + private static String buildExtra(String platformName, boolean drops, ExtraDdl read) { + if (read == null) { return null; } + StringBuilder sb = new StringBuilder(300); for (DdlScript script : read.getDdlScript()) { if (script.isDrop() == drops && matchPlatform(platformName, script.getPlatforms())) { @@ -90,6 +103,13 @@ public class ExtraDdlXmlReader { return read("/io/ebeaninternal/dbmigration/builtin-extra-ddl.xml"); } + /** + * Read the builtin extra ddl to support table partitioning. + */ + public static ExtraDdl readBuiltinTablePartitioning() { + return read("/io/ebeaninternal/dbmigration/builtin-extra-ddl-partitioning.xml"); + } + /** * Read the extra ddl. */ diff --git a/src/main/resources/io/ebeaninternal/dbmigration/builtin-extra-ddl-partitioning.xml b/src/main/resources/io/ebeaninternal/dbmigration/builtin-extra-ddl-partitioning.xml new file mode 100644 index 000000000..237e563cb --- /dev/null +++ b/src/main/resources/io/ebeaninternal/dbmigration/builtin-extra-ddl-partitioning.xml @@ -0,0 +1,242 @@ + + + + +-- partitioning helper functions (UTC based) + +------------------------------------------------------------------------------------ +-- Type: partition_meta +-- +-- Type used to hold common partitioning parameters such as period start and end etc +------------------------------------------------------------------------------------ +do $$ +begin + if not exists (select 1 from pg_type where typname = 'partition_meta') THEN + create type partition_meta as + ( + period_start timestamptz, + period_end timestamptz, + period_name text, + base_name text, + part_name text, + unique_column text, + index_column text + ); + end if; +end$$; + +------------------------------------------------------------------------------------ +-- Function: _partition_create +-- +-- Internal helper method to create a partition given meta data and +-- optional extra function to call (typically to create additional indexes) +------------------------------------------------------------------------------------ +create or replace function _partition_create(meta partition_meta, extra text) + returns text +language plpgsql +set timezone to 'UTC' +as $$ +begin + + execute format('create table if not exists %I partition of %I for values from (''%s'') TO (''%s'')', meta.part_name, meta.base_name, meta.period_start, meta.period_end); + + if (length(meta.unique_column) > 0) then + execute format('create unique index if not exists uq_%I ON %I (%I)', meta.part_name, meta.part_name, meta.unique_column); + end if; + + if (length(meta.index_column) > 0) then + execute format('create index if not exists ix_%I_%s ON %I (%I)', meta.part_name, meta.index_column, meta.part_name, meta.index_column); + end if; + + if (length(extra) > 0) then + execute 'select ' || extra || '($1)' using meta; + end if; + + return meta.part_name; +end; +$$; + + +------------------------------------------------------------------------------------ +-- Function: _partition_meta +-- +-- Internal helper method to create and return meta data used to create a partition. +-- Helps work out start and end periods for day, week, month and year partitions. +------------------------------------------------------------------------------------ +create or replace function _partition_meta( + mode text, + asOf date, + baseName text, + uniqueColumn text, + indexColumn text) + returns partition_meta +language plpgsql +set timezone to 'UTC' +as $$ +declare + partName text; + meta partition_meta; + asOfUtc timestamptz; +begin + asOfUtc = timezone('utc', asOf); + if (mode = 'day') then + asOfUtc = date_trunc('day', asOfUtc); + partName = to_char(asOfUtc, 'YYYY_MM_DD'); + select asOfUtc, asOfUtc + interval '1 days' into meta.period_start, meta.period_end; + + elseif (mode = 'week') then + asOfUtc = date_trunc('week', asOfUtc); + partName = format('%s_w%s', extract(ISOYEAR FROM asOfUtc), extract(WEEK FROM asOfUtc)); + select asOfUtc, asOfUtc + interval '7 days' into meta.period_start, meta.period_end; + + elseif (mode = 'year') then + asOfUtc = date_trunc('year', asOfUtc); + partName = to_char(date_trunc('year', asOfUtc), 'YYYY'); + select asOfUtc, asOfUtc + interval '1 year' into meta.period_start, meta.period_end; + + else + asOfUtc = date_trunc('month', asOfUtc); + partName = to_char(asOfUtc, 'YYYY_MM'); + select asOfUtc, asOfUtc + interval '1 month' into meta.period_start, meta.period_end; + end if; + + select partName, baseName, format('%s_%s', baseName, partName), uniqueColumn, indexColumn + into meta.period_name, meta.base_name, meta.part_name, meta.unique_column, meta.index_column; + + return meta; +end; +$$; + +create or replace function _partition_meta_initdate( + meta partition_meta, + initDate date) + returns partition_meta +language plpgsql +set timezone to 'UTC' +as $$ +begin + meta.period_start = initDate; + return meta; +end; +$$; + + +-- select _partition_over('week', current_date, 4); + +------------------------------------------------------------------------------------ +-- Function: _partition_over +-- +-- Internal helper method to return a set/table of dates to ensure partitions exists for. +-- Typically we want to ensure some future partitions exist and this helps return dates +-- for which we loop to create partitions. +------------------------------------------------------------------------------------ +create or replace function _partition_over( + mode text, + fromDate date default current_date, + _count integer default 0) + returns TABLE(of_date date) +language plpgsql +as $$ +declare + endDate date; +begin + if (mode = 'day') then + endDate = fromDate + (interval '1 day' * _count); + fromDate = fromDate - interval '1 day'; -- allow for timezone + return query select s::date from generate_series(fromDate, endDate, '1 day') s; + + elseif (mode = 'week') then + fromDate = date_trunc('week', fromDate); + endDate = fromDate + (interval '1 week' * _count); + return query select s::date from generate_series(fromDate, endDate, '1 week') s; + + elseif (mode = 'year') then + fromDate = date_trunc('year', fromDate); + endDate = fromDate + (interval '1 year' * _count); + return query select s::date from generate_series(fromDate, endDate, '1 year') s; + + else + fromDate = date_trunc('month', fromDate); + endDate = fromDate + (interval '1 month' * _count); + return query select s::date from generate_series(fromDate, endDate, '1 month') s; + end if; +end; +$$; + + +------------------------------------------------------------------------------------ +-- Function: partition +-- +-- Helper to ensure we create partitions into the future as needed for day, week, month +-- and year based partitioning. Typically we call this periodically (e.g. every day). +-- +-- Examples: +-- +-- select partition('week', 'trip', 'id', 'when_started', 4); +-- select partition('month', 'event', 'id', 'event_timestamp', 1); +-- +------------------------------------------------------------------------------------ +create or replace function partition( + mode text, -- one of 'day','week','month','year' + baseName text, -- base table name + uniqueColumn text, -- optional unique column + indexColumn text, -- optional column to index + partitionCount integer default 0, -- number of additional partitions + fromDate date default current_date, -- date to create first partition for + extra text default '') -- custom function to call per partition + returns text +language plpgsql +set timezone to 'UTC' +as $$ +begin + perform _partition_create(_partition_meta(mode, poDate, baseName, uniqueColumn, indexColumn), extra) + from _partition_over(mode, fromDate, partitionCount) poDate; + return 'done'; +end; +$$; + + +------------------------------------------------------------------------------------ +-- Function: partition_init +-- +-- Similar to partition but allows the first partition to be bigger with an explicit +-- initDate typically to allow back dated rows to go into the initial partition. +-- +-- Examples: +-- +-- select partition_init(date '2001-01-01', 'week', 'event', 'id', 'event_timestamp', 4); +-- +------------------------------------------------------------------------------------ +create or replace function partition_init( + initDate date, -- first partition period start date + mode text, -- one of 'day','week','month','year' + baseName text, -- base table name + uniqueColumn text, -- optional unique column + indexColumn text, -- optional column to index + partitionCount integer default 0, -- number of additional partitions + fromDate date default current_date, -- date to create first partition for + extra text default '') -- custom function to call per partition + returns text +language plpgsql +set timezone to 'UTC' +as $$ +declare + meta partition_meta; +begin + -- override the period start for the first partition + meta = _partition_meta(mode, fromDate, baseName, uniqueColumn, indexColumn); + meta = _partition_meta_initdate(meta, initDate); + perform _partition_create(meta, extra); + + if (partitionCount > 0) then + -- create additional migrations normally + perform _partition_create(_partition_meta(mode, poDate, baseName, uniqueColumn, indexColumn), extra) + from _partition_over(mode, fromDate, partitionCount) poDate; + end if; + + return 'done'; +end; +$$; + + +