Compare commits

...
12 changed files with 464 additions and 27 deletions
+3 -3
View File
@@ -9,7 +9,7 @@
<groupId>io.ebean</groupId>
<artifactId>ebean</artifactId>
<version>11.20.1</version>
<version>11.20.2</version>
<packaging>jar</packaging>
<name>ebean</name>
@@ -22,7 +22,7 @@
<scm>
<developerConnection>scm:git:git@github.com:ebean-orm/ebean.git</developerConnection>
<tag>ebean-11.20.1</tag>
<tag>ebean-11.20.2</tag>
</scm>
<profiles>
@@ -135,7 +135,7 @@
<dependency>
<groupId>io.ebean</groupId>
<artifactId>ebean-migration</artifactId>
<version>11.8.1</version>
<version>11.8.2</version>
</dependency>
<dependency>
@@ -107,6 +107,11 @@ public class DbMigrationConfig {
*/
protected String dbSchema;
/**
* Set to true if we consider this the 'default schema' (Postgres schema that matches DB username)
*/
protected boolean defaultDbSchema;
/**
* DB user used to run the DB migration.
*/
@@ -400,9 +405,15 @@ public class DbMigrationConfig {
* Set the Db schema if it hasn't already been defined.
*/
public void setDefaultDbSchema(String dbSchema) {
if (this.dbSchema == null) {
this.dbSchema = dbSchema;
}
this.defaultDbSchema = true;
this.dbSchema = dbSchema;
}
/**
* Return true if this is considered the default DB schema (Postgres schema matching DB username).
*/
public boolean isDefaultDbSchema() {
return defaultDbSchema;
}
/**
@@ -591,6 +602,9 @@ public class DbMigrationConfig {
runnerConfig.setDbUsername(getDbUsername());
runnerConfig.setDbPassword(getDbPassword());
runnerConfig.setDbSchema(getDbSchema());
if (defaultDbSchema) {
runnerConfig.setSetCurrentSchema(false);
}
runnerConfig.setClassLoader(classLoader);
if (patchInsertOn != null) {
runnerConfig.setPatchInsertOn(patchInsertOn);
@@ -2,6 +2,7 @@ package io.ebean.config.dbplatform;
import io.ebean.BackgroundExecutor;
import io.ebean.Query;
import io.ebean.annotation.PartitionMode;
import io.ebean.annotation.PersistBatch;
import io.ebean.annotation.Platform;
import io.ebean.config.CustomDbTypeMapping;
@@ -706,6 +707,20 @@ public class DatabasePlatform {
}
}
/**
* Return true if partitions exist for the given table.
*/
public boolean tablePartitionsExist(Connection connection, String table) throws SQLException {
return true;
}
/**
* Return the SQL to create an initial partition for the given table.
*/
public String tablePartitionInit(String tableName, PartitionMode mode, String property, String singlePrimaryKey) {
return null;
}
/**
* Escapes the like string for this DB-Platform
*/
@@ -2,6 +2,7 @@ package io.ebean.config.dbplatform.postgres;
import io.ebean.BackgroundExecutor;
import io.ebean.Query;
import io.ebean.annotation.PartitionMode;
import io.ebean.annotation.Platform;
import io.ebean.config.dbplatform.DatabasePlatform;
import io.ebean.config.dbplatform.DbPlatformType;
@@ -11,6 +12,10 @@ import io.ebean.config.dbplatform.PlatformIdGenerator;
import io.ebean.config.dbplatform.SqlErrorCodes;
import javax.sql.DataSource;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Types;
/**
@@ -116,4 +121,31 @@ public class PostgresPlatform extends DatabasePlatform {
return sql + " for update";
}
}
@Override
public boolean tablePartitionsExist(Connection connection, String table) throws SQLException {
try (PreparedStatement statement = connection.prepareStatement("select count(*) from pg_inherits i WHERE i.inhparent = ?::regclass")) {
statement.setString(1, table);
try (ResultSet resultSet = statement.executeQuery()) {
return resultSet.next() && resultSet.getInt(1) > 0;
}
}
}
/**
* Return SQL using built in partition helper functions to create some initial partitions.
*
* Only use this if extra-dll doesn't have some initial partitions defined (which it should).
*/
public String tablePartitionInit(String tableName, PartitionMode mode, String property, String pkey) {
if (property == null) {
property = "";
}
if (pkey == null) {
pkey = "";
}
return "select partition('" + mode.name().toLowerCase() + "','" + tableName + "','" + pkey + "','" + property + "',1);";
}
}
@@ -98,6 +98,11 @@ public interface DbMigration {
*/
void setIncludeGeneratedFileComment(boolean includeGeneratedFileComment);
/**
* Set this to false to exclude the builtin support for table partitioning (with @DbPartition).
*/
void setIncludeBuiltInPartitioning(boolean includeBuiltInPartitioning);
/**
* Set the header that is included in the generated DDL script.
*/
@@ -1,11 +1,14 @@
package io.ebeaninternal.dbmigration;
import io.ebean.config.ServerConfig;
import io.ebean.config.dbplatform.DatabasePlatform;
import io.ebean.migration.ddl.DdlRunner;
import io.ebean.util.JdbcClose;
import io.ebeaninternal.api.SpiEbeanServer;
import io.ebeaninternal.dbmigration.model.CurrentModel;
import io.ebeaninternal.dbmigration.model.MTable;
import io.ebeaninternal.extraddl.model.ExtraDdlXmlReader;
import io.ebeaninternal.server.deploy.PartitionMeta;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -180,10 +183,50 @@ public class DdlGenerator {
String ignoreExtraDdl = System.getProperty("ebean.ignoreExtraDdl");
if (!"true".equalsIgnoreCase(ignoreExtraDdl) && jaxbPresent) {
if (currentModel.isTablePartitioning()) {
String extraPartitioning = ExtraDdlXmlReader.buildPartitioning(server.getDatabasePlatform().getName());
if (extraPartitioning != null && !extraPartitioning.isEmpty()) {
runScript(connection, false, extraPartitioning, "builtin-partitioning-dll");
}
}
String extraApply = ExtraDdlXmlReader.buildExtra(server.getDatabasePlatform().getName(), false);
if (extraApply != null) {
runScript(connection, false, extraApply, "extra-dll");
}
if (currentModel.isTablePartitioning()) {
checkInitialTablePartitions(connection);
}
}
}
/**
* Check if table partitions exist and if not create some. The expectation is that
* extra-dll.xml should have some partition initialisation but this helps people get going.
*/
private void checkInitialTablePartitions(Connection connection) {
DatabasePlatform databasePlatform = server.getDatabasePlatform();
try {
StringBuilder sb = new StringBuilder();
for (MTable table : currentModel.getPartitionedTables()) {
String tableName = table.getName();
if (!databasePlatform.tablePartitionsExist(connection, tableName)) {
log.info("No table partitions for table {}", tableName);
PartitionMeta meta = table.getPartitionMeta();
String initPart = databasePlatform.tablePartitionInit(tableName, meta.getMode(), meta.getProperty(), table.singlePrimaryKey());
sb.append(initPart).append("\n");
}
}
String initialPartitionSql = sb.toString();
if (!initialPartitionSql.isEmpty()) {
runScript(connection, false, initialPartitionSql, "initial table partitions");
}
} catch (SQLException e) {
log.error("Error checking initial table partitions", e);
}
}
@@ -100,6 +100,8 @@ public class DefaultDbMigration implements DbMigration {
protected String name;
protected String generatePendingDrop;
protected boolean includeBuiltInPartitioning = true;
/**
* Create for offline migration generation.
*/
@@ -181,6 +183,11 @@ public class DefaultDbMigration implements DbMigration {
this.includeGeneratedFileComment = includeGeneratedFileComment;
}
@Override
public void setIncludeBuiltInPartitioning(boolean includeBuiltInPartitioning) {
this.includeBuiltInPartitioning = includeBuiltInPartitioning;
}
@Override
public void setHeader(String header) {
this.header = header;
@@ -279,7 +286,7 @@ public class DefaultDbMigration implements DbMigration {
try {
Request request = createRequest();
if (platforms.isEmpty()) {
generateExtraDdl(request.migrationDir, databasePlatform);
generateExtraDdl(request.migrationDir, databasePlatform, request.isTablePartitioning());
}
String pendingVersion = generatePendingDrop();
@@ -314,24 +321,27 @@ public class DefaultDbMigration implements DbMigration {
* migration runner.
* </p>
*/
private void generateExtraDdl(File migrationDir, DatabasePlatform dbPlatform) throws IOException {
private void generateExtraDdl(File migrationDir, DatabasePlatform dbPlatform, boolean tablePartitioning) throws IOException {
if (dbPlatform != null) {
generateExtraDdl(migrationDir, dbPlatform, ExtraDdlXmlReader.readBuiltin());
generateExtraDdl(migrationDir, dbPlatform, ExtraDdlXmlReader.read());
if (tablePartitioning && includeBuiltInPartitioning) {
generateExtraDdlFor(migrationDir, dbPlatform, ExtraDdlXmlReader.readBuiltinTablePartitioning());
}
generateExtraDdlFor(migrationDir, dbPlatform, ExtraDdlXmlReader.readBuiltin());
generateExtraDdlFor(migrationDir, dbPlatform, ExtraDdlXmlReader.read());
}
}
private void generateExtraDdl(File migrationDir, DatabasePlatform dbPlatform, ExtraDdl extraDdl) throws IOException {
private void generateExtraDdlFor(File migrationDir, DatabasePlatform dbPlatform, ExtraDdl extraDdl) throws IOException {
if (extraDdl != null) {
List<DdlScript> ddlScript = extraDdl.getDdlScript();
for (DdlScript script : ddlScript) {
if (!script.isDrop() && ExtraDdlXmlReader.matchPlatform(dbPlatform.getName(), script.getPlatforms())) {
writeExtraDdl(migrationDir, script);
}
writeExtraDdl(migrationDir, script);
}
}
}
}
/**
* Write (or override) the "repeatable" migration script.
@@ -349,8 +359,6 @@ public class DefaultDbMigration implements DbMigration {
}
}
private String repeatableMigrationName(boolean init, String scriptName) {
StringBuilder sb = new StringBuilder();
if (init) {
@@ -421,6 +429,10 @@ public class DefaultDbMigration implements DbMigration {
this.current = currentModel.read();
}
boolean isTablePartitioning() {
return current.isTablePartitioning();
}
/**
* Return the migration for the pending drops for a given version.
*/
@@ -528,7 +540,7 @@ public class DefaultDbMigration implements DbMigration {
File subPath = platformWriter.subPath(writePath, pair.prefix);
platformWriter.processMigration(dbMigration, platformBuffer, subPath, fullVersion);
generateExtraDdl(subPath, pair.platform);
generateExtraDdl(subPath, pair.platform, currentModel.isTablePartitioning());
}
}
@@ -60,6 +60,20 @@ public class CurrentModel {
this.platformTypes = platformTypes;
}
/**
* Return true if the model contains tables that are partitioned.
*/
public boolean isTablePartitioning() {
return model.isTablePartitioning();
}
/**
* Return the tables that have partitioning.
*/
public List<MTable> getPartitionedTables() {
return model.getPartitionedTables();
}
private static DbConstraintNaming.MaxLength maxLength(SpiEbeanServer server, DbConstraintNaming naming) {
if (naming.getMaxLength() != null) {
@@ -116,17 +130,7 @@ public class CurrentModel {
ddl.append(header).append('\n');
}
ExtraDdl extraDdl = ExtraDdlXmlReader.readBuiltin();
if (extraDdl != null) {
List<DdlScript> ddlScript = extraDdl.getDdlScript();
for (DdlScript script : ddlScript) {
if (script.isInit() && ExtraDdlXmlReader.matchPlatform(server.getDatabasePlatform().getName(), script.getPlatforms())) {
ddl.append("-- init script " + script.getName()).append('\n');
ddl.append(script.getValue());
}
}
}
addExtraDdl(ddl, ExtraDdlXmlReader.readBuiltin(), "-- init script ");
ddl.append(write.apply().getBuffer());
ddl.append(write.applyForeignKeys().getBuffer());
@@ -136,6 +140,18 @@ public class CurrentModel {
return ddl.toString();
}
private void addExtraDdl(StringBuilder ddl, ExtraDdl extraDdl, String prefix) {
if (extraDdl != null) {
List<DdlScript> ddlScript = extraDdl.getDdlScript();
for (DdlScript script : ddlScript) {
if (script.isInit() && ExtraDdlXmlReader.matchPlatform(server.getDatabasePlatform().getName(), script.getPlatforms())) {
ddl.append(prefix + script.getName()).append('\n');
ddl.append(script.getValue());
}
}
}
}
/**
* Return the 'Drop' DDL.
*/
@@ -434,6 +434,13 @@ public class MTable {
return partitionMeta != null;
}
/**
* Return the partition meta for this table.
*/
public PartitionMeta getPartitionMeta() {
return partitionMeta;
}
public void setPkName(String pkName) {
this.pkName = pkName;
}
@@ -546,6 +553,17 @@ public class MTable {
return pk;
}
/**
* Return the primary key column if it is a simple primary key.
*/
public String singlePrimaryKey() {
List<MColumn> columns = primaryKeyColumns();
if (columns.size() == 1) {
return columns.get(0).getName();
}
return null;
}
private void checkTableName(String tableName) {
if (!name.equals(tableName)) {
throw new IllegalArgumentException("addColumn tableName [" + tableName + "] does not match [" + name + "]");
@@ -19,6 +19,7 @@ import io.ebeaninternal.dbmigration.migration.DropTable;
import io.ebeaninternal.dbmigration.migration.Migration;
import io.ebeaninternal.dbmigration.migration.Sql;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
@@ -43,9 +44,25 @@ public class ModelContainer {
private final PendingDrops pendingDrops = new PendingDrops();
private final List<MTable> partitionedTables = new ArrayList<>();
public ModelContainer() {
}
/**
* Return true if the model contains tables that are partitioned.
*/
public boolean isTablePartitioning() {
return !partitionedTables.isEmpty();
}
/**
* Return the list of partitioned tables.
*/
public List<MTable> getPartitionedTables() {
return partitionedTables;
}
/**
* Adjust the FK references on all the draft tables.
*/
@@ -300,6 +317,9 @@ public class ModelContainer {
* Add a table (typically from reading EbeanServer meta data).
*/
public MTable addTable(MTable table) {
if (table.isPartitioned()) {
partitionedTables.add(table);
}
return tables.put(table.getName(), table);
}
@@ -26,9 +26,22 @@ public class ExtraDdlXmlReader {
public static String buildExtra(String platformName, boolean drops) {
ExtraDdl read = ExtraDdlXmlReader.read("/extra-ddl.xml");
return buildExtra(platformName, drops, read);
}
/**
* Return any extra DDL for supporting partitioning given the database platform.
*/
public static String buildPartitioning(String platformName) {
return buildExtra(platformName, false, readBuiltinTablePartitioning());
}
private static String buildExtra(String platformName, boolean drops, ExtraDdl read) {
if (read == null) {
return null;
}
StringBuilder sb = new StringBuilder(300);
for (DdlScript script : read.getDdlScript()) {
if (script.isDrop() == drops && matchPlatform(platformName, script.getPlatforms())) {
@@ -90,6 +103,13 @@ public class ExtraDdlXmlReader {
return read("/io/ebeaninternal/dbmigration/builtin-extra-ddl.xml");
}
/**
* Read the builtin extra ddl to support table partitioning.
*/
public static ExtraDdl readBuiltinTablePartitioning() {
return read("/io/ebeaninternal/dbmigration/builtin-extra-ddl-partitioning.xml");
}
/**
* Read the extra ddl.
*/
@@ -0,0 +1,242 @@
<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
<extra-ddl xmlns="http://ebean-orm.github.io/xml/ns/extraddl">
<ddl-script name="partition help" init="true" platforms="postgres">
-- partitioning helper functions (UTC based)
------------------------------------------------------------------------------------
-- Type: partition_meta
--
-- Type used to hold common partitioning parameters such as period start and end etc
------------------------------------------------------------------------------------
do $$
begin
if not exists (select 1 from pg_type where typname = 'partition_meta') THEN
create type partition_meta as
(
period_start timestamptz,
period_end timestamptz,
period_name text,
base_name text,
part_name text,
unique_column text,
index_column text
);
end if;
end$$;
------------------------------------------------------------------------------------
-- Function: _partition_create
--
-- Internal helper method to create a partition given meta data and
-- optional extra function to call (typically to create additional indexes)
------------------------------------------------------------------------------------
create or replace function _partition_create(meta partition_meta, extra text)
returns text
language plpgsql
set timezone to 'UTC'
as $$
begin
execute format('create table if not exists %I partition of %I for values from (''%s'') TO (''%s'')', meta.part_name, meta.base_name, meta.period_start, meta.period_end);
if (length(meta.unique_column) > 0) then
execute format('create unique index if not exists uq_%I ON %I (%I)', meta.part_name, meta.part_name, meta.unique_column);
end if;
if (length(meta.index_column) > 0) then
execute format('create index if not exists ix_%I_%s ON %I (%I)', meta.part_name, meta.index_column, meta.part_name, meta.index_column);
end if;
if (length(extra) > 0) then
execute 'select ' || extra || '($1)' using meta;
end if;
return meta.part_name;
end;
$$;
------------------------------------------------------------------------------------
-- Function: _partition_meta
--
-- Internal helper method to create and return meta data used to create a partition.
-- Helps work out start and end periods for day, week, month and year partitions.
------------------------------------------------------------------------------------
create or replace function _partition_meta(
mode text,
asOf date,
baseName text,
uniqueColumn text,
indexColumn text)
returns partition_meta
language plpgsql
set timezone to 'UTC'
as $$
declare
partName text;
meta partition_meta;
asOfUtc timestamptz;
begin
asOfUtc = timezone('utc', asOf);
if (mode = 'day') then
asOfUtc = date_trunc('day', asOfUtc);
partName = to_char(asOfUtc, 'YYYY_MM_DD');
select asOfUtc, asOfUtc + interval '1 days' into meta.period_start, meta.period_end;
elseif (mode = 'week') then
asOfUtc = date_trunc('week', asOfUtc);
partName = format('%s_w%s', extract(ISOYEAR FROM asOfUtc), extract(WEEK FROM asOfUtc));
select asOfUtc, asOfUtc + interval '7 days' into meta.period_start, meta.period_end;
elseif (mode = 'year') then
asOfUtc = date_trunc('year', asOfUtc);
partName = to_char(date_trunc('year', asOfUtc), 'YYYY');
select asOfUtc, asOfUtc + interval '1 year' into meta.period_start, meta.period_end;
else
asOfUtc = date_trunc('month', asOfUtc);
partName = to_char(asOfUtc, 'YYYY_MM');
select asOfUtc, asOfUtc + interval '1 month' into meta.period_start, meta.period_end;
end if;
select partName, baseName, format('%s_%s', baseName, partName), uniqueColumn, indexColumn
into meta.period_name, meta.base_name, meta.part_name, meta.unique_column, meta.index_column;
return meta;
end;
$$;
create or replace function _partition_meta_initdate(
meta partition_meta,
initDate date)
returns partition_meta
language plpgsql
set timezone to 'UTC'
as $$
begin
meta.period_start = initDate;
return meta;
end;
$$;
-- select _partition_over('week', current_date, 4);
------------------------------------------------------------------------------------
-- Function: _partition_over
--
-- Internal helper method to return a set/table of dates to ensure partitions exists for.
-- Typically we want to ensure some future partitions exist and this helps return dates
-- for which we loop to create partitions.
------------------------------------------------------------------------------------
create or replace function _partition_over(
mode text,
fromDate date default current_date,
_count integer default 0)
returns TABLE(of_date date)
language plpgsql
as $$
declare
endDate date;
begin
if (mode = 'day') then
endDate = fromDate + (interval '1 day' * _count);
fromDate = fromDate - interval '1 day'; -- allow for timezone
return query select s::date from generate_series(fromDate, endDate, '1 day') s;
elseif (mode = 'week') then
fromDate = date_trunc('week', fromDate);
endDate = fromDate + (interval '1 week' * _count);
return query select s::date from generate_series(fromDate, endDate, '1 week') s;
elseif (mode = 'year') then
fromDate = date_trunc('year', fromDate);
endDate = fromDate + (interval '1 year' * _count);
return query select s::date from generate_series(fromDate, endDate, '1 year') s;
else
fromDate = date_trunc('month', fromDate);
endDate = fromDate + (interval '1 month' * _count);
return query select s::date from generate_series(fromDate, endDate, '1 month') s;
end if;
end;
$$;
------------------------------------------------------------------------------------
-- Function: partition
--
-- Helper to ensure we create partitions into the future as needed for day, week, month
-- and year based partitioning. Typically we call this periodically (e.g. every day).
--
-- Examples:
--
-- select partition('week', 'trip', 'id', 'when_started', 4);
-- select partition('month', 'event', 'id', 'event_timestamp', 1);
--
------------------------------------------------------------------------------------
create or replace function partition(
mode text, -- one of 'day','week','month','year'
baseName text, -- base table name
uniqueColumn text, -- optional unique column
indexColumn text, -- optional column to index
partitionCount integer default 0, -- number of additional partitions
fromDate date default current_date, -- date to create first partition for
extra text default '') -- custom function to call per partition
returns text
language plpgsql
set timezone to 'UTC'
as $$
begin
perform _partition_create(_partition_meta(mode, poDate, baseName, uniqueColumn, indexColumn), extra)
from _partition_over(mode, fromDate, partitionCount) poDate;
return 'done';
end;
$$;
------------------------------------------------------------------------------------
-- Function: partition_init
--
-- Similar to partition but allows the first partition to be bigger with an explicit
-- initDate typically to allow back dated rows to go into the initial partition.
--
-- Examples:
--
-- select partition_init(date '2001-01-01', 'week', 'event', 'id', 'event_timestamp', 4);
--
------------------------------------------------------------------------------------
create or replace function partition_init(
initDate date, -- first partition period start date
mode text, -- one of 'day','week','month','year'
baseName text, -- base table name
uniqueColumn text, -- optional unique column
indexColumn text, -- optional column to index
partitionCount integer default 0, -- number of additional partitions
fromDate date default current_date, -- date to create first partition for
extra text default '') -- custom function to call per partition
returns text
language plpgsql
set timezone to 'UTC'
as $$
declare
meta partition_meta;
begin
-- override the period start for the first partition
meta = _partition_meta(mode, fromDate, baseName, uniqueColumn, indexColumn);
meta = _partition_meta_initdate(meta, initDate);
perform _partition_create(meta, extra);
if (partitionCount > 0) then
-- create additional migrations normally
perform _partition_create(_partition_meta(mode, poDate, baseName, uniqueColumn, indexColumn), extra)
from _partition_over(mode, fromDate, partitionCount) poDate;
end if;
return 'done';
end;
$$;
</ddl-script>
</extra-ddl>