From d0020ab8ce2e5094f7f4dada24648a1730ac7bc3 Mon Sep 17 00:00:00 2001 From: Roland Praml Date: Tue, 20 Jun 2023 14:47:20 +0200 Subject: [PATCH] Length validation less invasive --- .../java/io/ebean/DataIntegrityException.java | 9 +- .../java/io/ebean/config/DatabaseConfig.java | 16 +++ .../java/io/ebean/config/LengthCheck.java | 22 ++++ .../ebeaninternal/server/bind/DataBind.java | 8 +- .../server/deploy/BeanProperty.java | 58 ++++++++++ .../test/java/org/tests/basic/TestLength.java | 101 ++++++++++++++++++ .../org/tests/model/types/SomeFileBean.java | 6 +- ebean-test/testconfig/ebean-db2.properties | 1 + ebean-test/testconfig/ebean-oracle.properties | 1 + .../testconfig/ebean-sqlserver19.properties | 1 + 10 files changed, 212 insertions(+), 11 deletions(-) create mode 100644 ebean-api/src/main/java/io/ebean/config/LengthCheck.java create mode 100644 ebean-test/src/test/java/org/tests/basic/TestLength.java diff --git a/ebean-api/src/main/java/io/ebean/DataIntegrityException.java b/ebean-api/src/main/java/io/ebean/DataIntegrityException.java index 0d622313c..bc1bc9de0 100644 --- a/ebean-api/src/main/java/io/ebean/DataIntegrityException.java +++ b/ebean-api/src/main/java/io/ebean/DataIntegrityException.java @@ -3,7 +3,7 @@ package io.ebean; import javax.persistence.PersistenceException; /** - * Thrown when a foreign key constraint is enforced. + * Thrown when a foreign key constraint is enforced or a field is too large. */ public class DataIntegrityException extends PersistenceException { private static final long serialVersionUID = -6740171949170180970L; @@ -14,4 +14,11 @@ public class DataIntegrityException extends PersistenceException { public DataIntegrityException(String message, Throwable cause) { super(message, cause); } + + /** + * Create with message only. + */ + public DataIntegrityException(String message) { + super(message); + } } diff --git a/ebean-api/src/main/java/io/ebean/config/DatabaseConfig.java b/ebean-api/src/main/java/io/ebean/config/DatabaseConfig.java index 8abaf3d13..4776c1681 100644 --- a/ebean-api/src/main/java/io/ebean/config/DatabaseConfig.java +++ b/ebean-api/src/main/java/io/ebean/config/DatabaseConfig.java @@ -540,6 +540,7 @@ public class DatabaseConfig { private String dumpMetricsOptions; + private LengthCheck lengthCheck = LengthCheck.OFF; private Function metricNaming = MetricNamingMatch.INSTANCE; /** @@ -2919,6 +2920,7 @@ public class DatabaseConfig { jdbcFetchSizeFindEach = p.getInt("jdbcFetchSizeFindEach", jdbcFetchSizeFindEach); jdbcFetchSizeFindList = p.getInt("jdbcFetchSizeFindList", jdbcFetchSizeFindList); databasePlatformName = p.get("databasePlatformName", databasePlatformName); + lengthCheck = p.getEnum(LengthCheck.class, "lengthCheck", lengthCheck); uuidVersion = p.getEnum(UuidVersion.class, "uuidVersion", uuidVersion); uuidStateFile = p.get("uuidStateFile", uuidStateFile); @@ -3419,6 +3421,20 @@ public class DatabaseConfig { this.metricNaming = metricNaming; } + /** + * Returns the length check mode. + */ + public LengthCheck getLengthCheck() { + return lengthCheck; + } + + /** + * Sets the length check mode. + */ + public void setLengthCheck(LengthCheck lengthCheck) { + this.lengthCheck = lengthCheck; + } + public enum UuidVersion { VERSION4, VERSION1, diff --git a/ebean-api/src/main/java/io/ebean/config/LengthCheck.java b/ebean-api/src/main/java/io/ebean/config/LengthCheck.java new file mode 100644 index 000000000..4f4abf887 --- /dev/null +++ b/ebean-api/src/main/java/io/ebean/config/LengthCheck.java @@ -0,0 +1,22 @@ +package io.ebean.config; + +/** + * Defines the length-check mode. + * + * @author Roland Praml, FOCONIS AG + */ +public enum LengthCheck { + /** + * By default, length checking is off. This means, strings/jsons and files are passed to the DB and the DB might or might not check the length. + * The DB has to check the data length. Note this is not possible for certain datatypes (e.g. clob without size) + */ + OFF, + /** + * When enabling length check, ebean validates strings/json strings and files before saving them to DB. + */ + ON, + /** + * Same as "ON", but take the UTF8-bytelength for validation. This may be useful, if you have an UTF8 based charset (default for DB2) + */ + UTF8 +} diff --git a/ebean-core/src/main/java/io/ebeaninternal/server/bind/DataBind.java b/ebean-core/src/main/java/io/ebeaninternal/server/bind/DataBind.java index 371a8cc3a..bb3e79e97 100644 --- a/ebean-core/src/main/java/io/ebeaninternal/server/bind/DataBind.java +++ b/ebean-core/src/main/java/io/ebeaninternal/server/bind/DataBind.java @@ -19,7 +19,6 @@ import static java.lang.System.Logger.Level.WARNING; public class DataBind implements DataBinder { - private static final Object UNBOUND = new Object(); private final DataTimeZone dataTimeZone; private final PreparedStatement pstmt; private final Connection connection; @@ -28,7 +27,7 @@ public class DataBind implements DataBinder { protected int pos; private String json; - private Object lastObject = UNBOUND; + private Object lastObject = null; public DataBind(DataTimeZone dataTimeZone, PreparedStatement pstmt, Connection connection) { this.dataTimeZone = dataTimeZone; @@ -259,10 +258,7 @@ public class DataBind implements DataBinder { @Override public Object popLastObject() { Object ret = lastObject; - lastObject = UNBOUND; - if (ret == UNBOUND) { - throw new IllegalStateException("No object bound"); - } + lastObject = null; return ret; } } diff --git a/ebean-core/src/main/java/io/ebeaninternal/server/deploy/BeanProperty.java b/ebean-core/src/main/java/io/ebeaninternal/server/deploy/BeanProperty.java index 4ad4fa35e..c77e8b67f 100644 --- a/ebean-core/src/main/java/io/ebeaninternal/server/deploy/BeanProperty.java +++ b/ebean-core/src/main/java/io/ebeaninternal/server/deploy/BeanProperty.java @@ -1,16 +1,20 @@ package io.ebeaninternal.server.deploy; import com.fasterxml.jackson.core.JsonToken; +import io.ebean.DataIntegrityException; import io.ebean.ValuePair; import io.ebean.bean.EntityBean; import io.ebean.bean.EntityBeanIntercept; import io.ebean.bean.MutableValueInfo; import io.ebean.bean.PersistenceContext; import io.ebean.config.EncryptKey; +import io.ebean.config.LengthCheck; import io.ebean.config.dbplatform.DbEncryptFunction; import io.ebean.config.dbplatform.DbPlatformType; +import io.ebean.config.dbplatform.ExtraDbTypes; import io.ebean.core.type.DataReader; import io.ebean.core.type.DocPropertyType; +import io.ebean.core.type.InputStreamInfo; import io.ebean.core.type.ScalarType; import io.ebean.plugin.Property; import io.ebean.text.StringParser; @@ -46,6 +50,7 @@ import java.io.DataInput; import java.io.DataOutput; import java.io.IOException; import java.lang.reflect.Field; +import java.nio.charset.StandardCharsets; import java.sql.SQLException; import java.sql.Types; import java.util.List; @@ -554,6 +559,22 @@ public class BeanProperty implements ElPropertyValue, Property, STreeProperty { @SuppressWarnings("unchecked") public void bind(DataBind b, Object value) throws SQLException { scalarType.bind(b, value); + + if (needsLengthCheck()) { + LengthCheck lengthCheck = descriptor().config().getLengthCheck(); + if (lengthCheck != LengthCheck.OFF) { + Object obj = b.popLastObject(); + long l = getLength(obj, lengthCheck == LengthCheck.UTF8); + if (l > dbLength) { + b.closeInputStreams(); + String s = String.valueOf(value); // take original bind value here. + if (s.length() > 100) { + s = s.substring(0, 97) + "..."; + } + throw new DataIntegrityException("Cannot bind value '" + s + "' (effective length=" + l + ") to column '" + dbColumn + "' (length=" + dbLength + ")"); + } + } + } } @SuppressWarnings(value = "unchecked") @@ -565,6 +586,43 @@ public class BeanProperty implements ElPropertyValue, Property, STreeProperty { return scalarType.readData(dataInput); } + /** + * Returns, if this property needs a length check. + */ + boolean needsLengthCheck() { + if (dbLength == 0) { + return false; + } + switch (dbType) { + case Types.VARCHAR: + case Types.BLOB: + case ExtraDbTypes.JSON: + return true; + default: + return false; + } + } + + /** + * Returns the length of obj. Note: for UTF8 strings -1 will be retuned, if the string length is lower than 1/4th of db length + */ + private long getLength(Object obj, boolean utf8) { + if (obj instanceof String) { + String s = (String) obj; + if (utf8) { + return s.length() * 4 <= dbLength ? -1 : s.getBytes(StandardCharsets.UTF_8).length; + } else { + return s.length(); + } + } else if (obj instanceof byte[]) { + return ((byte[]) obj).length; + } else if (obj instanceof InputStreamInfo) { + return ((InputStreamInfo) obj).length(); + } else { + return -1; + } + } + @Override public BeanProperty beanProperty() { return this; diff --git a/ebean-test/src/test/java/org/tests/basic/TestLength.java b/ebean-test/src/test/java/org/tests/basic/TestLength.java new file mode 100644 index 000000000..fa30635f9 --- /dev/null +++ b/ebean-test/src/test/java/org/tests/basic/TestLength.java @@ -0,0 +1,101 @@ +package org.tests.basic; + +import io.ebean.DB; +import io.ebean.DataIntegrityException; +import io.ebean.xtest.BaseTestCase; +import org.junit.jupiter.api.Test; +import org.tests.model.json.EBasicJsonList; +import org.tests.model.json.EBasicJsonMap; +import org.tests.model.types.SomeFileBean; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.util.List; +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** + * @author Roland Praml, FOCONIS AG + */ +public class TestLength extends BaseTestCase { + + @Test + void testFileSize() throws IOException { + File f1 = File.createTempFile("testfile", "tmp"); + byte[] buf = new byte[1024]; + try (FileOutputStream fos = new FileOutputStream(f1)) { + for (int i = 0; i < 100; i++) { + fos.write(buf); + } + } + + SomeFileBean sfb1 = new SomeFileBean(); + sfb1.setContent(f1); + DB.save(sfb1); + + try (FileOutputStream fos = new FileOutputStream(f1)) { + for (int i = 0; i < 101; i++) { + fos.write(buf); + } + } + + SomeFileBean sfb2 = new SomeFileBean(); + sfb2.setContent(f1); + assertThatThrownBy(() -> DB.save(sfb2)).isInstanceOf(DataIntegrityException.class); + } + + + /** + * The property 'EBasicJsonMap.content' is annotated with @DbJson(length=5000). So we assume, that we cannot save Json-objects + * where the serialized form exceed that limit and we would expect an error on save. + * The length check works for platforms like h2, as H2 uses a 'varchar(5000)'. So it is impossible to save such long jsons, + * but it won't work for SqlServer, as here 'nvarchar(max)' is used. No validation happens at DB level and you might get very + * large Json objects in your database. This mostly happens unintentionally (programming error, misconfiguration) + * So they are in the database and they cannot be accessed by ebean any more, because there are new limits in Jackson: + * - Max 5 Meg per string in 2.15.0 + * - Max 20 Meg per string in 2.15.1 + * see https://github.com/FasterXML/jackson-core/issues/1014 + */ + @Test + void testLongString() { + // s is so big, that it could not be deserialized by jackson + String s = new String(new char[20_000_001]).replace('\0', 'x'); + + EBasicJsonMap bean = new EBasicJsonMap(); + bean.setName("b1"); + bean.setContent(Map.of("string", s)); + + assertThatThrownBy(() -> { + // we expect, that we can NOT save the bean, this is ensured by the bind validator. + DB.save(bean); + }).isInstanceOf(DataIntegrityException.class); + + } + + + /** + * Tests the UTF8 validation. + */ + @Test + void testUtf8() { + + String s = new String(new char[40]).replace('\0', '€'); + + EBasicJsonList bean = new EBasicJsonList(); + bean.setName("b1"); + bean.setTags(List.of(s)); + + if (isDb2() || isOracle()) { + // by default, DB2 && oracle uses bytes in varchar, so an '€' symbol needs 3 bytes + assertThatThrownBy(() -> { + // we expect, that we can NOT save the bean, this is ensured by the bind validator. + DB.save(bean); + }).isInstanceOf(DataIntegrityException.class); + } else { + DB.save(bean); + } + } + +} diff --git a/ebean-test/src/test/java/org/tests/model/types/SomeFileBean.java b/ebean-test/src/test/java/org/tests/model/types/SomeFileBean.java index 4b2cc1f5b..9f92086fa 100644 --- a/ebean-test/src/test/java/org/tests/model/types/SomeFileBean.java +++ b/ebean-test/src/test/java/org/tests/model/types/SomeFileBean.java @@ -1,9 +1,6 @@ package org.tests.model.types; -import javax.persistence.Entity; -import javax.persistence.Id; -import javax.persistence.Lob; -import javax.persistence.Version; +import javax.persistence.*; import java.io.File; @Entity @@ -18,6 +15,7 @@ public class SomeFileBean { String name; @Lob + @Column(length = 100 * 1024) // limit to 100kb File content; public Long getId() { diff --git a/ebean-test/testconfig/ebean-db2.properties b/ebean-test/testconfig/ebean-db2.properties index 4ffab60f1..23006ac4b 100644 --- a/ebean-test/testconfig/ebean-db2.properties +++ b/ebean-test/testconfig/ebean-db2.properties @@ -5,3 +5,4 @@ ebean.test.username=admin ebean.test.password=admin datasource.default=db2-11 ebean.db2-11.databasePlatformName=db2luw +ebean.lengthCheck=utf8 diff --git a/ebean-test/testconfig/ebean-oracle.properties b/ebean-test/testconfig/ebean-oracle.properties index 32b3af249..388c2ec82 100644 --- a/ebean-test/testconfig/ebean-oracle.properties +++ b/ebean-test/testconfig/ebean-oracle.properties @@ -1,3 +1,4 @@ ebean.test.platform=oracle ebean.test.dbName=test_eb datasource.default=oracle +ebean.lengthCheck=utf8 diff --git a/ebean-test/testconfig/ebean-sqlserver19.properties b/ebean-test/testconfig/ebean-sqlserver19.properties index 80ffe7320..478bff787 100644 --- a/ebean-test/testconfig/ebean-sqlserver19.properties +++ b/ebean-test/testconfig/ebean-sqlserver19.properties @@ -7,6 +7,7 @@ ebean.test.sqlserver.port=9434 ebean.test.sqlserver.url=jdbc:sqlserver://localhost:9434;databaseName=test_ebean;sendTimeAsDateTime=false;integratedSecurity=false;trustServerCertificate=true datasource.default=sqlserver2019 ebean.sqlserver2019.databasePlatformName=sqlserver17 +ebean.lengthCheck=on ## A case sensitive collation example: #ebean.test.sqlserver.collation=LATIN1_GENERAL_100_CI_AS_SC_UTF8