From 5b10d0634895f41d5cce4e5f3c2ef90489f45eaa Mon Sep 17 00:00:00 2001 From: Robin Bygrave Date: Thu, 3 Mar 2016 17:46:32 +1300 Subject: [PATCH] #584 - ElasticSearch: Internal changes for config, expressions and JSON conversion --- .../avaje/ebean/config/DocStoreConfig.java | 90 +++++----- .../server/core/DefaultServer.java | 15 +- .../server/deploy/BeanProperty.java | 2 +- .../CaseInsensitiveEqualExpression.java | 12 +- .../expression/ElasticExpressionContext.java | 159 +++++++++++++++--- .../server/expression/LikeExpression.java | 15 +- .../server/querydefn/DefaultOrmQuery.java | 3 +- 7 files changed, 213 insertions(+), 83 deletions(-) diff --git a/src/main/java/com/avaje/ebean/config/DocStoreConfig.java b/src/main/java/com/avaje/ebean/config/DocStoreConfig.java index 3f9532d2b..8bfb2264d 100644 --- a/src/main/java/com/avaje/ebean/config/DocStoreConfig.java +++ b/src/main/java/com/avaje/ebean/config/DocStoreConfig.java @@ -3,40 +3,43 @@ package com.avaje.ebean.config; import com.avaje.ebean.annotation.DocStoreEvent; /** - * Configuration for the Document store (ElasticSearch) integration. + * Configuration for the Document store integration (e.g. ElasticSearch). */ public class DocStoreConfig { /** * True when the Document store integration is active/on. */ - boolean active; + protected boolean active; /** * When true the Document store should drop and re-create any document mapping (like DDL). */ - boolean dropCreate; + protected boolean dropCreate; /** * The URL of the Document store server. For example: http://localhost:9200. */ - String url; + protected String url; /** * The default mode used by indexes. */ - DocStoreEvent persist = DocStoreEvent.UPDATE; + protected DocStoreEvent persist = DocStoreEvent.UPDATE; /** * The default batch size to use for the Bulk API calls. */ - int bulkBatchSize = 1000; - + protected int bulkBatchSize = 1000; /** * Return true if the Document store (ElasticSearch) integration is active. */ public boolean isActive() { + String systemValue = System.getProperty("ebean.docstore.active"); + if (systemValue != null) { + return Boolean.parseBoolean(systemValue); + } return active; } @@ -47,6 +50,27 @@ public class DocStoreConfig { this.active = active; } + /** + * Return the URL to the Document store. + */ + public String getUrl() { + String systemValue = System.getProperty("ebean.docstore.url"); + if (systemValue != null) { + return systemValue; + } + + return url; + } + + /** + * Set the URL to the Document store server. + *

+ * For a local ElasticSearch server this would be: http://localhost:9200 + */ + public void setUrl(String url) { + this.url = url; + } + /** * Return true if the document store should recreate mappings. */ @@ -61,6 +85,24 @@ public class DocStoreConfig { this.dropCreate = dropCreate; } + /** + * Return the default batch size to use for calls to the Bulk API. + */ + public int getBulkBatchSize() { + return bulkBatchSize; + } + + /** + * Set the default batch size to use for calls to the Bulk API. + *

+ * The batch size can be set on a transaction via {@link com.avaje.ebean.Transaction#setDocStoreUpdateBatchSize(int)}. + *

+ */ + public void setBulkBatchSize(int bulkBatchSize) { + this.bulkBatchSize = bulkBatchSize; + } + + /** * Return the default behavior for when Insert, Update and Delete events occur on beans that have an associated * Document store. @@ -91,40 +133,6 @@ public class DocStoreConfig { this.persist = persist; } - /** - * Return the URL to the Document store. - */ - public String getUrl() { - return url; - } - - /** - * Set the URL to the Document store server. - * - * For a local ElasticSearch server this would be: http://localhost:9200 - */ - public void setUrl(String url) { - this.url = url; - } - - - /** - * Return the default batch size to use for calls to the Bulk API. - */ - public int getBulkBatchSize() { - return bulkBatchSize; - } - - /** - * Set the default batch size to use for calls to the Bulk API. - *

- * The batch size can be set on a transaction via {@link com.avaje.ebean.Transaction#setDocStoreUpdateBatchSize(int)}. - *

- */ - public void setBulkBatchSize(int bulkBatchSize) { - this.bulkBatchSize = bulkBatchSize; - } - /** * Load settings specified in properties files. */ diff --git a/src/main/java/com/avaje/ebeaninternal/server/core/DefaultServer.java b/src/main/java/com/avaje/ebeaninternal/server/core/DefaultServer.java index c4f35fd77..f2599708f 100644 --- a/src/main/java/com/avaje/ebeaninternal/server/core/DefaultServer.java +++ b/src/main/java/com/avaje/ebeaninternal/server/core/DefaultServer.java @@ -1129,7 +1129,6 @@ public final class DefaultServer implements SpiServer, SpiEbeanServer { SpiQuery spiQuery = (SpiQuery) query; spiQuery.setType(Type.BEAN); - if (SpiQuery.Mode.NORMAL.equals(spiQuery.getMode()) && !spiQuery.isLoadBeanCache()) { // See if we can skip doing the fetch completely by getting the bean from the // persistence context or the bean cache @@ -1140,6 +1139,9 @@ public final class DefaultServer implements SpiServer, SpiEbeanServer { } SpiOrmQueryRequest request = createQueryRequest(spiQuery, t); + if (request.isUseDocStore()) { + return docStore().getById(query.getBeanType(), query.getId()); + } try { request.initTransIfRequired(); return (T) request.findId(); @@ -1158,9 +1160,10 @@ public final class DefaultServer implements SpiServer, SpiEbeanServer { return findId(query, t); } - BeanDescriptor desc = beanDescriptorManager.getBeanDescriptor(query.getBeanType()); + SpiQuery spiQuery = (SpiQuery) query; + BeanDescriptor desc = spiQuery.getBeanDescriptor(); - T bean = desc.cacheNaturalKeyLookup((SpiQuery) query, (SpiTransaction) t); + T bean = desc.cacheNaturalKeyLookup(spiQuery, (SpiTransaction) t); if (bean != null) { return bean; } @@ -1352,6 +1355,11 @@ public final class DefaultServer implements SpiServer, SpiEbeanServer { SpiOrmQueryRequest request = createQueryRequest(Type.ITERATE, query, t); + if (request.isUseDocStore()) { + docStore().findEach(query, consumer); + return; + } + request.initTransIfRequired(); request.findEach(consumer); // no try finally - findVisit guarantee's cleanup of the transaction if required @@ -1397,7 +1405,6 @@ public final class DefaultServer implements SpiServer, SpiEbeanServer { public List findList(Query query, Transaction t) { SpiOrmQueryRequest request = createQueryRequest(Type.LIST, query, t); - Object result = request.getFromQueryCache(); if (result != null) { return (List) result; diff --git a/src/main/java/com/avaje/ebeaninternal/server/deploy/BeanProperty.java b/src/main/java/com/avaje/ebeaninternal/server/deploy/BeanProperty.java index 9d88e6bdd..b726ed6fd 100644 --- a/src/main/java/com/avaje/ebeaninternal/server/deploy/BeanProperty.java +++ b/src/main/java/com/avaje/ebeaninternal/server/deploy/BeanProperty.java @@ -462,7 +462,7 @@ public class BeanProperty implements ElPropertyValue, Property { public ElPropertyValue buildElPropertyValue(String propName, String remainder, ElPropertyChainBuilder chain, boolean propertyDeploy) { - throw new PersistenceException("Not valid on scalar bean property " + getFullBeanName()); + return null; } /** diff --git a/src/main/java/com/avaje/ebeaninternal/server/expression/CaseInsensitiveEqualExpression.java b/src/main/java/com/avaje/ebeaninternal/server/expression/CaseInsensitiveEqualExpression.java index eca057435..ea2c99f15 100644 --- a/src/main/java/com/avaje/ebeaninternal/server/expression/CaseInsensitiveEqualExpression.java +++ b/src/main/java/com/avaje/ebeaninternal/server/expression/CaseInsensitiveEqualExpression.java @@ -20,17 +20,7 @@ class CaseInsensitiveEqualExpression extends AbstractExpression { @Override public void writeElastic(ElasticExpressionContext context) throws IOException { - - String[] values = value.split(" "); - if (values.length == 1) { - context.writeMatch(propName, value); - } else { - context.writeBoolStart(true); - for (String val : values) { - context.writeMatch(propName, val); - } - context.writeBoolEnd(); - } + context.writeIEqual(propName, value); } @Override diff --git a/src/main/java/com/avaje/ebeaninternal/server/expression/ElasticExpressionContext.java b/src/main/java/com/avaje/ebeaninternal/server/expression/ElasticExpressionContext.java index 0f70fddce..5d5835450 100644 --- a/src/main/java/com/avaje/ebeaninternal/server/expression/ElasticExpressionContext.java +++ b/src/main/java/com/avaje/ebeaninternal/server/expression/ElasticExpressionContext.java @@ -36,6 +36,9 @@ public class ElasticExpressionContext { private String currentNestedPath; + /** + * Construct given the JSON generator and root bean type. + */ public ElasticExpressionContext(JsonGenerator json, BeanType desc) { this.json = json; this.desc = desc; @@ -66,23 +69,38 @@ public class ElasticExpressionContext { /** * Return an associated 'raw' property given the property name. + * This just returns the original propertyName if no 'raw' property is mapped. */ private String rawProperty(String propertyName) { return desc.docStore().rawProperty(propertyName); } + /** + * Start Bool MUST or SHOULD. + * + * If conjunction is true then MUST(and) and if false is SHOULD(or). + */ public void writeBoolStart(boolean conjunction) throws IOException { writeBoolStart((conjunction) ? MUST : SHOULD); } + /** + * Start Bool MUST. + */ public void writeBoolMustStart() throws IOException { writeBoolStart(MUST); } + /** + * Start Bool MUST_NOT. + */ public void writeBoolMustNotStart() throws IOException { writeBoolStart(MUST_NOT); } + /** + * Start a Bool expression list with the given type (MUST, MUST_NOT, SHOULD). + */ private void writeBoolStart(String type) throws IOException { endNested(); json.writeStartObject(); @@ -90,17 +108,28 @@ public class ElasticExpressionContext { json.writeArrayFieldStart(type); } + /** + * Write the end of a Bool expression list. + */ public void writeBoolEnd() throws IOException { json.writeEndArray(); json.writeEndObject(); json.writeEndObject(); } + /** + * Write a term expression. + */ public void writeTerm(String propertyName, Object value) throws IOException { - writeRawType(TERM, rawProperty(propertyName), value); + // prepareNested on propertyName and expression uses raw + prepareNestedPath(propertyName); + writeRawExpression(TERM, rawProperty(propertyName), value); } + /** + * Write a range expression with a single value. + */ public void writeRange(String propertyName, String rangeType, Object value) throws IOException { prepareNestedPath(propertyName); @@ -114,8 +143,14 @@ public class ElasticExpressionContext { json.writeEndObject(); } + /** + * Write a range expression with a low and high value. + */ public void writeRange(String propertyName, Op lowOp, Object valueLow, Op highOp, Object valueHigh) throws IOException { + //Property property = desc.getProperty(propertyName); + //property. + prepareNestedPath(propertyName); json.writeStartObject(); json.writeObjectFieldStart(RANGE); @@ -129,6 +164,9 @@ public class ElasticExpressionContext { json.writeEndObject(); } + /** + * Write a terms expression. + */ public void writeTerms(String propertyName, Object[] values) throws IOException { prepareNestedPath(propertyName); @@ -143,7 +181,9 @@ public class ElasticExpressionContext { json.writeEndObject(); } - + /** + * Write an Ids expression. + */ public void writeIds(List idList) throws IOException { endNested(); @@ -158,6 +198,9 @@ public class ElasticExpressionContext { json.writeEndObject(); } + /** + * Write an Id expression. + */ public void writeId(Object value) throws IOException { List ids = new ArrayList(1); @@ -165,33 +208,89 @@ public class ElasticExpressionContext { writeIds(ids); } - public void writeSuffix(String propertyName, String value) { - throw new IllegalArgumentException("Not implemented yet. Could search for a mapped 'reversed' property and do prefix query"); - } - - public void writePrefix(String propertyName, String value) throws IOException { + /** + * Write a prefix expression. + */ + public void writeStartsWith(String propertyName, String value) throws IOException { // use analysed field - prepareNestedPath(propertyName); - writeRawType(PREFIX, propertyName, value); + writeRawWithPrepareNested(PREFIX, propertyName, value.toLowerCase()); } + /** + * Suffix expression not supported yet. + */ + public void writeEndsWith(String propertyName, String value) throws IOException { + // use analysed field + // this will likely be slow - best to avoid if you can + writeWildcard(propertyName, "*" + value.toLowerCase()); + } + + /** + * Write a match expression. + */ + public void writeContains(String propertyName, String value) throws IOException { + // use analysed field + writeWildcard(propertyName, "*" + value.toLowerCase() + "*"); + } + + /** + * Write a wildcard expression. + */ + public void writeLike(String propertyName, String value) throws IOException { + // use analysed field + String val = value.toLowerCase(); + // replace SQL wildcard characters with ElasticSearch ones + val = val.replace('_', '?'); + val = val.replace('%', '*'); + writeRawWithPrepareNested(WILDCARD, propertyName, val); + } + + /** + * Write case-insensitive equal to. + */ + public void writeIEqual(String propName, String value) throws IOException { + + String[] values = value.toLowerCase().split(" "); + if (values.length == 1) { + writeMatch(propName, value); + } else { + // Boolean AND all the terms together + writeBoolStart(true); + for (String val : values) { + writeMatch(propName, val); + } + writeBoolEnd(); + } + } + + /** + * Write a prefix expression. + */ public void writeMatch(String propertyName, String value) throws IOException { // use analysed field - prepareNestedPath(propertyName); - writeRawType(MATCH, propertyName, value); + writeRawWithPrepareNested(MATCH, propertyName, value.toLowerCase()); } + /** + * Write a wildcard expression. + */ public void writeWildcard(String propertyName, String value) throws IOException { - prepareNestedPath(propertyName); - writeRawType(WILDCARD, propertyName, value); + writeRawWithPrepareNested(WILDCARD, propertyName, value); } + /** + * Write raw JSON to the query buffer. + */ public void writeRaw(String jsonExpression) throws IOException { json.writeRaw(jsonExpression); } + /** + * Write an exists expression. + */ public void writeExists(boolean notNull, String propertyName) throws IOException { + // prepareNestedPath prior to BoolMustNotStart prepareNestedPath(propertyName); if (!notNull) { writeBoolMustNotStart(); @@ -203,12 +302,23 @@ public class ElasticExpressionContext { } private void writeExists(String propertyName) throws IOException { - writeRawType(EXISTS, FIELD, propertyName); + writeRawExpression(EXISTS, FIELD, propertyName); } - private void writeRawType(String type, String propertyName, Object value) throws IOException { + /** + * Write with prepareNestedPath() on the propertyName + */ + private void writeRawWithPrepareNested(String type, String propertyName, Object value) throws IOException { prepareNestedPath(propertyName); + writeRawExpression(type, propertyName, value); + } + + /** + * Write raw. prepareNestedPath() should already be done. + */ + private void writeRawExpression(String type, String propertyName, Object value) throws IOException { + json.writeStartObject(); json.writeObjectFieldStart(type); json.writeFieldName(propertyName); @@ -217,10 +327,12 @@ public class ElasticExpressionContext { json.writeEndObject(); } - - + /** + * Write an expression for the core operations. + */ public void writeSimple(Op type, String propertyName, Object value) throws IOException { + // prepareNested prior to boolMustNotStart prepareNestedPath(propertyName); switch (type) { case EQ: @@ -243,7 +355,6 @@ public class ElasticExpressionContext { default: writeRange(propertyName, type.docExp(), value); } - } /** @@ -264,6 +375,9 @@ public class ElasticExpressionContext { } } + /** + * Check if we need to start a nested path filter and do so if required. + */ private void prepareNestedPath(String propName) throws IOException { ExpressionPath exprPath = desc.getExpressionPath(propName); if (exprPath != null && exprPath.containsMany()) { @@ -274,6 +388,9 @@ public class ElasticExpressionContext { } } + /** + * Start a nested path filter. + */ private void startNested(String nestedPath) throws IOException { if (currentNestedPath != null) { @@ -281,6 +398,7 @@ public class ElasticExpressionContext { // just add to currentNestedPath return; } else { + // end the prior one as this is different endNested(); } } @@ -292,12 +410,15 @@ public class ElasticExpressionContext { json.writeFieldName("filter"); } + /** + * End a nested path filter if one is still open. + */ private void endNested() throws IOException { if (currentNestedPath != null) { currentNestedPath = null; - //json.writeEndObject(); json.writeEndObject(); json.writeEndObject(); } } + } diff --git a/src/main/java/com/avaje/ebeaninternal/server/expression/LikeExpression.java b/src/main/java/com/avaje/ebeaninternal/server/expression/LikeExpression.java index 907358ad0..f7f4033c2 100644 --- a/src/main/java/com/avaje/ebeaninternal/server/expression/LikeExpression.java +++ b/src/main/java/com/avaje/ebeaninternal/server/expression/LikeExpression.java @@ -28,26 +28,29 @@ class LikeExpression extends AbstractExpression { @Override public void writeElastic(ElasticExpressionContext context) throws IOException { - String paramVal = (caseInsensitive) ? val.toLowerCase() : val; switch (type) { case RAW: - context.writeWildcard(propName, paramVal); + context.writeLike(propName, val); break; case STARTS_WITH: - context.writePrefix(propName, paramVal); + context.writeStartsWith(propName, val); break; case ENDS_WITH: - context.writeSuffix(propName, paramVal); + context.writeEndsWith(propName, val); break; case CONTAINS: - context.writeMatch(propName, paramVal); + context.writeContains(propName, val); break; case EQUAL_TO: - context.writeTerm(propName, paramVal); + if (caseInsensitive) { + context.writeIEqual(propName, val); + } else { + context.writeTerm(propName, val); + } break; default: diff --git a/src/main/java/com/avaje/ebeaninternal/server/querydefn/DefaultOrmQuery.java b/src/main/java/com/avaje/ebeaninternal/server/querydefn/DefaultOrmQuery.java index 3eff78a71..ba11936f8 100644 --- a/src/main/java/com/avaje/ebeaninternal/server/querydefn/DefaultOrmQuery.java +++ b/src/main/java/com/avaje/ebeaninternal/server/querydefn/DefaultOrmQuery.java @@ -303,7 +303,8 @@ public class DefaultOrmQuery implements SpiQuery { try { writeElastic(context); context.flush(); - return sw.toString(); + generatedSql = sw.toString(); + return generatedSql; } catch (IOException e) { throw new PersistenceIOException(e);