#919 - io.ebean package initial

This commit is contained in:
Rob Bygrave
2016-12-11 23:23:22 +13:00
parent 5821dc96b9
commit 971e2dc91b
2134 changed files with 10721 additions and 8652 deletions
@@ -0,0 +1,26 @@
package io.ebeanservice.docstore.api;
import io.ebean.text.json.JsonReadOptions;
import io.ebeaninternal.api.SpiQuery;
/**
* A Query request for the document store.
*/
public interface DocQueryRequest<T> {
/**
* Return the query for this request.
*/
SpiQuery<T> getQuery();
/**
* Create JsonReadOptions taking into account persistence context and lazy loading support.
*/
JsonReadOptions createJsonReadOptions();
/**
* Execute secondary queries.
*/
void executeSecondaryQueries(boolean forEach);
}
@@ -0,0 +1,121 @@
package io.ebeanservice.docstore.api;
import io.ebean.Query;
import io.ebean.annotation.DocStoreMode;
import io.ebean.plugin.BeanDocType;
import io.ebeaninternal.server.core.PersistRequest;
import io.ebeaninternal.server.core.PersistRequestBean;
import io.ebeanservice.docstore.api.mapping.DocumentMapping;
import java.io.IOException;
import java.util.Set;
/**
* Doc store specific adapter to process doc store events for a given bean type.
*/
public interface DocStoreBeanAdapter<T> extends BeanDocType<T> {
/**
* In deployment phase read the embedded/nested document information.
*/
void registerPaths();
/**
* Register invalidation events for embedded/nested documents the given path and properties.
*/
void registerInvalidationPath(String queueId, String path, Set<String> properties);
/**
* Apply the document structure to the query so that it fetches the required properties to build
* the document (typically in JSON form).
*/
void applyPath(Query<T> query);
/**
* Return true if this type is mapped for doc storage.
*/
boolean isMapped();
/**
* Return the unique queueId for this bean type. This is expected to be a relatively short unique
* string (rather than a fully qualified class name).
*/
String getQueueId();
/**
* Determine and return how this persist type will be processed given the transaction mode.
* <p>
* Some transactions (like bulk updates) might specifically turn off indexing for example.
*/
DocStoreMode getMode(PersistRequest.Type persistType, DocStoreMode txnMode);
/**
* Return the index type for this bean type.
*/
String getIndexType();
/**
* Return the index name for this bean type.
*/
String getIndexName();
/**
* Process a delete by id of a given document.
*/
void deleteById(Object idValue, DocStoreUpdateContext txn) throws IOException;
/**
* Process an index event which is effectively an insert or update (or put).
*/
void index(Object idValue, T entityBean, DocStoreUpdateContext txn) throws IOException;
/**
* Process an insert persist request.
*/
void insert(Object idValue, PersistRequestBean<T> persistRequest, DocStoreUpdateContext txn) throws IOException;
/**
* Process an update persist request.
*/
void update(Object idValue, PersistRequestBean<T> persistRequest, DocStoreUpdateContext txn) throws IOException;
/**
* Process the persist request adding any embedded/nested document invalidation to the docStoreUpdates.
* <p>
* This is expected to check the specific properties to see what other documents they are nested in
* and register invalidation events based on that.
*
* @param request The persist request
* @param docStoreUpdates Invalidation events are registered to this docStoreUpdates
*/
void updateEmbedded(PersistRequestBean<T> request, DocStoreUpdates docStoreUpdates);
/**
* Process an update of an embedded document.
*
* @param idValue the id of the bean effected by an embedded document update
* @param embeddedProperty the path of the property
* @param embeddedRawContent the embedded content for this property in JSON form
* @param txn the doc store transaction to use to process the update
*/
void updateEmbedded(Object idValue, String embeddedProperty, String embeddedRawContent, DocStoreUpdateContext txn) throws IOException;
/**
* Create the document mapping.
*/
DocumentMapping createDocMapping();
/**
* Return an un-analysed property to use instead of the given property.
* <p>
* For analysed properties that we want to sort on we will map the property to an additional
* 'raw' property that we can use for sorting etc.
* </p>
*/
String rawProperty(String property);
/**
* Return true if this bean type as embedded invalidate registered.
*/
boolean hasEmbeddedInvalidation();
}
@@ -0,0 +1,22 @@
package io.ebeanservice.docstore.api;
import io.ebean.plugin.SpiServer;
import io.ebeaninternal.server.deploy.BeanDescriptor;
import io.ebeaninternal.server.deploy.meta.DeployBeanDescriptor;
/**
* Creates the integration components for DocStore integration.
*/
public interface DocStoreFactory {
/**
* Create and return the DocStore integration components.
*/
DocStoreIntegration create(SpiServer server);
/**
* Create the doc store specific adapter for the given bean type.
*/
<T> DocStoreBeanAdapter<T> createAdapter(BeanDescriptor<T> desc, DeployBeanDescriptor<T> deploy);
}
@@ -0,0 +1,20 @@
package io.ebeanservice.docstore.api;
import io.ebean.DocumentStore;
/**
* All the required features for DocStore integration.
*/
public interface DocStoreIntegration {
/**
* Return the DocStoreUpdateProcessor to use.
*/
DocStoreUpdateProcessor updateProcessor();
/**
* Return the DocStore.
*/
DocumentStore documentStore();
}
@@ -0,0 +1,23 @@
package io.ebeanservice.docstore.api;
import java.io.IOException;
/**
* Update the document store using a Ebean ORM query.
* <p>
* Executes a forEach query and updates the document store with the bean object graph returned by the query.
* </p>
*/
public interface DocStoreQueryUpdate<T> {
/**
* Process the bean storing in the document store.
*/
void store(Object idValue, T bean) throws IOException;
/**
* Flush the changes to the document store.
*/
void flush() throws IOException;
}
@@ -0,0 +1,29 @@
package io.ebeanservice.docstore.api;
/**
* A document store transaction.
* <p>
* This might just be a buffer to batch persist requests to the document store and may not
* support transactional semantics (like rollback).
*/
public interface DocStoreTransaction {
/**
* Obtain a context to persist to (like a buffer).
*/
DocStoreUpdateContext obtain();
/**
* Add changes that should be queued to the DocStoreUpdates.
* <p>
* This mostly means nested/embedded updates that need to be processed after the source
* persist event has propagated.
* </p>
*/
DocStoreUpdates queue();
/**
* Flush all changes to the document store.
*/
void flush();
}
@@ -0,0 +1,19 @@
package io.ebeanservice.docstore.api;
import java.io.IOException;
/**
* For persist events that know how to publish or queue their change to the Document store.
*/
public interface DocStoreUpdate {
/**
* Add the event to the doc store bulk update.
*/
void docStoreUpdate(DocStoreUpdateContext txn) throws IOException;
/**
* Add to the queue for deferred processing.
*/
void addToQueue(DocStoreUpdates docStoreUpdates);
}
@@ -0,0 +1,11 @@
package io.ebeanservice.docstore.api;
/**
* The doc store specific context/transaction used to collect updates to send to the document store.
* <p>
* Doc store specific implementations gather changes and bulk update the document store.
* </p>
*/
public interface DocStoreUpdateContext {
}
@@ -0,0 +1,46 @@
package io.ebeanservice.docstore.api;
import io.ebean.Transaction;
import io.ebean.plugin.BeanType;
import java.io.IOException;
/**
* Processes index updates.
* <p>
* This involves sending updates directly to ElasticSearch via it's Bulk API or
* queuing events for future processing.
* </p>
*/
public interface DocStoreUpdateProcessor {
/**
* Create a processor to handle updates per bean via a findEach query.
*/
<T> DocStoreQueryUpdate<T> createQueryUpdate(BeanType<T> beanType, int bulkBatchSize) throws IOException;
/**
* Process all the updates for a transaction.
* <p>
* Typically this makes calls to the Bulk API of the document store or simply adds entries
* to a queue for future processing.
* </p>
*
* @param docStoreUpdates The 'Bulk' and 'Queue' updates to the indexes for the transaction.
* @param bulkBatchSize The batch size to use for Bulk API calls specified on the transaction.
* If this is 0 then the default batch size is used.
*/
void process(DocStoreUpdates docStoreUpdates, int bulkBatchSize);
/**
* Create a document store transaction hinting at the batch size.
* <p>
* The batch size can be set via {@link Transaction#setDocStoreBatchSize(int)}
*/
DocStoreTransaction createTransaction(int batchSize);
/**
* Perform commit/flush of the changes made via the document store transaction.
*/
void commit(DocStoreTransaction docStoreTransaction);
}
@@ -0,0 +1,115 @@
package io.ebeanservice.docstore.api;
import io.ebean.DocStoreQueueEntry;
import io.ebean.DocStoreQueueEntry.Action;
import java.util.ArrayList;
import java.util.List;
/**
* Collection of document store updates that are either sent to the document store
* or queued for future processing
*/
public class DocStoreUpdates {
/**
* Persist inserts and updates.
*/
private final List<DocStoreUpdate> persistEvents = new ArrayList<>();
/**
* Delete by Id.
*/
private final List<DocStoreUpdate> deleteEvents = new ArrayList<>();
/**
* Nested updates.
*/
private final List<DocStoreQueueEntry> nestedEvents = new ArrayList<>();
/**
* Entries sent to the queue for later processing.
*/
private final List<DocStoreQueueEntry> queueEntries = new ArrayList<>();
public DocStoreUpdates() {
}
/**
* Return true if there are no events to process.
*/
public boolean isEmpty() {
return persistEvents.isEmpty() && deleteEvents.isEmpty() && nestedEvents.isEmpty() && queueEntries.isEmpty();
}
/**
* Add a persist request.
*/
public void addPersist(DocStoreUpdate bulkRequest) {
persistEvents.add(bulkRequest);
}
/**
* Add a delete request.
*/
public void addDelete(DocStoreUpdate bulkRequest) {
deleteEvents.add(bulkRequest);
}
/**
* Add a nested update.
*/
public void addNested(String queueId, String path, Object beanId) {
nestedEvents.add(new DocStoreQueueEntry(Action.NESTED, queueId, path, beanId));
}
/**
* Queue an 'index' request.
*/
public void queueIndex(String queueId, Object beanId) {
queueEntries.add(new DocStoreQueueEntry(Action.INDEX, queueId, beanId));
}
/**
* Queue a 'delete' request.
*/
public void queueDelete(String queueId, Object beanId) {
queueEntries.add(new DocStoreQueueEntry(Action.DELETE, queueId, beanId));
}
/**
* Queue an update to a nested/embedded object.
*/
public void queueNested(String queueId, String path, Object beanId) {
queueEntries.add(new DocStoreQueueEntry(Action.NESTED, queueId, path, beanId));
}
/**
* Return the persist insert and update requests to be sent to the document store.
*/
public List<DocStoreUpdate> getPersistEvents() {
return persistEvents;
}
/**
* Return delete events.
*/
public List<DocStoreUpdate> getDeleteEvents() {
return deleteEvents;
}
/**
* Return the list of nested update events.
*/
public List<DocStoreQueueEntry> getNestedEvents() {
return nestedEvents;
}
/**
* Return the entries for sending to the queue.
*/
public List<DocStoreQueueEntry> getQueueEntries() {
return queueEntries;
}
}
@@ -0,0 +1,17 @@
package io.ebeanservice.docstore.api;
/**
* Can be thrown when a document is unexpectedly not found in a document store.
*/
public class DocumentNotFoundException extends RuntimeException {
private static final long serialVersionUID = 2066138180892685276L;
/**
* Construct with a message.
*/
public DocumentNotFoundException(String message) {
super(message);
}
}
@@ -0,0 +1,143 @@
package io.ebeanservice.docstore.api.mapping;
import io.ebean.annotation.DocMapping;
import io.ebean.annotation.DocStore;
import io.ebean.text.PathProperties;
import io.ebeaninternal.server.query.SplitName;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.Stack;
/**
* Builds the DocumentMapping for a given bean type.
*/
public class DocMappingBuilder {
private final PathProperties paths;
private final DocStore docStore;
private final Stack<DocPropertyMapping> properties = new Stack<>();
private final Map<String, DocPropertyMapping> map = new LinkedHashMap<>();
/**
* Create with the document structure paths and docStore deployment annotation.
*/
public DocMappingBuilder(PathProperties paths, DocStore docStore) {
this.paths = paths;
this.docStore = docStore;
this.properties.push(new DocPropertyMapping());
}
/**
* Return true if the property is included in the document.
*/
public boolean includesProperty(String prefix, String name) {
return paths.includesProperty(prefix, name);
}
/**
* Return true if the path is included in the document.
*/
public boolean includesPath(String prefix, String name) {
return paths.includesProperty(prefix, name);
}
/**
* Add the property mapping.
*/
public void add(DocPropertyMapping docMapping) {
DocPropertyMapping currentParent = properties.peek();
currentParent.addChild(docMapping);
String parentName = currentParent.getName();
String fullName = SplitName.add(parentName, docMapping.getName());
map.put(fullName, docMapping);
}
/**
* Push the nested object or list onto the properties stack.
*/
public void push(DocPropertyMapping nested) {
properties.push(nested);
}
/**
* Pop the nested object or list off the properties stack.
*/
public void pop() {
properties.pop();
}
/**
* Apply any override mappings from the top level docStore annotation.
*/
public void applyMapping() {
DocMapping[] mapping = docStore.mapping();
for (DocMapping docMapping : mapping) {
applyFieldMapping(null, docMapping);
}
}
private void applyFieldMapping(String prefix, DocMapping docMapping) {
String name = docMapping.name();
String fullName = SplitName.add(prefix, name);
DocPropertyMapping mapping = map.get(fullName);
if (mapping == null) {
throw new IllegalStateException("DocMapping for [" + fullName + "] but property not included in document?");
}
mapping.apply(docMapping);
}
/**
* Collect the mapping of properties to 'raw' properties for those marked as sortable.
*/
public Map<String, String> collectSortable() {
DocPropertyMapping peek = properties.peek();
SortableVisitor visitor = new SortableVisitor();
peek.visit(visitor);
return visitor.getSortableMap();
}
/**
* Create the document mapping.
*/
public DocumentMapping create(String queueId, String indexName, String indexType) {
int shards = docStore.shards();
int replicas = docStore.replicas();
DocPropertyMapping root = properties.peek();
return new DocumentMapping(queueId, indexName, indexType, paths, root, shards, replicas);
}
/**
* Find sortable properties to build the mapping to 'raw' properties.
*/
private static class SortableVisitor extends DocPropertyAdapter {
private Map<String, String> sortableMap = new LinkedHashMap<>();
@Override
public void visitProperty(DocPropertyMapping property) {
DocPropertyOptions options = property.getOptions();
if (options != null && options.isSortable()) {
String fullPath = pathStack.peekFullPath(property.getName());
sortableMap.put(fullPath, fullPath + ".raw");
}
}
private Map<String, String> getSortableMap() {
return sortableMap;
}
}
}
@@ -0,0 +1,47 @@
package io.ebeanservice.docstore.api.mapping;
import io.ebeaninternal.server.text.json.PathStack;
/**
* Adapter for DocPropertyVisitor that does not do anything.
* Used to extend and implement only the desired methods.
*/
public abstract class DocPropertyAdapter implements DocPropertyVisitor {
protected PathStack pathStack = new PathStack();
@Override
public void visitProperty(DocPropertyMapping property) {
// do nothing
}
@Override
public void visitBegin() {
// do nothing
}
@Override
public void visitEnd() {
// do nothing
}
@Override
public void visitBeginObject(DocPropertyMapping property) {
pathStack.push(property.getName());
}
@Override
public void visitEndObject(DocPropertyMapping property) {
pathStack.pop();
}
@Override
public void visitBeginList(DocPropertyMapping property) {
pathStack.push(property.getName());
}
@Override
public void visitEndList(DocPropertyMapping property) {
pathStack.pop();
}
}
@@ -0,0 +1,130 @@
package io.ebeanservice.docstore.api.mapping;
import io.ebean.annotation.DocMapping;
import java.util.ArrayList;
import java.util.List;
/**
* Property mapping in a doc store document structure.
*/
public class DocPropertyMapping {
private String name;
private DocPropertyType type;
private DocPropertyOptions options;
private List<DocPropertyMapping> children = new ArrayList<>();
/**
* Construct ROOT.
*/
public DocPropertyMapping() {
this.type = DocPropertyType.ROOT;
}
/**
* Construct property mapping.
*/
public DocPropertyMapping(String name, DocPropertyType type) {
this.type = type;
this.name = name;
this.options = new DocPropertyOptions();
}
/**
* Construct property mapping with options.
*/
public DocPropertyMapping(String name, DocPropertyType type, DocPropertyOptions options) {
this.name = name;
this.type = type;
this.options = options;
}
/**
* Visit this property and any nested children.
*/
public void visit(DocPropertyVisitor visitor) {
switch (type) {
case ROOT:
visitor.visitBegin();
visitChildren(visitor);
visitor.visitEnd();
break;
case OBJECT:
visitor.visitBeginObject(this);
visitChildren(visitor);
visitor.visitEndObject(this);
break;
case LIST:
visitor.visitBeginList(this);
visitChildren(visitor);
visitor.visitEndList(this);
break;
default:
visitor.visitProperty(this);
}
}
private void visitChildren(DocPropertyVisitor visitor) {
for (DocPropertyMapping property : children) {
property.visit(visitor);
}
}
public String toString() {
return "name:" + name + " type:" + type + " options(" + options + ")";
}
/**
* Return the type of the property.
*/
public DocPropertyType getType() {
return type;
}
/**
* Set the type of the property.
*/
public void setType(DocPropertyType type) {
this.type = type;
}
/**
* Return the property name.
*/
public String getName() {
return name;
}
/**
* Return the property options.
*/
public DocPropertyOptions getOptions() {
return options;
}
/**
* Return the child nested properties.
*/
public List<DocPropertyMapping> getChildren() {
return children;
}
/**
* Add a child property.
*/
public void addChild(DocPropertyMapping docMapping) {
children.add(docMapping);
}
/**
* Apply mapping options to this property.
*/
public void apply(DocMapping docMapping) {
options.apply(docMapping);
}
}
@@ -0,0 +1,260 @@
package io.ebeanservice.docstore.api.mapping;
import io.ebean.annotation.DocMapping;
import io.ebean.annotation.DocProperty;
/**
* Options for mapping a property for document storage.
*/
public class DocPropertyOptions {
private Boolean code;
private Boolean sortable;
private Boolean store;
private Float boost;
private String nullValue;
private Boolean includeInAll;
private Boolean enabled;
private Boolean norms;
private Boolean docValues;
private String analyzer;
private String searchAnalyzer;
private String copyTo;
private DocProperty.Option options;
/**
* Construct with no values set.
*/
public DocPropertyOptions() {
}
/**
* Construct as a copy of the source options.
*/
protected DocPropertyOptions(DocPropertyOptions source) {
this.code = source.code;
this.sortable = source.sortable;
this.store = source.store;
this.boost = source.boost;
this.nullValue = source.nullValue;
this.includeInAll = source.includeInAll;
this.analyzer = source.analyzer;
this.searchAnalyzer = source.searchAnalyzer;
this.options = source.options;
this.docValues = source.docValues;
this.norms = source.norms;
this.copyTo = source.copyTo;
this.enabled = source.enabled;
}
public String toString() {
StringBuilder sb = new StringBuilder();
if (code != null) {
sb.append("code:").append(code).append(" ");
}
if (sortable != null) {
sb.append("sortable:").append(sortable).append(" ");
}
if (store != null) {
sb.append("store:").append(store).append(" ");
}
if (boost != null) {
sb.append("boost:").append(boost).append(" ");
}
if (nullValue != null) {
sb.append("nullValue:").append(nullValue).append(" ");
}
return sb.toString();
}
public boolean isCode() {
return Boolean.TRUE.equals(code);
}
public Boolean getCode() {
return code;
}
public void setCode(Boolean code) {
this.code = code;
}
public boolean isSortable() {
return Boolean.TRUE.equals(sortable);
}
public Boolean getSortable() {
return sortable;
}
public void setSortable(Boolean sortable) {
this.sortable = sortable;
}
public Float getBoost() {
return boost;
}
public void setBoost(Float boost) {
this.boost = boost;
}
public String getNullValue() {
return nullValue;
}
public void setNullValue(String nullValue) {
this.nullValue = nullValue;
}
public Boolean getStore() {
return store;
}
public void setStore(Boolean store) {
this.store = store;
}
public Boolean getIncludeInAll() {
return includeInAll;
}
public void setIncludeInAll(Boolean includeInAll) {
this.includeInAll = includeInAll;
}
public Boolean getDocValues() {
return docValues;
}
public void setDocValues(Boolean docValues) {
this.docValues = docValues;
}
public String getAnalyzer() {
return analyzer;
}
public void setAnalyzer(String analyzer) {
this.analyzer = analyzer;
}
public String getSearchAnalyzer() {
return searchAnalyzer;
}
public void setSearchAnalyzer(String searchAnalyzer) {
this.searchAnalyzer = searchAnalyzer;
}
public String getCopyTo() {
return copyTo;
}
public void setCopyTo(String copyTo) {
this.copyTo = copyTo;
}
public Boolean getEnabled() {
return enabled;
}
public void setEnabled(Boolean enabled) {
this.enabled = enabled;
}
public Boolean getNorms() {
return norms;
}
public void setNorms(Boolean norms) {
this.norms = norms;
}
/**
* Return true if the index options is set to a non-default value.
*/
public boolean isOptionsSet() {
return options != null && options != DocProperty.Option.DEFAULT;
}
public DocProperty.Option getOptions() {
return options;
}
public void setOptions(DocProperty.Option options) {
this.options = options;
}
/**
* Create a copy of this such that it can be overridden on a per index basis.
*/
public DocPropertyOptions copy() {
return new DocPropertyOptions(this);
}
/**
* Apply override mapping from the document level or embedded property level.
*/
public void apply(DocMapping docMapping) {
apply(docMapping.options());
}
/**
* Apply the property level mapping options.
*/
public void apply(DocProperty docMapping) {
options = docMapping.options();
if (docMapping.code()) {
code = true;
}
if (docMapping.sortable()) {
sortable = true;
}
if (docMapping.store()) {
store = true;
}
if (docMapping.boost() != 1) {
boost = docMapping.boost();
}
if (!"".equals(docMapping.nullValue())) {
nullValue = docMapping.nullValue();
}
if (!docMapping.includeInAll()) {
includeInAll = false;
}
if (!docMapping.docValues()) {
docValues = false;
}
if (!docMapping.enabled()) {
enabled = false;
}
if (!docMapping.norms()) {
norms = false;
}
if (!"".equals(docMapping.analyzer())) {
analyzer = docMapping.analyzer();
}
if (!"".equals(docMapping.searchAnalyzer())) {
searchAnalyzer = docMapping.searchAnalyzer();
}
if (!"".equals(docMapping.copyTo())) {
copyTo = docMapping.copyTo();
}
}
}
@@ -0,0 +1,88 @@
package io.ebeanservice.docstore.api.mapping;
/**
* Types as defined for document store property types.
*/
public enum DocPropertyType {
/**
* Enum.
*/
ENUM,
/**
* A UUID is a String Id implying it should not be analysed.
*/
UUID,
/**
* Keyword/code string content not expected to be analysed.
*/
KEYWORD,
/**
* String content expected to be analysed.
*/
TEXT,
/**
* Boolean.
*/
BOOLEAN,
/**
* Short.
*/
SHORT,
/**
* Integer.
*/
INTEGER,
/**
* Long.
*/
LONG,
/**
* Float.
*/
FLOAT,
/**
* Double.
*/
DOUBLE,
/**
* Date without time.
*/
DATE,
/**
* Date with time.
*/
DATETIME,
/**
* Binary type.
*/
BINARY,
/**
* A nested object.
*/
OBJECT,
/**
* A nested list of objects.
*/
LIST,
/**
* Root level type.
*/
ROOT
}
@@ -0,0 +1,43 @@
package io.ebeanservice.docstore.api.mapping;
/**
* Used to visit the properties in a document structure.
*/
public interface DocPropertyVisitor {
/**
* Begin visiting the document structure.
*/
void visitBegin();
/**
* Visit a property.
*/
void visitProperty(DocPropertyMapping property);
/**
* Start visiting a nested object.
*/
void visitBeginObject(DocPropertyMapping property);
/**
* End visiting a nested object.
*/
void visitEndObject(DocPropertyMapping property);
/**
* Start visiting a nested list.
*/
void visitBeginList(DocPropertyMapping property);
/**
* End visiting a nested list.
*/
void visitEndList(DocPropertyMapping property);
/**
* Finished visiting the document structure.
*/
void visitEnd();
}
@@ -0,0 +1,103 @@
package io.ebeanservice.docstore.api.mapping;
import io.ebean.FetchPath;
/**
* Mapping for a document stored in a doc store (like ElasticSearch).
*/
public class DocumentMapping {
protected final String queueId;
protected final String name;
protected final String type;
protected final FetchPath paths;
protected final DocPropertyMapping properties;
protected int shards;
protected int replicas;
public DocumentMapping(String queueId, String name, String type, FetchPath paths, DocPropertyMapping properties, int shards, int replicas) {
this.queueId = queueId;
this.name = name;
this.type = type;
this.paths = paths;
this.properties = properties;
this.shards = shards;
this.replicas = replicas;
}
/**
* Visit all the properties in the document structure.
*/
public void visit(DocPropertyVisitor visitor) {
properties.visit(visitor);
}
/**
* Return the queueId.
*/
public String getQueueId() {
return queueId;
}
/**
* Return the name.
*/
public String getName() {
return name;
}
/**
* Return the type.
*/
public String getType() {
return type;
}
/**
* Return the document structure as PathProperties.
*/
public FetchPath getPaths() {
return paths;
}
/**
* Return the document structure with mapping details.
*/
public DocPropertyMapping getProperties() {
return properties;
}
/**
* Return the number of shards.
*/
public int getShards() {
return shards;
}
/**
* Set the number of shards.
*/
public void setShards(int shards) {
this.shards = shards;
}
/**
* Return the number of replicas.
*/
public int getReplicas() {
return replicas;
}
/**
* Set the number of replicas.
*/
public void setReplicas(int replicas) {
this.replicas = replicas;
}
}
@@ -0,0 +1,4 @@
/**
* Mapping for document store integration.
*/
package io.ebeanservice.docstore.api.mapping;
@@ -0,0 +1,4 @@
/**
* The service API for document store integration.
*/
package io.ebeanservice.docstore.api;
@@ -0,0 +1,341 @@
package io.ebeanservice.docstore.api.support;
import io.ebean.FetchPath;
import io.ebean.Query;
import io.ebean.annotation.DocStore;
import io.ebean.annotation.DocStoreMode;
import io.ebean.plugin.BeanType;
import io.ebean.text.PathProperties;
import io.ebeaninternal.api.SpiEbeanServer;
import io.ebeaninternal.server.core.PersistRequest;
import io.ebeaninternal.server.core.PersistRequestBean;
import io.ebeaninternal.server.deploy.BeanDescriptor;
import io.ebeaninternal.server.deploy.BeanProperty;
import io.ebeaninternal.server.deploy.InheritInfo;
import io.ebeaninternal.server.deploy.meta.DeployBeanDescriptor;
import io.ebeanservice.docstore.api.DocStoreBeanAdapter;
import io.ebeanservice.docstore.api.DocStoreUpdateContext;
import io.ebeanservice.docstore.api.DocStoreUpdates;
import io.ebeanservice.docstore.api.mapping.DocMappingBuilder;
import io.ebeanservice.docstore.api.mapping.DocumentMapping;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.Map;
import java.util.Set;
/**
* Base implementation for much of DocStoreBeanAdapter.
*/
public abstract class DocStoreBeanBaseAdapter<T> implements DocStoreBeanAdapter<T> {
protected final SpiEbeanServer server;
/**
* The associated BeanDescriptor.
*/
protected final BeanDescriptor<T> desc;
/**
* The type of index.
*/
protected final boolean mapped;
/**
* Identifier used in the queue system to identify the index.
*/
protected final String queueId;
/**
* ElasticSearch index type.
*/
protected final String indexType;
/**
* ElasticSearch index name.
*/
protected final String indexName;
/**
* Doc store deployment annotation.
*/
private final DocStore docStore;
/**
* Behavior on insert.
*/
protected final DocStoreMode insert;
/**
* Behavior on update.
*/
protected DocStoreMode update;
/**
* Behavior on delete.
*/
protected final DocStoreMode delete;
/**
* List of embedded paths from other documents that include this document type.
* As such an update to this doc type means that those embedded documents need to be updated.
*/
protected final List<DocStoreEmbeddedInvalidation> embeddedInvalidation = new ArrayList<>();
protected final PathProperties pathProps;
/**
* Map of properties to 'raw' properties.
*/
protected Map<String, String> sortableMap;
/**
* Nested path properties defining the doc structure for indexing.
*/
protected DocStructure docStructure;
protected DocumentMapping documentMapping;
private boolean registerPaths;
public DocStoreBeanBaseAdapter(BeanDescriptor<T> desc, DeployBeanDescriptor<T> deploy) {
this.desc = desc;
this.server = desc.getEbeanServer();
this.mapped = deploy.isDocStoreMapped();
this.pathProps = deploy.getDocStorePathProperties();
this.docStore = deploy.getDocStore();
this.queueId = derive(desc, deploy.getDocStoreQueueId());
this.indexName = derive(desc, deploy.getDocStoreIndexName());
this.indexType = derive(desc, deploy.getDocStoreIndexType());
this.insert = deploy.getDocStoreInsertEvent();
this.update = deploy.getDocStoreUpdateEvent();
this.delete = deploy.getDocStoreDeleteEvent();
}
@Override
public boolean hasEmbeddedInvalidation() {
return !embeddedInvalidation.isEmpty();
}
@Override
public DocumentMapping createDocMapping() {
if (documentMapping != null) {
return documentMapping;
}
if (!mapped) return null;
this.docStructure = derivePathProperties(pathProps);
DocMappingBuilder mappingBuilder = new DocMappingBuilder(docStructure.doc(), docStore);
desc.docStoreMapping(mappingBuilder, null);
mappingBuilder.applyMapping();
sortableMap = mappingBuilder.collectSortable();
docStructure.prepareMany(desc);
documentMapping = mappingBuilder.create(queueId, indexName, indexType);
return documentMapping;
}
@Override
public String getIndexType() {
return indexType;
}
@Override
public String getIndexName() {
return indexName;
}
@Override
public void applyPath(Query<T> query) {
query.apply(docStructure.doc());
}
@Override
public String rawProperty(String property) {
String rawProperty = sortableMap.get(property);
return rawProperty == null ? property : rawProperty;
}
/**
* Register invalidation paths for embedded documents.
*/
@Override
public void registerPaths() {
if (mapped && !registerPaths) {
Collection<PathProperties.Props> pathProps = docStructure.doc().getPathProps();
for (PathProperties.Props pathProp : pathProps) {
String path = pathProp.getPath();
if (path != null) {
BeanDescriptor<?> targetDesc = desc.getBeanDescriptor(path);
BeanProperty idProperty = targetDesc.getIdProperty();
if (idProperty != null) {
// embedded beans don't have id property
String fullPath = path + "." + idProperty.getName();
targetDesc.docStoreAdapter().registerInvalidationPath(desc.getDocStoreQueueId(), fullPath, pathProp.getProperties());
}
}
}
registerPaths = true;
}
}
/**
* Register a doc store invalidation listener for the given bean type, path and properties.
*/
@Override
public void registerInvalidationPath(String queueId, String path, Set<String> properties) {
if (!mapped) {
if (update == DocStoreMode.IGNORE) {
// bean type not mapped but is included as nested document
// in a doc store index so we need to update
update = DocStoreMode.UPDATE;
}
}
embeddedInvalidation.add(getEmbeddedInvalidation(queueId, path, properties));
}
/**
* Return the DsInvalidationListener based on the properties, path.
*/
protected DocStoreEmbeddedInvalidation getEmbeddedInvalidation(String queueId, String path, Set<String> properties) {
if (properties.contains("*")) {
return new DocStoreEmbeddedInvalidation(queueId, path);
} else {
return new DocStoreEmbeddedInvalidationProperties(queueId, path, getPropertyPositions(properties));
}
}
/**
* Return the property names as property index positions.
*/
protected int[] getPropertyPositions(Set<String> properties) {
List<Integer> posList = new ArrayList<>();
for (String property : properties) {
BeanProperty prop = desc.getBeanProperty(property);
if (prop != null) {
posList.add(prop.getPropertyIndex());
}
}
int[] pos = new int[posList.size()];
for (int i = 0; i < pos.length; i++) {
pos[i] = posList.get(i);
}
return pos;
}
@Override
public void updateEmbedded(PersistRequestBean<T> request, DocStoreUpdates docStoreUpdates) {
for (DocStoreEmbeddedInvalidation anEmbeddedInvalidation : embeddedInvalidation) {
anEmbeddedInvalidation.embeddedInvalidate(request, docStoreUpdates);
}
}
/**
* Return the pathProperties which defines the JSON document to index.
* This can add derived/embedded/nested parts to the document.
*/
protected DocStructure derivePathProperties(PathProperties pathProps) {
boolean includeByDefault = (pathProps == null);
if (pathProps == null) {
pathProps = new PathProperties();
}
return getDocStructure(pathProps, includeByDefault);
}
protected DocStructure getDocStructure(PathProperties pathProps, final boolean includeByDefault) {
final DocStructure docStructure = new DocStructure(pathProps);
BeanProperty[] properties = desc.propertiesNonTransient();
for (BeanProperty property : properties) {
property.docStoreInclude(includeByDefault, docStructure);
}
InheritInfo inheritInfo = desc.getInheritInfo();
if (inheritInfo != null) {
inheritInfo.visitChildren(inheritInfo1 -> {
for (BeanProperty localProperty : inheritInfo1.localProperties()) {
localProperty.docStoreInclude(includeByDefault, docStructure);
}
});
}
return docStructure;
}
public FetchPath getEmbedded(String path) {
return docStructure.getEmbedded(path);
}
public FetchPath getEmbeddedManyRoot(String path) {
return docStructure.getEmbeddedManyRoot(path);
}
@Override
public boolean isMapped() {
return mapped;
}
@Override
public String getQueueId() {
return queueId;
}
@Override
public DocStoreMode getMode(PersistRequest.Type persistType, DocStoreMode txnMode) {
if (txnMode == null) {
return getMode(persistType);
} else if (txnMode == DocStoreMode.IGNORE) {
return DocStoreMode.IGNORE;
}
return mapped ? txnMode : getMode(persistType);
}
private DocStoreMode getMode(PersistRequest.Type persistType) {
switch (persistType) {
case INSERT:
return insert;
case UPDATE:
return update;
case DELETE:
return delete;
default:
return DocStoreMode.IGNORE;
}
}
/**
* Return the supplied value or default to the bean name lower case.
*/
protected String derive(BeanType<?> desc, String suppliedValue) {
return (suppliedValue != null && !suppliedValue.isEmpty()) ? suppliedValue : desc.getName().toLowerCase();
}
@Override
public abstract void deleteById(Object idValue, DocStoreUpdateContext txn) throws IOException;
@Override
public abstract void index(Object idValue, T entityBean, DocStoreUpdateContext txn) throws IOException;
@Override
public abstract void insert(Object idValue, PersistRequestBean<T> persistRequest, DocStoreUpdateContext txn) throws IOException;
@Override
public abstract void update(Object idValue, PersistRequestBean<T> persistRequest, DocStoreUpdateContext txn) throws IOException;
@Override
public abstract void updateEmbedded(Object idValue, String embeddedProperty, String embeddedRawContent, DocStoreUpdateContext txn) throws IOException;
}
@@ -0,0 +1,39 @@
package io.ebeanservice.docstore.api.support;
import io.ebean.plugin.BeanType;
import io.ebeanservice.docstore.api.DocStoreUpdate;
import io.ebeanservice.docstore.api.DocStoreUpdateContext;
import io.ebeanservice.docstore.api.DocStoreUpdates;
import java.io.IOException;
/**
* A 'Delete by Id' request that is send to the document store.
*/
public class DocStoreDeleteEvent implements DocStoreUpdate {
private final BeanType<?> beanType;
private final Object idValue;
public DocStoreDeleteEvent(BeanType<?> beanType, Object idValue) {
this.beanType = beanType;
this.idValue = idValue;
}
/**
* Add appropriate JSON content for sending to the ElasticSearch Bulk API.
*/
@Override
public void docStoreUpdate(DocStoreUpdateContext txn) throws IOException {
beanType.docStore().deleteById(idValue, txn);
}
/**
* Add this event to the queue (for queue delayed processing).
*/
@Override
public void addToQueue(DocStoreUpdates docStoreUpdates) {
docStoreUpdates.queueDelete(beanType.getDocStoreQueueId(), idValue);
}
}
@@ -0,0 +1,23 @@
package io.ebeanservice.docstore.api.support;
import io.ebeaninternal.server.core.PersistRequestBean;
import io.ebeanservice.docstore.api.DocStoreUpdates;
/**
* Checks if a persist request means an embedded/nested object in another document needs updating.
*/
public class DocStoreEmbeddedInvalidation {
protected final String queueId;
protected final String path;
public DocStoreEmbeddedInvalidation(String queueId, String path) {
this.queueId = queueId;
this.path = path;
}
public void embeddedInvalidate(PersistRequestBean<?> request, DocStoreUpdates docStoreUpdates) {
docStoreUpdates.addNested(queueId, path, request.getBeanId());
}
}
@@ -0,0 +1,30 @@
package io.ebeanservice.docstore.api.support;
import io.ebeaninternal.server.core.PersistRequestBean;
import io.ebeanservice.docstore.api.DocStoreUpdates;
/**
* Checks if a persist request means an embedded/nested object in another document needs updating.
* <p>
* This has specific properties to check (so not all properties invalidate).
*/
public final class DocStoreEmbeddedInvalidationProperties extends DocStoreEmbeddedInvalidation {
/**
* Properties that trigger invalidation.
*/
private final int[] properties;
public DocStoreEmbeddedInvalidationProperties(String queueId, String path, int[] properties) {
super(queueId, path);
this.properties = properties;
}
@Override
public void embeddedInvalidate(PersistRequestBean<?> request, DocStoreUpdates docStoreUpdates) {
if (request.hasDirtyProperty(properties)) {
docStoreUpdates.addNested(queueId, path, request.getBeanId());
}
}
}
@@ -0,0 +1,42 @@
package io.ebeanservice.docstore.api.support;
import io.ebean.plugin.BeanType;
import io.ebeanservice.docstore.api.DocStoreUpdate;
import io.ebeanservice.docstore.api.DocStoreUpdateContext;
import io.ebeanservice.docstore.api.DocStoreUpdates;
import java.io.IOException;
/**
* A 'Delete by Id' request that is send to the document store.
*/
public class DocStoreIndexEvent<T> implements DocStoreUpdate {
private final BeanType<T> beanType;
private final Object idValue;
private final T bean;
public DocStoreIndexEvent(BeanType<T> beanType, Object idValue, T bean) {
this.beanType = beanType;
this.idValue = idValue;
this.bean = bean;
}
/**
* Add appropriate JSON content for sending to the ElasticSearch Bulk API.
*/
@Override
public void docStoreUpdate(DocStoreUpdateContext txn) throws IOException {
beanType.docStore().index(idValue, bean, txn);
}
/**
* Add this event to the queue (for queue delayed processing).
*/
@Override
public void addToQueue(DocStoreUpdates docStoreUpdates) {
docStoreUpdates.queueIndex(beanType.getDocStoreQueueId(), idValue);
}
}
@@ -0,0 +1,95 @@
package io.ebeanservice.docstore.api.support;
import io.ebean.FetchPath;
import io.ebean.text.PathProperties;
import io.ebeaninternal.server.deploy.BeanDescriptor;
import io.ebeaninternal.server.deploy.BeanPropertyAssoc;
import java.util.HashMap;
import java.util.Map;
import java.util.Set;
/**
* Document structure for mapping to document store.
*/
public class DocStructure {
/**
* The full document structure.
*/
private final PathProperties doc;
/**
* The embedded document structures by path.
*/
private final Map<String, PathProperties> embedded = new HashMap<>();
private final Map<String, PathProperties> manyRoot = new HashMap<>();
/**
* Create given an initial deployment doc mapping.
*/
public DocStructure(PathProperties pathProps) {
this.doc = pathProps;
}
/**
* Add a property at the root level.
*/
public void addProperty(String name) {
doc.addToPath(null, name);
}
/**
* Add an embedded property with it's document structure.
*/
public void addNested(String path, PathProperties embeddedDoc) {
doc.addNested(path, embeddedDoc);
embedded.put(path, embeddedDoc);
}
/**
* Return the document structure.
*/
public PathProperties doc() {
return doc;
}
/**
* Return the document structure for an embedded path.
*/
public FetchPath getEmbedded(String path) {
return embedded.get(path);
}
public FetchPath getEmbeddedManyRoot(String path) {
return manyRoot.get(path);
}
/**
* For 'many' nested properties we need an additional root based graph to fetch and update.
*/
public <T> void prepareMany(BeanDescriptor<T> desc) {
Set<String> strings = embedded.keySet();
for (String prop : strings) {
BeanPropertyAssoc<?> embProp = (BeanPropertyAssoc<?>) desc.findBeanProperty(prop);
if (embProp.isMany()) {
prepare(prop, embProp);
}
}
}
/**
* Add a PathProperties for an embedded 'many' property (at the root level).
*/
private void prepare(String prop, BeanPropertyAssoc<?> embProp) {
BeanDescriptor<?> targetDesc = embProp.getTargetDescriptor();
PathProperties manyRootPath = new PathProperties();
manyRootPath.addToPath(null, targetDesc.getIdProperty().getName());
manyRootPath.addNested(prop, embedded.get(prop));
manyRoot.put(prop, manyRootPath);
}
}
@@ -0,0 +1,4 @@
/**
* Support objects for implementing integration.
*/
package io.ebeanservice.docstore.api.support;
@@ -0,0 +1,98 @@
package io.ebeanservice.docstore.none;
import io.ebean.DocStoreQueueEntry;
import io.ebean.DocumentStore;
import io.ebean.PagedList;
import io.ebean.Query;
import io.ebeanservice.docstore.api.DocQueryRequest;
import java.io.IOException;
import java.util.List;
import java.util.Map;
import java.util.function.Consumer;
import java.util.function.Predicate;
/**
* DocumentStore that barfs it is used.
*/
public class NoneDocStore implements DocumentStore {
public static IllegalStateException implementationNotInClassPath() {
throw new IllegalStateException("DocStore implementation not included in the classPath. You need to add the maven dependency for avaje-ebeanorm-elastic");
}
@Override
public void indexSettings(String indexName, Map<String, Object> settings) {
throw implementationNotInClassPath();
}
@Override
public void dropIndex(String newIndex) {
throw implementationNotInClassPath();
}
@Override
public void createIndex(String indexName, String alias) {
throw implementationNotInClassPath();
}
@Override
public void indexAll(Class<?> countryClass) {
throw implementationNotInClassPath();
}
@Override
public long copyIndex(Class<?> beanType, String newIndex) {
throw implementationNotInClassPath();
}
@Override
public long copyIndex(Class<?> beanType, String newIndex, long epochMillis) {
throw implementationNotInClassPath();
}
@Override
public long copyIndex(Query<?> query, String newIndex, int bulkBatchSize) {
throw implementationNotInClassPath();
}
@Override
public <T> void indexByQuery(Query<T> query) {
throw implementationNotInClassPath();
}
@Override
public <T> void indexByQuery(Query<T> query, int bulkBatchSize) {
throw implementationNotInClassPath();
}
@Override
public <T> T find(DocQueryRequest<T> request) {
throw implementationNotInClassPath();
}
@Override
public <T> PagedList<T> findPagedList(DocQueryRequest<T> request) {
throw implementationNotInClassPath();
}
@Override
public <T> List<T> findList(DocQueryRequest<T> request) {
throw implementationNotInClassPath();
}
@Override
public <T> void findEach(DocQueryRequest<T> query, Consumer<T> consumer) {
throw implementationNotInClassPath();
}
@Override
public <T> void findEachWhile(DocQueryRequest<T> query, Predicate<T> consumer) {
throw implementationNotInClassPath();
}
@Override
public long process(List<DocStoreQueueEntry> queueEntries) throws IOException {
throw implementationNotInClassPath();
}
}
@@ -0,0 +1,49 @@
package io.ebeanservice.docstore.none;
import io.ebeaninternal.server.core.PersistRequestBean;
import io.ebeaninternal.server.deploy.BeanDescriptor;
import io.ebeaninternal.server.deploy.meta.DeployBeanDescriptor;
import io.ebeanservice.docstore.api.DocStoreUpdateContext;
import io.ebeanservice.docstore.api.support.DocStoreBeanBaseAdapter;
import java.io.IOException;
/**
* DocStoreBeanBaseAdapter that barfs if it is used.
*/
public class NoneDocStoreBeanAdapter<T> extends DocStoreBeanBaseAdapter<T> {
public NoneDocStoreBeanAdapter(BeanDescriptor<T> desc, DeployBeanDescriptor<T> deploy) {
super(desc, deploy);
}
@Override
public boolean isMapped() {
return false;
}
@Override
public void deleteById(Object idValue, DocStoreUpdateContext txn) throws IOException {
throw NoneDocStore.implementationNotInClassPath();
}
@Override
public void index(Object idValue, T entityBean, DocStoreUpdateContext txn) throws IOException {
throw NoneDocStore.implementationNotInClassPath();
}
@Override
public void insert(Object idValue, PersistRequestBean<T> persistRequest, DocStoreUpdateContext txn) throws IOException {
throw NoneDocStore.implementationNotInClassPath();
}
@Override
public void update(Object idValue, PersistRequestBean<T> persistRequest, DocStoreUpdateContext txn) throws IOException {
throw NoneDocStore.implementationNotInClassPath();
}
@Override
public void updateEmbedded(Object idValue, String embeddedProperty, String embeddedRawContent, DocStoreUpdateContext txn) throws IOException {
throw NoneDocStore.implementationNotInClassPath();
}
}
@@ -0,0 +1,39 @@
package io.ebeanservice.docstore.none;
import io.ebean.DocumentStore;
import io.ebean.plugin.SpiServer;
import io.ebeaninternal.server.deploy.BeanDescriptor;
import io.ebeaninternal.server.deploy.meta.DeployBeanDescriptor;
import io.ebeanservice.docstore.api.DocStoreBeanAdapter;
import io.ebeanservice.docstore.api.DocStoreFactory;
import io.ebeanservice.docstore.api.DocStoreIntegration;
import io.ebeanservice.docstore.api.DocStoreUpdateProcessor;
/**
* A stub implementation of DocStoreFactory that will barf if the docStore features are used.
*/
public class NoneDocStoreFactory implements DocStoreFactory {
@Override
public DocStoreIntegration create(SpiServer server) {
return new NoneIntegration();
}
@Override
public <T> DocStoreBeanAdapter<T> createAdapter(BeanDescriptor<T> desc, DeployBeanDescriptor<T> deploy) {
return new NoneDocStoreBeanAdapter<>(desc, deploy);
}
static class NoneIntegration implements DocStoreIntegration {
@Override
public DocStoreUpdateProcessor updateProcessor() {
return new NoneDocStoreUpdateProcessor();
}
@Override
public DocumentStore documentStore() {
return new NoneDocStore();
}
}
}
@@ -0,0 +1,35 @@
package io.ebeanservice.docstore.none;
import io.ebean.plugin.BeanType;
import io.ebeanservice.docstore.api.DocStoreQueryUpdate;
import io.ebeanservice.docstore.api.DocStoreTransaction;
import io.ebeanservice.docstore.api.DocStoreUpdateProcessor;
import io.ebeanservice.docstore.api.DocStoreUpdates;
import java.io.IOException;
/**
* DocStoreUpdateProcessor that barfs if it is used.
*/
class NoneDocStoreUpdateProcessor implements DocStoreUpdateProcessor {
@Override
public <T> DocStoreQueryUpdate<T> createQueryUpdate(BeanType<T> beanType, int bulkBatchSize) throws IOException {
throw NoneDocStore.implementationNotInClassPath();
}
@Override
public void process(DocStoreUpdates docStoreUpdates, int bulkBatchSize) {
throw NoneDocStore.implementationNotInClassPath();
}
@Override
public DocStoreTransaction createTransaction(int batchSize) {
throw NoneDocStore.implementationNotInClassPath();
}
@Override
public void commit(DocStoreTransaction docStoreTransaction) {
throw NoneDocStore.implementationNotInClassPath();
}
}
@@ -0,0 +1,8 @@
/**
* "No op" implementation of document store.
* <p>
* This is 'placeholder' implementation used if there is no document store service found
* and if there is an attempt to use the document store features an error will be thrown.
* </p>
*/
package io.ebeanservice.docstore.none;