Examples with DocumentMapping com.b2international.index.mapping.DocumentMapping used on opensource projects

Example 6 with DocumentMapping

use of com.b2international.index.mapping.DocumentMapping in project snow-owl by b2ihealthcare.

the class EsIndexAdmin method bulkUpdate.

public boolean bulkUpdate(final BulkUpdate<?> update) {
    final DocumentMapping mapping = mappings().getMapping(update.getType());
    final String rawScript = mapping.getScript(update.getScript()).script();
    org.elasticsearch.script.Script script = new org.elasticsearch.script.Script(ScriptType.INLINE, "painless", rawScript, Map.copyOf(update.getParams()));
    return bulkIndexByScroll(client, mapping, update.getFilter(), "update", script, update.toString());
}

Example 7 with DocumentMapping

use of com.b2international.index.mapping.DocumentMapping in project snow-owl by b2ihealthcare.

the class EsIndexAdmin method toProperties.

private Map<String, Object> toProperties(DocumentMapping mapping) {
    Map<String, Object> properties = newHashMap();
    for (Field field : mapping.getFields()) {
        // skip transient fields
        if (Modifier.isTransient(field.getModifiers())) {
            continue;
        }
        com.b2international.index.mapping.Field fieldAnnotation = field.getAnnotation(com.b2international.index.mapping.Field.class);
        final String property = field.getName();
        final Class<?> fieldType = NumericClassUtils.unwrapCollectionType(field);
        if (Map.class.isAssignableFrom(fieldType)) {
            // allow dynamic mappings for dynamic objects like field using Map
            final Map<String, Object> prop = newHashMap();
            prop.put("type", "object");
            prop.put("dynamic", "true");
            properties.put(property, prop);
            continue;
        } else if (fieldType.isAnnotationPresent(Doc.class)) {
            Doc annotation = fieldType.getAnnotation(Doc.class);
            // this is a nested document type create a nested mapping
            final Map<String, Object> prop = newHashMap();
            // XXX type: object is the default for nested objects, ES won't store it in the mapping and will default to object even if explicitly set, which would cause unnecessary mapping update during boot
            if (annotation.nested()) {
                prop.put("type", "nested");
            }
            // XXX enabled: true is the default, ES won't store it in the mapping and will default to true even if explicitly set, which would cause unnecessary mapping update during boot
            if (!annotation.index() || (fieldAnnotation != null && !fieldAnnotation.index())) {
                prop.put("enabled", false);
            }
            prop.putAll(toProperties(new DocumentMapping(fieldType, true)));
            properties.put(property, prop);
        } else {
            final Map<String, Object> prop = newHashMap();
            addFieldProperties(prop, fieldType);
            // add aliases
            final Map<String, FieldAlias> fieldAliases = mapping.getFieldAliases(property);
            if (!fieldAliases.isEmpty()) {
                final Map<String, Object> fields = newHashMapWithExpectedSize(fieldAliases.size());
                for (FieldAlias fieldAlias : fieldAliases.values()) {
                    final Map<String, Object> fieldAliasProps = newHashMap();
                    // only keywords can have normalizers
                    switch(fieldAlias.type()) {
                        case KEYWORD:
                            fieldAliasProps.put("type", "keyword");
                            String normalizer = fieldAlias.normalizer().getNormalizer();
                            if (!Strings.isNullOrEmpty(normalizer)) {
                                fieldAliasProps.put("normalizer", normalizer);
                            }
                            // XXX doc_values: true is the default, ES won't store it in the mapping and will default to true even if explicitly set, which would cause unnecessary mapping update during boot
                            if (!fieldAlias.index()) {
                                fieldAliasProps.put("index", false);
                                fieldAliasProps.put("doc_values", false);
                            }
                            break;
                        case TEXT:
                            fieldAliasProps.put("type", "text");
                            fieldAliasProps.put("analyzer", fieldAlias.analyzer().getAnalyzer());
                            if (fieldAlias.searchAnalyzer() != Analyzers.INDEX) {
                                fieldAliasProps.put("search_analyzer", fieldAlias.searchAnalyzer().getAnalyzer());
                            }
                            // XXX index: true is the default, ES won't store it in the mapping and will default to true even if explicitly set, which would cause unnecessary mapping update during boot
                            if (!fieldAlias.index()) {
                                fieldAliasProps.put("index", false);
                            }
                            break;
                        default:
                            throw new UnsupportedOperationException("Unknown field alias type: " + fieldAlias.type());
                    }
                    fields.put(fieldAlias.name(), fieldAliasProps);
                }
                prop.put("fields", fields);
            }
            // XXX enabled: true is the default, ES won't store it in the mapping and will default to true even if explicitly set, which would cause unnecessary mapping update during boot
            if (fieldAnnotation != null && !fieldAnnotation.index()) {
                prop.put("index", false);
                prop.put("doc_values", false);
            }
            // register mapping
            properties.put(property, prop);
        }
    }
    return ImmutableMap.of("properties", properties);
}

Example 8 with DocumentMapping

use of com.b2international.index.mapping.DocumentMapping in project snow-owl by b2ihealthcare.

the class EsDocumentSearcher method get.

@Override
public <T> T get(Class<T> type, String key) throws IOException {
    checkArgument(!Strings.isNullOrEmpty(key), "Key cannot be empty");
    final DocumentMapping mapping = admin.mappings().getMapping(type);
    final GetRequest req = new GetRequest(admin.getTypeIndex(mapping), key).fetchSourceContext(FetchSourceContext.FETCH_SOURCE);
    final GetResponse res = admin.client().get(req);
    if (res.isExists()) {
        final byte[] bytes = res.getSourceAsBytes();
        return mapper.readValue(bytes, 0, bytes.length, type);
    } else {
        return null;
    }
}

Example 9 with DocumentMapping

use of com.b2international.index.mapping.DocumentMapping in project snow-owl by b2ihealthcare.

the class EsDocumentWriter method put.

@Override
public void put(Object object) {
    DocumentMapping mapping = admin.mappings().getMapping(object.getClass());
    final String id = mapping.isAutoGeneratedId() ? UUIDs.randomBase64UUID() : mapping.getIdFieldValue(object);
    indexOperations.put(object.getClass(), id, object);
}

Example 10 with DocumentMapping

use of com.b2international.index.mapping.DocumentMapping in project snow-owl by b2ihealthcare.

the class EsDocumentWriter method commit.

@Override
public void commit() throws IOException {
    if (isEmpty()) {
        return;
    }
    final Set<DocumentMapping> mappingsToRefresh = Collections.synchronizedSet(newHashSet());
    final EsClient client = admin.client();
    // apply bulk updates first
    final ListeningExecutorService executor;
    if (bulkUpdateOperations.size() > 1 || bulkDeleteOperations.size() > 1) {
        final int threads = Math.min(4, Math.max(bulkUpdateOperations.size(), bulkDeleteOperations.size()));
        executor = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(threads));
    } else {
        executor = MoreExecutors.newDirectExecutorService();
    }
    final List<ListenableFuture<?>> updateFutures = newArrayList();
    for (BulkUpdate<?> update : bulkUpdateOperations) {
        updateFutures.add(executor.submit(() -> {
            if (admin.bulkUpdate(update)) {
                mappingsToRefresh.add(admin.mappings().getMapping(update.getType()));
            }
        }));
    }
    for (BulkDelete<?> delete : bulkDeleteOperations) {
        updateFutures.add(executor.submit(() -> {
            if (admin.bulkDelete(delete)) {
                mappingsToRefresh.add(admin.mappings().getMapping(delete.getType()));
            }
        }));
    }
    try {
        executor.shutdown();
        Futures.allAsList(updateFutures).get();
        executor.awaitTermination(10, TimeUnit.SECONDS);
    } catch (InterruptedException | ExecutionException e) {
        admin.log().error("Couldn't execute bulk updates", e);
        throw new IndexException("Couldn't execute bulk updates", e);
    }
    // then bulk indexes/deletes
    if (!indexOperations.isEmpty() || !deleteOperations.isEmpty()) {
        final BulkProcessor processor = client.bulk(new BulkProcessor.Listener() {

            @Override
            public void beforeBulk(long executionId, BulkRequest request) {
                admin.log().debug("Sending bulk request {}", request.numberOfActions());
            }

            @Override
            public void afterBulk(long executionId, BulkRequest request, Throwable failure) {
                admin.log().error("Failed bulk request", failure);
            }

            @Override
            public void afterBulk(long executionId, BulkRequest request, BulkResponse response) {
                admin.log().debug("Successfully processed bulk request ({}) in {}.", request.numberOfActions(), response.getTook());
                if (response.hasFailures()) {
                    for (BulkItemResponse itemResponse : response.getItems()) {
                        checkState(!itemResponse.isFailed(), "Failed to commit bulk request in index '%s', %s", admin.name(), itemResponse.getFailureMessage());
                    }
                }
            }
        }).setConcurrentRequests(getConcurrencyLevel()).setBulkActions((int) admin.settings().get(IndexClientFactory.BULK_ACTIONS_SIZE)).setBulkSize(new ByteSizeValue((int) admin.settings().get(IndexClientFactory.BULK_ACTIONS_SIZE_IN_MB), ByteSizeUnit.MB)).build();
        for (Class<?> type : ImmutableSet.copyOf(indexOperations.rowKeySet())) {
            final Map<String, Object> indexOperationsForType = indexOperations.row(type);
            final DocumentMapping mapping = admin.mappings().getMapping(type);
            final String typeIndex = admin.getTypeIndex(mapping);
            mappingsToRefresh.add(mapping);
            for (Entry<String, Object> entry : Iterables.consumingIterable(indexOperationsForType.entrySet())) {
                final String id = entry.getKey();
                if (!deleteOperations.containsValue(id)) {
                    final Object obj = entry.getValue();
                    final byte[] _source = mapper.writeValueAsBytes(obj);
                    IndexRequest indexRequest = new IndexRequest().index(typeIndex).opType(OpType.INDEX).source(_source, XContentType.JSON);
                    // XXX revisions has their special local ID, but that's not needed when sending them to ES, ES will autogenerate a non-conflicting ID for them
                    if (!mapping.isAutoGeneratedId()) {
                        indexRequest.id(id);
                    }
                    processor.add(indexRequest);
                }
            }
            for (String id : deleteOperations.removeAll(type)) {
                processor.add(new DeleteRequest(typeIndex, id));
            }
            // Flush processor between index boundaries
            processor.flush();
        }
        // Remaining delete operations can be executed on their own
        for (Class<?> type : ImmutableSet.copyOf(deleteOperations.keySet())) {
            final DocumentMapping mapping = admin.mappings().getMapping(type);
            final String typeIndex = admin.getTypeIndex(mapping);
            mappingsToRefresh.add(mapping);
            for (String id : deleteOperations.removeAll(type)) {
                processor.add(new DeleteRequest(typeIndex, id));
            }
            // Flush processor between index boundaries
            processor.flush();
        }
        try {
            processor.awaitClose(5, TimeUnit.MINUTES);
        } catch (InterruptedException e) {
            throw new IndexException("Interrupted bulk processing part of the commit", e);
        }
    }
    // refresh the index if there were only updates
    admin.refresh(mappingsToRefresh);
}