Examples with VersionType org.opensearch.index.VersionType used on opensource projects

Example 1 with VersionType

use of org.opensearch.index.VersionType in project OpenSearch by opensearch-project.

the class RequestConverters method bulk.

static Request bulk(BulkRequest bulkRequest) throws IOException {
    Request request = new Request(HttpPost.METHOD_NAME, "/_bulk");
    Params parameters = new Params();
    parameters.withTimeout(bulkRequest.timeout());
    parameters.withRefreshPolicy(bulkRequest.getRefreshPolicy());
    parameters.withPipeline(bulkRequest.pipeline());
    parameters.withRouting(bulkRequest.routing());
    // Bulk API only supports newline delimited JSON or Smile. Before executing
    // the bulk, we need to check that all requests have the same content-type
    // and this content-type is supported by the Bulk API.
    XContentType bulkContentType = null;
    for (int i = 0; i < bulkRequest.numberOfActions(); i++) {
        DocWriteRequest<?> action = bulkRequest.requests().get(i);
        DocWriteRequest.OpType opType = action.opType();
        if (opType == DocWriteRequest.OpType.INDEX || opType == DocWriteRequest.OpType.CREATE) {
            bulkContentType = enforceSameContentType((IndexRequest) action, bulkContentType);
        } else if (opType == DocWriteRequest.OpType.UPDATE) {
            UpdateRequest updateRequest = (UpdateRequest) action;
            if (updateRequest.doc() != null) {
                bulkContentType = enforceSameContentType(updateRequest.doc(), bulkContentType);
            }
            if (updateRequest.upsertRequest() != null) {
                bulkContentType = enforceSameContentType(updateRequest.upsertRequest(), bulkContentType);
            }
        }
    }
    if (bulkContentType == null) {
        bulkContentType = XContentType.JSON;
    }
    final byte separator = bulkContentType.xContent().streamSeparator();
    final ContentType requestContentType = createContentType(bulkContentType);
    ByteArrayOutputStream content = new ByteArrayOutputStream();
    for (DocWriteRequest<?> action : bulkRequest.requests()) {
        DocWriteRequest.OpType opType = action.opType();
        try (XContentBuilder metadata = XContentBuilder.builder(bulkContentType.xContent())) {
            metadata.startObject();
            {
                metadata.startObject(opType.getLowercase());
                if (Strings.hasLength(action.index())) {
                    metadata.field("_index", action.index());
                }
                if (Strings.hasLength(action.id())) {
                    metadata.field("_id", action.id());
                }
                if (Strings.hasLength(action.routing())) {
                    metadata.field("routing", action.routing());
                }
                if (action.version() != Versions.MATCH_ANY) {
                    metadata.field("version", action.version());
                }
                VersionType versionType = action.versionType();
                if (versionType != VersionType.INTERNAL) {
                    if (versionType == VersionType.EXTERNAL) {
                        metadata.field("version_type", "external");
                    } else if (versionType == VersionType.EXTERNAL_GTE) {
                        metadata.field("version_type", "external_gte");
                    }
                }
                if (action.ifSeqNo() != SequenceNumbers.UNASSIGNED_SEQ_NO) {
                    metadata.field("if_seq_no", action.ifSeqNo());
                    metadata.field("if_primary_term", action.ifPrimaryTerm());
                }
                if (opType == DocWriteRequest.OpType.INDEX || opType == DocWriteRequest.OpType.CREATE) {
                    IndexRequest indexRequest = (IndexRequest) action;
                    if (Strings.hasLength(indexRequest.getPipeline())) {
                        metadata.field("pipeline", indexRequest.getPipeline());
                    }
                } else if (opType == DocWriteRequest.OpType.UPDATE) {
                    UpdateRequest updateRequest = (UpdateRequest) action;
                    if (updateRequest.retryOnConflict() > 0) {
                        metadata.field("retry_on_conflict", updateRequest.retryOnConflict());
                    }
                    if (updateRequest.fetchSource() != null) {
                        metadata.field("_source", updateRequest.fetchSource());
                    }
                }
                metadata.endObject();
            }
            metadata.endObject();
            BytesRef metadataSource = BytesReference.bytes(metadata).toBytesRef();
            content.write(metadataSource.bytes, metadataSource.offset, metadataSource.length);
            content.write(separator);
        }
        BytesRef source = null;
        if (opType == DocWriteRequest.OpType.INDEX || opType == DocWriteRequest.OpType.CREATE) {
            IndexRequest indexRequest = (IndexRequest) action;
            BytesReference indexSource = indexRequest.source();
            XContentType indexXContentType = indexRequest.getContentType();
            try (XContentParser parser = XContentHelper.createParser(/*
                         * EMPTY and THROW are fine here because we just call
                         * copyCurrentStructure which doesn't touch the
                         * registry or deprecation.
                         */
            NamedXContentRegistry.EMPTY, DeprecationHandler.THROW_UNSUPPORTED_OPERATION, indexSource, indexXContentType)) {
                try (XContentBuilder builder = XContentBuilder.builder(bulkContentType.xContent())) {
                    builder.copyCurrentStructure(parser);
                    source = BytesReference.bytes(builder).toBytesRef();
                }
            }
        } else if (opType == DocWriteRequest.OpType.UPDATE) {
            source = XContentHelper.toXContent((UpdateRequest) action, bulkContentType, false).toBytesRef();
        }
        if (source != null) {
            content.write(source.bytes, source.offset, source.length);
            content.write(separator);
        }
    }
    request.addParameters(parameters.asMap());
    request.setEntity(new NByteArrayEntity(content.toByteArray(), 0, content.size(), requestContentType));
    return request;
}

Example 2 with VersionType

use of org.opensearch.index.VersionType in project OpenSearch by opensearch-project.

the class DocWriteRequest method validateSeqNoBasedCASParams.

static ActionRequestValidationException validateSeqNoBasedCASParams(DocWriteRequest request, ActionRequestValidationException validationException) {
    final long version = request.version();
    final VersionType versionType = request.versionType();
    if (versionType.validateVersionForWrites(version) == false) {
        validationException = addValidationError("illegal version value [" + version + "] for version type [" + versionType.name() + "]", validationException);
    }
    if (versionType == VersionType.INTERNAL && version != Versions.MATCH_ANY && version != Versions.MATCH_DELETED) {
        validationException = addValidationError("internal versioning can not be used for optimistic concurrency control. " + "Please use `if_seq_no` and `if_primary_term` instead", validationException);
    }
    if (request.ifSeqNo() != UNASSIGNED_SEQ_NO && (versionType != VersionType.INTERNAL || version != Versions.MATCH_ANY)) {
        validationException = addValidationError("compare and write operations can not use versioning", validationException);
    }
    if (request.ifPrimaryTerm() == UNASSIGNED_PRIMARY_TERM && request.ifSeqNo() != UNASSIGNED_SEQ_NO) {
        validationException = addValidationError("ifSeqNo is set, but primary term is [0]", validationException);
    }
    if (request.ifPrimaryTerm() != UNASSIGNED_PRIMARY_TERM && request.ifSeqNo() == UNASSIGNED_SEQ_NO) {
        validationException = addValidationError("ifSeqNo is unassigned, but primary term is [" + request.ifPrimaryTerm() + "]", validationException);
    }
    return validationException;
}

Example 3 with VersionType

use of org.opensearch.index.VersionType in project OpenSearch by opensearch-project.

the class BulkRequestParser method parse.

/**
 * Parse the provided {@code data} assuming the provided default values. Index requests
 * will be passed to the {@code indexRequestConsumer}, update requests to the
 * {@code updateRequestConsumer} and delete requests to the {@code deleteRequestConsumer}.
 */
public void parse(BytesReference data, @Nullable String defaultIndex, @Nullable String defaultRouting, @Nullable FetchSourceContext defaultFetchSourceContext, @Nullable String defaultPipeline, @Nullable Boolean defaultRequireAlias, boolean allowExplicitIndex, XContentType xContentType, BiConsumer<IndexRequest, String> indexRequestConsumer, Consumer<UpdateRequest> updateRequestConsumer, Consumer<DeleteRequest> deleteRequestConsumer) throws IOException {
    XContent xContent = xContentType.xContent();
    int line = 0;
    int from = 0;
    byte marker = xContent.streamSeparator();
    // Bulk requests can contain a lot of repeated strings for the index, pipeline and routing parameters. This map is used to
    // deduplicate duplicate strings parsed for these parameters. While it does not prevent instantiating the duplicate strings, it
    // reduces their lifetime to the lifetime of this parse call instead of the lifetime of the full bulk request.
    final Map<String, String> stringDeduplicator = new HashMap<>();
    while (true) {
        int nextMarker = findNextMarker(marker, from, data);
        if (nextMarker == -1) {
            break;
        }
        line++;
        // now parse the action
        try (XContentParser parser = createParser(data, xContent, from, nextMarker)) {
            // move pointers
            from = nextMarker + 1;
            // Move to START_OBJECT
            XContentParser.Token token = parser.nextToken();
            if (token == null) {
                continue;
            }
            if (token != XContentParser.Token.START_OBJECT) {
                throw new IllegalArgumentException("Malformed action/metadata line [" + line + "], expected " + XContentParser.Token.START_OBJECT + " but found [" + token + "]");
            }
            // Move to FIELD_NAME, that's the action
            token = parser.nextToken();
            if (token != XContentParser.Token.FIELD_NAME) {
                throw new IllegalArgumentException("Malformed action/metadata line [" + line + "], expected " + XContentParser.Token.FIELD_NAME + " but found [" + token + "]");
            }
            String action = parser.currentName();
            String index = defaultIndex;
            String type = null;
            String id = null;
            String routing = defaultRouting;
            FetchSourceContext fetchSourceContext = defaultFetchSourceContext;
            String opType = null;
            long version = Versions.MATCH_ANY;
            VersionType versionType = VersionType.INTERNAL;
            long ifSeqNo = SequenceNumbers.UNASSIGNED_SEQ_NO;
            long ifPrimaryTerm = UNASSIGNED_PRIMARY_TERM;
            int retryOnConflict = 0;
            String pipeline = defaultPipeline;
            boolean requireAlias = defaultRequireAlias != null && defaultRequireAlias;
            // at this stage, next token can either be END_OBJECT (and use default index and type, with auto generated id)
            // or START_OBJECT which will have another set of parameters
            token = parser.nextToken();
            if (token == XContentParser.Token.START_OBJECT) {
                String currentFieldName = null;
                while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
                    if (token == XContentParser.Token.FIELD_NAME) {
                        currentFieldName = parser.currentName();
                    } else if (token.isValue()) {
                        if (INDEX.match(currentFieldName, parser.getDeprecationHandler())) {
                            if (allowExplicitIndex == false) {
                                throw new IllegalArgumentException("explicit index in bulk is not allowed");
                            }
                            index = stringDeduplicator.computeIfAbsent(parser.text(), Function.identity());
                        } else if (TYPE.match(currentFieldName, parser.getDeprecationHandler())) {
                            if (errorOnType) {
                                throw new IllegalArgumentException("Action/metadata line [" + line + "] contains an unknown parameter [" + currentFieldName + "]");
                            }
                            type = stringDeduplicator.computeIfAbsent(parser.text(), Function.identity());
                        } else if (ID.match(currentFieldName, parser.getDeprecationHandler())) {
                            id = parser.text();
                        } else if (ROUTING.match(currentFieldName, parser.getDeprecationHandler())) {
                            routing = stringDeduplicator.computeIfAbsent(parser.text(), Function.identity());
                        } else if (OP_TYPE.match(currentFieldName, parser.getDeprecationHandler())) {
                            opType = parser.text();
                        } else if (VERSION.match(currentFieldName, parser.getDeprecationHandler())) {
                            version = parser.longValue();
                        } else if (VERSION_TYPE.match(currentFieldName, parser.getDeprecationHandler())) {
                            versionType = VersionType.fromString(parser.text());
                        } else if (IF_SEQ_NO.match(currentFieldName, parser.getDeprecationHandler())) {
                            ifSeqNo = parser.longValue();
                        } else if (IF_PRIMARY_TERM.match(currentFieldName, parser.getDeprecationHandler())) {
                            ifPrimaryTerm = parser.longValue();
                        } else if (RETRY_ON_CONFLICT.match(currentFieldName, parser.getDeprecationHandler())) {
                            retryOnConflict = parser.intValue();
                        } else if (PIPELINE.match(currentFieldName, parser.getDeprecationHandler())) {
                            pipeline = stringDeduplicator.computeIfAbsent(parser.text(), Function.identity());
                        } else if (SOURCE.match(currentFieldName, parser.getDeprecationHandler())) {
                            fetchSourceContext = FetchSourceContext.fromXContent(parser);
                        } else if (REQUIRE_ALIAS.match(currentFieldName, parser.getDeprecationHandler())) {
                            requireAlias = parser.booleanValue();
                        } else {
                            throw new IllegalArgumentException("Action/metadata line [" + line + "] contains an unknown parameter [" + currentFieldName + "]");
                        }
                    } else if (token == XContentParser.Token.START_ARRAY) {
                        throw new IllegalArgumentException("Malformed action/metadata line [" + line + "], expected a simple value for field [" + currentFieldName + "] but found [" + token + "]");
                    } else if (token == XContentParser.Token.START_OBJECT && SOURCE.match(currentFieldName, parser.getDeprecationHandler())) {
                        fetchSourceContext = FetchSourceContext.fromXContent(parser);
                    } else if (token != XContentParser.Token.VALUE_NULL) {
                        throw new IllegalArgumentException("Malformed action/metadata line [" + line + "], expected a simple value for field [" + currentFieldName + "] but found [" + token + "]");
                    }
                }
            } else if (token != XContentParser.Token.END_OBJECT) {
                throw new IllegalArgumentException("Malformed action/metadata line [" + line + "], expected " + XContentParser.Token.START_OBJECT + " or " + XContentParser.Token.END_OBJECT + " but found [" + token + "]");
            }
            if ("delete".equals(action)) {
                deleteRequestConsumer.accept(new DeleteRequest(index).id(id).routing(routing).version(version).versionType(versionType).setIfSeqNo(ifSeqNo).setIfPrimaryTerm(ifPrimaryTerm));
            } else {
                nextMarker = findNextMarker(marker, from, data);
                if (nextMarker == -1) {
                    break;
                }
                line++;
                // of index request.
                if ("index".equals(action)) {
                    if (opType == null) {
                        indexRequestConsumer.accept(new IndexRequest(index).id(id).routing(routing).version(version).versionType(versionType).setPipeline(pipeline).setIfSeqNo(ifSeqNo).setIfPrimaryTerm(ifPrimaryTerm).source(sliceTrimmingCarriageReturn(data, from, nextMarker, xContentType), xContentType).setRequireAlias(requireAlias), type);
                    } else {
                        indexRequestConsumer.accept(new IndexRequest(index).id(id).routing(routing).version(version).versionType(versionType).create("create".equals(opType)).setPipeline(pipeline).setIfSeqNo(ifSeqNo).setIfPrimaryTerm(ifPrimaryTerm).source(sliceTrimmingCarriageReturn(data, from, nextMarker, xContentType), xContentType).setRequireAlias(requireAlias), type);
                    }
                } else if ("create".equals(action)) {
                    indexRequestConsumer.accept(new IndexRequest(index).id(id).routing(routing).version(version).versionType(versionType).create(true).setPipeline(pipeline).setIfSeqNo(ifSeqNo).setIfPrimaryTerm(ifPrimaryTerm).source(sliceTrimmingCarriageReturn(data, from, nextMarker, xContentType), xContentType).setRequireAlias(requireAlias), type);
                } else if ("update".equals(action)) {
                    if (version != Versions.MATCH_ANY || versionType != VersionType.INTERNAL) {
                        throw new IllegalArgumentException("Update requests do not support versioning. " + "Please use `if_seq_no` and `if_primary_term` instead");
                    }
                    UpdateRequest updateRequest = new UpdateRequest().index(index).id(id).routing(routing).retryOnConflict(retryOnConflict).setIfSeqNo(ifSeqNo).setIfPrimaryTerm(ifPrimaryTerm).setRequireAlias(requireAlias).routing(routing);
                    try (XContentParser sliceParser = createParser(sliceTrimmingCarriageReturn(data, from, nextMarker, xContentType), xContent)) {
                        updateRequest.fromXContent(sliceParser);
                    }
                    if (fetchSourceContext != null) {
                        updateRequest.fetchSource(fetchSourceContext);
                    }
                    IndexRequest upsertRequest = updateRequest.upsertRequest();
                    if (upsertRequest != null) {
                        upsertRequest.setPipeline(defaultPipeline);
                    }
                    updateRequestConsumer.accept(updateRequest);
                }
                // move pointers
                from = nextMarker + 1;
            }
        }
    }
}

Example 4 with VersionType

use of org.opensearch.index.VersionType in project OpenSearch by opensearch-project.

the class MultiGetRequest method parseDocuments.

private static void parseDocuments(XContentParser parser, List<Item> items, @Nullable String defaultIndex, @Nullable String[] defaultFields, @Nullable FetchSourceContext defaultFetchSource, @Nullable String defaultRouting, boolean allowExplicitIndex) throws IOException {
    String currentFieldName = null;
    Token token;
    while ((token = parser.nextToken()) != Token.END_ARRAY) {
        if (token != Token.START_OBJECT) {
            throw new IllegalArgumentException("docs array element should include an object");
        }
        String index = defaultIndex;
        String id = null;
        String routing = defaultRouting;
        List<String> storedFields = null;
        long version = Versions.MATCH_ANY;
        VersionType versionType = VersionType.INTERNAL;
        FetchSourceContext fetchSourceContext = FetchSourceContext.FETCH_SOURCE;
        while ((token = parser.nextToken()) != Token.END_OBJECT) {
            if (token == Token.FIELD_NAME) {
                currentFieldName = parser.currentName();
            } else if (token.isValue()) {
                if (INDEX.match(currentFieldName, parser.getDeprecationHandler())) {
                    if (!allowExplicitIndex) {
                        throw new IllegalArgumentException("explicit index in multi get is not allowed");
                    }
                    index = parser.text();
                } else if (ID.match(currentFieldName, parser.getDeprecationHandler())) {
                    id = parser.text();
                } else if (ROUTING.match(currentFieldName, parser.getDeprecationHandler())) {
                    routing = parser.text();
                } else if (FIELDS.match(currentFieldName, parser.getDeprecationHandler())) {
                    throw new ParsingException(parser.getTokenLocation(), "Unsupported field [fields] used, expected [stored_fields] instead");
                } else if (STORED_FIELDS.match(currentFieldName, parser.getDeprecationHandler())) {
                    storedFields = new ArrayList<>();
                    storedFields.add(parser.text());
                } else if (VERSION.match(currentFieldName, parser.getDeprecationHandler())) {
                    version = parser.longValue();
                } else if (VERSION_TYPE.match(currentFieldName, parser.getDeprecationHandler())) {
                    versionType = VersionType.fromString(parser.text());
                } else if (SOURCE.match(currentFieldName, parser.getDeprecationHandler())) {
                    if (parser.isBooleanValue()) {
                        fetchSourceContext = new FetchSourceContext(parser.booleanValue(), fetchSourceContext.includes(), fetchSourceContext.excludes());
                    } else if (token == Token.VALUE_STRING) {
                        fetchSourceContext = new FetchSourceContext(fetchSourceContext.fetchSource(), new String[] { parser.text() }, fetchSourceContext.excludes());
                    } else {
                        throw new OpenSearchParseException("illegal type for _source: [{}]", token);
                    }
                } else {
                    throw new OpenSearchParseException("failed to parse multi get request. unknown field [{}]", currentFieldName);
                }
            } else if (token == Token.START_ARRAY) {
                if (FIELDS.match(currentFieldName, parser.getDeprecationHandler())) {
                    throw new ParsingException(parser.getTokenLocation(), "Unsupported field [fields] used, expected [stored_fields] instead");
                } else if (STORED_FIELDS.match(currentFieldName, parser.getDeprecationHandler())) {
                    storedFields = new ArrayList<>();
                    while ((token = parser.nextToken()) != Token.END_ARRAY) {
                        storedFields.add(parser.text());
                    }
                } else if (SOURCE.match(currentFieldName, parser.getDeprecationHandler())) {
                    ArrayList<String> includes = new ArrayList<>();
                    while ((token = parser.nextToken()) != Token.END_ARRAY) {
                        includes.add(parser.text());
                    }
                    fetchSourceContext = new FetchSourceContext(fetchSourceContext.fetchSource(), includes.toArray(Strings.EMPTY_ARRAY), fetchSourceContext.excludes());
                }
            } else if (token == Token.START_OBJECT) {
                if (SOURCE.match(currentFieldName, parser.getDeprecationHandler())) {
                    List<String> currentList = null, includes = null, excludes = null;
                    while ((token = parser.nextToken()) != Token.END_OBJECT) {
                        if (token == Token.FIELD_NAME) {
                            currentFieldName = parser.currentName();
                            if ("includes".equals(currentFieldName) || "include".equals(currentFieldName)) {
                                currentList = includes != null ? includes : (includes = new ArrayList<>(2));
                            } else if ("excludes".equals(currentFieldName) || "exclude".equals(currentFieldName)) {
                                currentList = excludes != null ? excludes : (excludes = new ArrayList<>(2));
                            } else {
                                throw new OpenSearchParseException("source definition may not contain [{}]", parser.text());
                            }
                        } else if (token == Token.START_ARRAY) {
                            while ((token = parser.nextToken()) != Token.END_ARRAY) {
                                currentList.add(parser.text());
                            }
                        } else if (token.isValue()) {
                            currentList.add(parser.text());
                        } else {
                            throw new OpenSearchParseException("unexpected token while parsing source settings");
                        }
                    }
                    fetchSourceContext = new FetchSourceContext(fetchSourceContext.fetchSource(), includes == null ? Strings.EMPTY_ARRAY : includes.toArray(new String[includes.size()]), excludes == null ? Strings.EMPTY_ARRAY : excludes.toArray(new String[excludes.size()]));
                }
            }
        }
        String[] aFields;
        if (storedFields != null) {
            aFields = storedFields.toArray(new String[storedFields.size()]);
        } else {
            aFields = defaultFields;
        }
        items.add(new Item(index, id).routing(routing).storedFields(aFields).version(version).versionType(versionType).fetchSourceContext(fetchSourceContext == FetchSourceContext.FETCH_SOURCE ? defaultFetchSource : fetchSourceContext));
    }
}

Example 5 with VersionType

use of org.opensearch.index.VersionType in project OpenSearch by opensearch-project.

the class IndexShard method applyTranslogOperation.

private Engine.Result applyTranslogOperation(Engine engine, Translog.Operation operation, Engine.Operation.Origin origin) throws IOException {
    // If a translog op is replayed on the primary (eg. ccr), we need to use external instead of null for its version type.
    final VersionType versionType = (origin == Engine.Operation.Origin.PRIMARY) ? VersionType.EXTERNAL : null;
    final Engine.Result result;
    switch(operation.opType()) {
        case INDEX:
            final Translog.Index index = (Translog.Index) operation;
            // we set canHaveDuplicates to true all the time such that we de-optimze the translog case and ensure that all
            // autoGeneratedID docs that are coming from the primary are updated correctly.
            result = applyIndexOperation(engine, index.seqNo(), index.primaryTerm(), index.version(), versionType, UNASSIGNED_SEQ_NO, 0, index.getAutoGeneratedIdTimestamp(), true, origin, new SourceToParse(shardId.getIndexName(), index.type(), index.id(), index.source(), XContentHelper.xContentType(index.source()), index.routing()));
            break;
        case DELETE:
            final Translog.Delete delete = (Translog.Delete) operation;
            result = applyDeleteOperation(engine, delete.seqNo(), delete.primaryTerm(), delete.version(), delete.type(), delete.id(), versionType, UNASSIGNED_SEQ_NO, 0, origin);
            break;
        case NO_OP:
            final Translog.NoOp noOp = (Translog.NoOp) operation;
            result = markSeqNoAsNoop(engine, noOp.seqNo(), noOp.primaryTerm(), noOp.reason(), origin);
            break;
        default:
            throw new IllegalStateException("No operation defined for [" + operation + "]");
    }
    return result;
}