Examples with VersionConflictEngineException org.elasticsearch.index.engine.VersionConflictEngineException used on opensource projects

Example 6 with VersionConflictEngineException

use of org.elasticsearch.index.engine.VersionConflictEngineException in project elasticsearch by elastic.

the class TransportShardBulkAction method shardOperationOnReplica.

@Override
public WriteReplicaResult<BulkShardRequest> shardOperationOnReplica(BulkShardRequest request, IndexShard replica) throws Exception {
    Translog.Location location = null;
    for (int i = 0; i < request.items().length; i++) {
        BulkItemRequest item = request.items()[i];
        if (shouldExecuteReplicaItem(item, i)) {
            DocWriteRequest docWriteRequest = item.request();
            DocWriteResponse primaryResponse = item.getPrimaryResponse().getResponse();
            final Engine.Result operationResult;
            try {
                switch(docWriteRequest.opType()) {
                    case CREATE:
                    case INDEX:
                        operationResult = executeIndexRequestOnReplica(primaryResponse, (IndexRequest) docWriteRequest, replica);
                        break;
                    case DELETE:
                        operationResult = executeDeleteRequestOnReplica(primaryResponse, (DeleteRequest) docWriteRequest, replica);
                        break;
                    default:
                        throw new IllegalStateException("Unexpected request operation type on replica: " + docWriteRequest.opType().getLowercase());
                }
                if (operationResult.hasFailure()) {
                    // check if any transient write operation failures should be bubbled up
                    Exception failure = operationResult.getFailure();
                    assert failure instanceof VersionConflictEngineException || failure instanceof MapperParsingException : "expected any one of [version conflict, mapper parsing, engine closed, index shard closed]" + " failures. got " + failure;
                    if (!TransportActions.isShardNotAvailableException(failure)) {
                        throw failure;
                    }
                } else {
                    location = locationToSync(location, operationResult.getTranslogLocation());
                }
            } catch (Exception e) {
                // so we will fail the shard
                if (!TransportActions.isShardNotAvailableException(e)) {
                    throw e;
                }
            }
        }
    }
    return new WriteReplicaResult<>(request, location, null, replica, logger);
}

Example 7 with VersionConflictEngineException

use of org.elasticsearch.index.engine.VersionConflictEngineException in project elasticsearch by elastic.

the class TransportShardBulkAction method executeUpdateRequest.

/**
     * Executes update request, delegating to a index or delete operation after translation,
     * handles retries on version conflict and constructs update response
     * NOTE: reassigns bulk item request at <code>requestIndex</code> for replicas to
     * execute translated update request (NOOP update is an exception). NOOP updates are
     * indicated by returning a <code>null</code> operation in {@link BulkItemResultHolder}
     * */
private static BulkItemResultHolder executeUpdateRequest(UpdateRequest updateRequest, IndexShard primary, IndexMetaData metaData, BulkShardRequest request, int requestIndex, UpdateHelper updateHelper, LongSupplier nowInMillis, final MappingUpdatePerformer mappingUpdater) throws Exception {
    Engine.Result updateOperationResult = null;
    UpdateResponse updateResponse = null;
    BulkItemRequest replicaRequest = request.items()[requestIndex];
    int maxAttempts = updateRequest.retryOnConflict();
    for (int attemptCount = 0; attemptCount <= maxAttempts; attemptCount++) {
        final UpdateHelper.Result translate;
        // translate update request
        try {
            translate = updateHelper.prepare(updateRequest, primary, nowInMillis);
        } catch (Exception failure) {
            // we may fail translating a update to index or delete operation
            // we use index result to communicate failure while translating update request
            updateOperationResult = new Engine.IndexResult(failure, updateRequest.version(), SequenceNumbersService.UNASSIGNED_SEQ_NO);
            // out of retry loop
            break;
        }
        // execute translated update request
        switch(translate.getResponseResult()) {
            case CREATED:
            case UPDATED:
                IndexRequest indexRequest = translate.action();
                MappingMetaData mappingMd = metaData.mappingOrDefault(indexRequest.type());
                indexRequest.process(mappingMd, request.index());
                updateOperationResult = executeIndexRequestOnPrimary(indexRequest, primary, mappingUpdater);
                break;
            case DELETED:
                DeleteRequest deleteRequest = translate.action();
                updateOperationResult = executeDeleteRequestOnPrimary(deleteRequest, primary);
                break;
            case NOOP:
                primary.noopUpdate(updateRequest.type());
                break;
            default:
                throw new IllegalStateException("Illegal update operation " + translate.getResponseResult());
        }
        if (updateOperationResult == null) {
            // this is a noop operation
            updateResponse = translate.action();
            // out of retry loop
            break;
        } else if (updateOperationResult.hasFailure() == false) {
            // set translated update (index/delete) request for replica execution in bulk items
            switch(updateOperationResult.getOperationType()) {
                case INDEX:
                    IndexRequest updateIndexRequest = translate.action();
                    final IndexResponse indexResponse = new IndexResponse(primary.shardId(), updateIndexRequest.type(), updateIndexRequest.id(), updateOperationResult.getSeqNo(), updateOperationResult.getVersion(), ((Engine.IndexResult) updateOperationResult).isCreated());
                    BytesReference indexSourceAsBytes = updateIndexRequest.source();
                    updateResponse = new UpdateResponse(indexResponse.getShardInfo(), indexResponse.getShardId(), indexResponse.getType(), indexResponse.getId(), indexResponse.getSeqNo(), indexResponse.getVersion(), indexResponse.getResult());
                    if ((updateRequest.fetchSource() != null && updateRequest.fetchSource().fetchSource()) || (updateRequest.fields() != null && updateRequest.fields().length > 0)) {
                        Tuple<XContentType, Map<String, Object>> sourceAndContent = XContentHelper.convertToMap(indexSourceAsBytes, true, updateIndexRequest.getContentType());
                        updateResponse.setGetResult(updateHelper.extractGetResult(updateRequest, request.index(), indexResponse.getVersion(), sourceAndContent.v2(), sourceAndContent.v1(), indexSourceAsBytes));
                    }
                    // set translated request as replica request
                    replicaRequest = new BulkItemRequest(request.items()[requestIndex].id(), updateIndexRequest);
                    break;
                case DELETE:
                    DeleteRequest updateDeleteRequest = translate.action();
                    DeleteResponse deleteResponse = new DeleteResponse(primary.shardId(), updateDeleteRequest.type(), updateDeleteRequest.id(), updateOperationResult.getSeqNo(), updateOperationResult.getVersion(), ((Engine.DeleteResult) updateOperationResult).isFound());
                    updateResponse = new UpdateResponse(deleteResponse.getShardInfo(), deleteResponse.getShardId(), deleteResponse.getType(), deleteResponse.getId(), deleteResponse.getSeqNo(), deleteResponse.getVersion(), deleteResponse.getResult());
                    updateResponse.setGetResult(updateHelper.extractGetResult(updateRequest, request.index(), deleteResponse.getVersion(), translate.updatedSourceAsMap(), translate.updateSourceContentType(), null));
                    // set translated request as replica request
                    replicaRequest = new BulkItemRequest(request.items()[requestIndex].id(), updateDeleteRequest);
                    break;
            }
            assert updateOperationResult.getSeqNo() != SequenceNumbersService.UNASSIGNED_SEQ_NO;
            // out of retry loop
            break;
        } else if (updateOperationResult.getFailure() instanceof VersionConflictEngineException == false) {
            // out of retry loop
            break;
        }
    }
    return new BulkItemResultHolder(updateResponse, updateOperationResult, replicaRequest);
}

Example 8 with VersionConflictEngineException

use of org.elasticsearch.index.engine.VersionConflictEngineException in project elasticsearch by elastic.

the class TransportShardBulkActionTests method testUpdateReplicaRequestWithConflictFailure.

public void testUpdateReplicaRequestWithConflictFailure() throws Exception {
    DocWriteRequest writeRequest = new IndexRequest("index", "type", "id").source(Requests.INDEX_CONTENT_TYPE, "field", "value");
    BulkItemRequest replicaRequest = new BulkItemRequest(0, writeRequest);
    Exception err = new VersionConflictEngineException(shardId, "type", "id", "I'm conflicted <(;_;)>");
    Engine.IndexResult indexResult = new Engine.IndexResult(err, 0, 0);
    BulkItemResultHolder failedResults = new BulkItemResultHolder(null, indexResult, replicaRequest);
    Translog.Location location = new Translog.Location(0, 0, 0);
    BulkItemRequest[] items = new BulkItemRequest[0];
    BulkShardRequest bulkShardRequest = new BulkShardRequest(shardId, RefreshPolicy.NONE, items);
    Translog.Location newLocation = TransportShardBulkAction.updateReplicaRequest(failedResults, DocWriteRequest.OpType.UPDATE, location, bulkShardRequest);
    BulkItemResponse primaryResponse = replicaRequest.getPrimaryResponse();
    // Since this was not a conflict failure, the primary response
    // should be filled out with the failure information
    assertThat(newLocation, equalTo(location));
    assertThat(primaryResponse.getItemId(), equalTo(0));
    assertThat(primaryResponse.getId(), equalTo("id"));
    assertThat(primaryResponse.getOpType(), equalTo(DocWriteRequest.OpType.INDEX));
    assertTrue(primaryResponse.isFailed());
    assertThat(primaryResponse.getFailureMessage(), containsString("I'm conflicted <(;_;)>"));
    BulkItemResponse.Failure failure = primaryResponse.getFailure();
    assertThat(failure.getIndex(), equalTo("index"));
    assertThat(failure.getType(), equalTo("type"));
    assertThat(failure.getId(), equalTo("id"));
    assertThat(failure.getCause(), equalTo(err));
    assertThat(failure.getStatus(), equalTo(RestStatus.CONFLICT));
}

Example 9 with VersionConflictEngineException

use of org.elasticsearch.index.engine.VersionConflictEngineException in project elasticsearch by elastic.

the class UpdateByQueryWhileModifyingTests method testUpdateWhileReindexing.

public void testUpdateWhileReindexing() throws Exception {
    AtomicReference<String> value = new AtomicReference<>(randomSimpleString(random()));
    indexRandom(true, client().prepareIndex("test", "test", "test").setSource("test", value.get()));
    AtomicReference<Exception> failure = new AtomicReference<>();
    AtomicBoolean keepUpdating = new AtomicBoolean(true);
    Thread updater = new Thread(() -> {
        while (keepUpdating.get()) {
            try {
                BulkByScrollResponse response = updateByQuery().source("test").refresh(true).abortOnVersionConflict(false).get();
                assertThat(response, matcher().updated(either(equalTo(0L)).or(equalTo(1L))).versionConflicts(either(equalTo(0L)).or(equalTo(1L))));
            } catch (Exception e) {
                failure.set(e);
            }
        }
    });
    updater.start();
    try {
        for (int i = 0; i < MAX_MUTATIONS; i++) {
            GetResponse get = client().prepareGet("test", "test", "test").get();
            assertEquals(value.get(), get.getSource().get("test"));
            value.set(randomSimpleString(random()));
            IndexRequestBuilder index = client().prepareIndex("test", "test", "test").setSource("test", value.get()).setRefreshPolicy(IMMEDIATE);
            /*
                 * Update by query increments the version number so concurrent
                 * indexes might get version conflict exceptions so we just
                 * blindly retry.
                 */
            int attempts = 0;
            while (true) {
                attempts++;
                try {
                    index.setVersion(get.getVersion()).get();
                    break;
                } catch (VersionConflictEngineException e) {
                    if (attempts >= MAX_ATTEMPTS) {
                        throw new RuntimeException("Failed to index after [" + MAX_ATTEMPTS + "] attempts. Too many version conflicts!");
                    }
                    logger.info("Caught expected version conflict trying to perform mutation number [{}] with version [{}] " + "on attempt [{}]. Retrying.", i, get.getVersion(), attempts);
                    get = client().prepareGet("test", "test", "test").get();
                }
            }
        }
    } finally {
        keepUpdating.set(false);
        updater.join(TimeUnit.SECONDS.toMillis(10));
        if (failure.get() != null) {
            throw new RuntimeException(failure.get());
        }
    }
}

Example 10 with VersionConflictEngineException

use of org.elasticsearch.index.engine.VersionConflictEngineException in project elasticsearch by elastic.

the class SimpleVersioningIT method testRandomIDsAndVersions.

public void testRandomIDsAndVersions() throws Exception {
    createIndex("test");
    ensureGreen();
    // TODO: sometimes use _bulk API
    // TODO: test non-aborting exceptions (Rob suggested field where positions overflow)
    // TODO: not great we don't test deletes GC here:
    // We test deletes, but can't rely on wall-clock delete GC:
    HashMap<String, Object> newSettings = new HashMap<>();
    newSettings.put("index.gc_deletes", "1000000h");
    assertAcked(client().admin().indices().prepareUpdateSettings("test").setSettings(newSettings).execute().actionGet());
    Random random = random();
    // Generate random IDs:
    IDSource idSource = getRandomIDs();
    Set<String> idsSet = new HashSet<>();
    String idPrefix;
    if (randomBoolean()) {
        idPrefix = "";
    } else {
        idPrefix = TestUtil.randomSimpleString(random);
        logger.debug("--> use id prefix {}", idPrefix);
    }
    int numIDs;
    if (TEST_NIGHTLY) {
        numIDs = scaledRandomIntBetween(300, 1000);
    } else {
        numIDs = scaledRandomIntBetween(50, 100);
    }
    while (idsSet.size() < numIDs) {
        idsSet.add(idPrefix + idSource.next());
    }
    String[] ids = idsSet.toArray(new String[numIDs]);
    boolean useMonotonicVersion = randomBoolean();
    // Attach random versions to them:
    long version = 0;
    final IDAndVersion[] idVersions = new IDAndVersion[TestUtil.nextInt(random, numIDs / 2, numIDs * (TEST_NIGHTLY ? 8 : 2))];
    final Map<String, IDAndVersion> truth = new HashMap<>();
    logger.debug("--> use {} ids; {} operations", numIDs, idVersions.length);
    for (int i = 0; i < idVersions.length; i++) {
        if (useMonotonicVersion) {
            version += TestUtil.nextInt(random, 1, 10);
        } else {
            version = random.nextLong() & 0x3fffffffffffffffL;
        }
        idVersions[i] = new IDAndVersion();
        idVersions[i].id = ids[random.nextInt(numIDs)];
        idVersions[i].version = version;
        // 20% of the time we delete:
        idVersions[i].delete = random.nextInt(5) == 2;
        IDAndVersion curVersion = truth.get(idVersions[i].id);
        if (curVersion == null || idVersions[i].version > curVersion.version) {
            // Save highest version per id:
            truth.put(idVersions[i].id, idVersions[i]);
        }
    }
    // Shuffle
    for (int i = idVersions.length - 1; i > 0; i--) {
        int index = random.nextInt(i + 1);
        IDAndVersion x = idVersions[index];
        idVersions[index] = idVersions[i];
        idVersions[i] = x;
    }
    for (IDAndVersion idVersion : idVersions) {
        logger.debug("--> id={} version={} delete?={} truth?={}", idVersion.id, idVersion.version, idVersion.delete, truth.get(idVersion.id) == idVersion);
    }
    final AtomicInteger upto = new AtomicInteger();
    final CountDownLatch startingGun = new CountDownLatch(1);
    Thread[] threads = new Thread[TestUtil.nextInt(random, 1, TEST_NIGHTLY ? 20 : 5)];
    final long startTime = System.nanoTime();
    for (int i = 0; i < threads.length; i++) {
        final int threadID = i;
        threads[i] = new Thread() {

            @Override
            public void run() {
                try {
                    //final Random threadRandom = RandomizedContext.current().getRandom();
                    final Random threadRandom = random();
                    startingGun.await();
                    while (true) {
                        // TODO: sometimes use bulk:
                        int index = upto.getAndIncrement();
                        if (index >= idVersions.length) {
                            break;
                        }
                        if (index % 100 == 0) {
                            logger.trace("{}: index={}", Thread.currentThread().getName(), index);
                        }
                        IDAndVersion idVersion = idVersions[index];
                        String id = idVersion.id;
                        idVersion.threadID = threadID;
                        idVersion.indexStartTime = System.nanoTime() - startTime;
                        long version = idVersion.version;
                        if (idVersion.delete) {
                            try {
                                idVersion.response = client().prepareDelete("test", "type", id).setVersion(version).setVersionType(VersionType.EXTERNAL).execute().actionGet();
                            } catch (VersionConflictEngineException vcee) {
                                // OK: our version is too old
                                assertThat(version, lessThanOrEqualTo(truth.get(id).version));
                                idVersion.versionConflict = true;
                            }
                        } else {
                            try {
                                idVersion.response = client().prepareIndex("test", "type", id).setSource("foo", "bar").setVersion(version).setVersionType(VersionType.EXTERNAL).get();
                            } catch (VersionConflictEngineException vcee) {
                                // OK: our version is too old
                                assertThat(version, lessThanOrEqualTo(truth.get(id).version));
                                idVersion.versionConflict = true;
                            }
                        }
                        idVersion.indexFinishTime = System.nanoTime() - startTime;
                        if (threadRandom.nextInt(100) == 7) {
                            logger.trace("--> {}: TEST: now refresh at {}", threadID, System.nanoTime() - startTime);
                            refresh();
                            logger.trace("--> {}: TEST: refresh done at {}", threadID, System.nanoTime() - startTime);
                        }
                        if (threadRandom.nextInt(100) == 7) {
                            logger.trace("--> {}: TEST: now flush at {}", threadID, System.nanoTime() - startTime);
                            flush();
                            logger.trace("--> {}: TEST: flush done at {}", threadID, System.nanoTime() - startTime);
                        }
                    }
                } catch (Exception e) {
                    throw new RuntimeException(e);
                }
            }
        };
        threads[i].start();
    }
    startingGun.countDown();
    for (Thread thread : threads) {
        thread.join();
    }
    // Verify against truth:
    boolean failed = false;
    for (String id : ids) {
        long expected;
        IDAndVersion idVersion = truth.get(id);
        if (idVersion != null && idVersion.delete == false) {
            expected = idVersion.version;
        } else {
            expected = -1;
        }
        long actualVersion = client().prepareGet("test", "type", id).execute().actionGet().getVersion();
        if (actualVersion != expected) {
            logger.error("--> FAILED: idVersion={} actualVersion= {}", idVersion, actualVersion);
            failed = true;
        }
    }
    if (failed) {
        StringBuilder sb = new StringBuilder();
        for (int i = 0; i < idVersions.length; i++) {
            sb.append("i=").append(i).append(" ").append(idVersions[i]).append(System.lineSeparator());
        }
        logger.error("All versions: {}", sb);
        fail("wrong versions for some IDs");
    }
}