Examples with ShardInfo org.elasticsearch.action.support.replication.ReplicationResponse.ShardInfo used on opensource projects

Example 1 with ShardInfo

use of org.elasticsearch.action.support.replication.ReplicationResponse.ShardInfo in project elasticsearch by elastic.

the class ReplicationOperationTests method testReplication.

public void testReplication() throws Exception {
    final String index = "test";
    final ShardId shardId = new ShardId(index, "_na_", 0);
    ClusterState state = stateWithActivePrimary(index, true, randomInt(5));
    IndexMetaData indexMetaData = state.getMetaData().index(index);
    final long primaryTerm = indexMetaData.primaryTerm(0);
    final IndexShardRoutingTable indexShardRoutingTable = state.getRoutingTable().shardRoutingTable(shardId);
    ShardRouting primaryShard = indexShardRoutingTable.primaryShard();
    if (primaryShard.relocating() && randomBoolean()) {
        // simulate execution of the replication phase on the relocation target node after relocation source was marked as relocated
        state = ClusterState.builder(state).nodes(DiscoveryNodes.builder(state.nodes()).localNodeId(primaryShard.relocatingNodeId())).build();
        primaryShard = primaryShard.getTargetRelocatingShard();
    }
    // add a few in-sync allocation ids that don't have corresponding routing entries
    Set<String> staleAllocationIds = Sets.newHashSet(generateRandomStringArray(4, 10, false));
    state = ClusterState.builder(state).metaData(MetaData.builder(state.metaData()).put(IndexMetaData.builder(indexMetaData).putInSyncAllocationIds(0, Sets.union(indexMetaData.inSyncAllocationIds(0), staleAllocationIds)))).build();
    final Set<ShardRouting> expectedReplicas = getExpectedReplicas(shardId, state);
    final Map<ShardRouting, Exception> expectedFailures = new HashMap<>();
    final Set<ShardRouting> expectedFailedShards = new HashSet<>();
    for (ShardRouting replica : expectedReplicas) {
        if (randomBoolean()) {
            Exception t;
            boolean criticalFailure = randomBoolean();
            if (criticalFailure) {
                t = new CorruptIndexException("simulated", (String) null);
            } else {
                t = new IndexShardNotStartedException(shardId, IndexShardState.RECOVERING);
            }
            logger.debug("--> simulating failure on {} with [{}]", replica, t.getClass().getSimpleName());
            expectedFailures.put(replica, t);
            if (criticalFailure) {
                expectedFailedShards.add(replica);
            }
        }
    }
    Request request = new Request(shardId);
    PlainActionFuture<TestPrimary.Result> listener = new PlainActionFuture<>();
    final ClusterState finalState = state;
    final TestReplicaProxy replicasProxy = new TestReplicaProxy(expectedFailures);
    final TestPrimary primary = new TestPrimary(primaryShard, primaryTerm);
    final TestReplicationOperation op = new TestReplicationOperation(request, primary, listener, replicasProxy, () -> finalState);
    op.execute();
    assertThat(request.primaryTerm(), equalTo(primaryTerm));
    assertThat("request was not processed on primary", request.processedOnPrimary.get(), equalTo(true));
    assertThat(request.processedOnReplicas, equalTo(expectedReplicas));
    assertThat(replicasProxy.failedReplicas, equalTo(expectedFailedShards));
    assertThat(replicasProxy.markedAsStaleCopies, equalTo(staleAllocationIds));
    assertTrue("listener is not marked as done", listener.isDone());
    ShardInfo shardInfo = listener.actionGet().getShardInfo();
    assertThat(shardInfo.getFailed(), equalTo(expectedFailedShards.size()));
    assertThat(shardInfo.getFailures(), arrayWithSize(expectedFailedShards.size()));
    assertThat(shardInfo.getSuccessful(), equalTo(1 + expectedReplicas.size() - expectedFailures.size()));
    final List<ShardRouting> unassignedShards = indexShardRoutingTable.shardsWithState(ShardRoutingState.UNASSIGNED);
    final int totalShards = 1 + expectedReplicas.size() + unassignedShards.size();
    assertThat(shardInfo.getTotal(), equalTo(totalShards));
    assertThat(primary.knownLocalCheckpoints.remove(primaryShard.allocationId().getId()), equalTo(primary.localCheckpoint));
    assertThat(primary.knownLocalCheckpoints, equalTo(replicasProxy.generatedLocalCheckpoints));
}

Example 2 with ShardInfo

use of org.elasticsearch.action.support.replication.ReplicationResponse.ShardInfo in project elasticsearch by elastic.

the class TransportShardBulkAction method updateReplicaRequest.

// Visible for unit testing
static Translog.Location updateReplicaRequest(BulkItemResultHolder bulkItemResult, final DocWriteRequest.OpType opType, final Translog.Location originalLocation, BulkShardRequest request) {
    final Engine.Result operationResult = bulkItemResult.operationResult;
    final DocWriteResponse response = bulkItemResult.response;
    final BulkItemRequest replicaRequest = bulkItemResult.replicaRequest;
    if (operationResult == null) {
        // in case of noop update operation
        assert response.getResult() == DocWriteResponse.Result.NOOP : "only noop updates can have a null operation";
        replicaRequest.setPrimaryResponse(new BulkItemResponse(replicaRequest.id(), opType, response));
        return originalLocation;
    } else if (operationResult.hasFailure() == false) {
        BulkItemResponse primaryResponse = new BulkItemResponse(replicaRequest.id(), opType, response);
        replicaRequest.setPrimaryResponse(primaryResponse);
        // set a blank ShardInfo so we can safely send it to the replicas. We won't use it in the real response though.
        primaryResponse.getResponse().setShardInfo(new ShardInfo());
        // The operation was successful, advance the translog
        return locationToSync(originalLocation, operationResult.getTranslogLocation());
    } else {
        DocWriteRequest docWriteRequest = replicaRequest.request();
        Exception failure = operationResult.getFailure();
        if (isConflictException(failure)) {
            logger.trace((Supplier<?>) () -> new ParameterizedMessage("{} failed to execute bulk item ({}) {}", request.shardId(), docWriteRequest.opType().getLowercase(), request), failure);
        } else {
            logger.debug((Supplier<?>) () -> new ParameterizedMessage("{} failed to execute bulk item ({}) {}", request.shardId(), docWriteRequest.opType().getLowercase(), request), failure);
        }
        // then just use the response we got from the failed execution
        if (replicaRequest.getPrimaryResponse() == null || isConflictException(failure) == false) {
            replicaRequest.setPrimaryResponse(new BulkItemResponse(replicaRequest.id(), docWriteRequest.opType(), // concrete index instead of an alias if used!
            new BulkItemResponse.Failure(request.index(), docWriteRequest.type(), docWriteRequest.id(), failure)));
        }
        return originalLocation;
    }
}

Example 3 with ShardInfo

use of org.elasticsearch.action.support.replication.ReplicationResponse.ShardInfo in project elasticsearch by elastic.

the class DocWriteResponseTests method testToXContentDoesntIncludeForcedRefreshUnlessForced.

/**
     * Tests that {@link DocWriteResponse#toXContent(XContentBuilder, ToXContent.Params)} doesn't include {@code forced_refresh} unless it
     * is true. We can't assert this in the yaml tests because "not found" is also "false" there....
     */
public void testToXContentDoesntIncludeForcedRefreshUnlessForced() throws IOException {
    DocWriteResponse response = new DocWriteResponse(new ShardId("index", "uuid", 0), "type", "id", SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, Result.CREATED) {
    };
    response.setShardInfo(new ShardInfo(1, 1));
    response.setForcedRefresh(false);
    try (XContentBuilder builder = JsonXContent.contentBuilder()) {
        response.toXContent(builder, ToXContent.EMPTY_PARAMS);
        try (XContentParser parser = createParser(JsonXContent.jsonXContent, builder.bytes())) {
            assertThat(parser.map(), not(hasKey("forced_refresh")));
        }
    }
    response.setForcedRefresh(true);
    try (XContentBuilder builder = JsonXContent.contentBuilder()) {
        response.toXContent(builder, ToXContent.EMPTY_PARAMS);
        try (XContentParser parser = createParser(JsonXContent.jsonXContent, builder.bytes())) {
            assertThat(parser.map(), hasEntry("forced_refresh", true));
        }
    }
}

Example 4 with ShardInfo

use of org.elasticsearch.action.support.replication.ReplicationResponse.ShardInfo in project metron by apache.

the class ElasticsearchDao method update.

@Override
public void update(Document update, Optional<String> index) throws IOException {
    String indexPostfix = ElasticsearchUtils.getIndexFormat(accessConfig.getGlobalConfigSupplier().get()).format(new Date());
    String sensorType = update.getSensorType();
    String indexName = getIndexName(update, index, indexPostfix);
    IndexRequest indexRequest = buildIndexRequest(update, sensorType, indexName);
    try {
        IndexResponse response = client.index(indexRequest).get();
        ShardInfo shardInfo = response.getShardInfo();
        int failed = shardInfo.getFailed();
        if (failed > 0) {
            throw new IOException("ElasticsearchDao index failed: " + Arrays.toString(shardInfo.getFailures()));
        }
    } catch (Exception e) {
        throw new IOException(e.getMessage(), e);
    }
}

Example 5 with ShardInfo

use of org.elasticsearch.action.support.replication.ReplicationResponse.ShardInfo in project crate by crate.

the class ReplicationOperationTests method testReplication.

@Test
public void testReplication() throws Exception {
    final String index = "test";
    final ShardId shardId = new ShardId(index, "_na_", 0);
    ClusterState initialState = stateWithActivePrimary(index, true, randomInt(5));
    IndexMetadata indexMetadata = initialState.getMetadata().index(index);
    final long primaryTerm = indexMetadata.primaryTerm(0);
    final IndexShardRoutingTable indexShardRoutingTable = initialState.getRoutingTable().shardRoutingTable(shardId);
    ShardRouting primaryShard = indexShardRoutingTable.primaryShard();
    if (primaryShard.relocating() && randomBoolean()) {
        // simulate execution of the replication phase on the relocation target node after relocation source was marked as relocated
        initialState = ClusterState.builder(initialState).nodes(DiscoveryNodes.builder(initialState.nodes()).localNodeId(primaryShard.relocatingNodeId())).build();
        primaryShard = primaryShard.getTargetRelocatingShard();
    }
    // add a few in-sync allocation ids that don't have corresponding routing entries
    final Set<String> staleAllocationIds = Set.of(generateRandomStringArray(4, 10, false));
    final Set<String> inSyncAllocationIds = Sets.union(indexMetadata.inSyncAllocationIds(0), staleAllocationIds);
    final Set<String> trackedShards = new HashSet<>();
    final Set<String> untrackedShards = new HashSet<>();
    addTrackingInfo(indexShardRoutingTable, primaryShard, trackedShards, untrackedShards);
    trackedShards.addAll(staleAllocationIds);
    final ReplicationGroup replicationGroup = new ReplicationGroup(indexShardRoutingTable, inSyncAllocationIds, trackedShards);
    final Set<ShardRouting> expectedReplicas = getExpectedReplicas(shardId, initialState, trackedShards);
    final Map<ShardRouting, Exception> simulatedFailures = new HashMap<>();
    final Map<ShardRouting, Exception> reportedFailures = new HashMap<>();
    for (ShardRouting replica : expectedReplicas) {
        if (randomBoolean()) {
            Exception t;
            boolean criticalFailure = randomBoolean();
            if (criticalFailure) {
                t = new CorruptIndexException("simulated", (String) null);
                reportedFailures.put(replica, t);
            } else {
                t = new IndexShardNotStartedException(shardId, IndexShardState.RECOVERING);
            }
            logger.debug("--> simulating failure on {} with [{}]", replica, t.getClass().getSimpleName());
            simulatedFailures.put(replica, t);
        }
    }
    Request request = new Request(shardId);
    PlainActionFuture<TestPrimary.Result> listener = new PlainActionFuture<>();
    final TestReplicaProxy replicasProxy = new TestReplicaProxy(simulatedFailures);
    final TestPrimary primary = new TestPrimary(primaryShard, () -> replicationGroup);
    final TestReplicationOperation op = new TestReplicationOperation(request, primary, listener, replicasProxy, primaryTerm);
    op.execute();
    assertThat("request was not processed on primary", request.processedOnPrimary.get(), equalTo(true));
    assertThat(request.processedOnReplicas, equalTo(expectedReplicas));
    assertThat(replicasProxy.failedReplicas, equalTo(simulatedFailures.keySet()));
    assertThat(replicasProxy.markedAsStaleCopies, equalTo(staleAllocationIds));
    assertThat("post replication operations not run on primary", request.runPostReplicationActionsOnPrimary.get(), equalTo(true));
    assertTrue("listener is not marked as done", listener.isDone());
    ShardInfo shardInfo = listener.actionGet().getShardInfo();
    assertThat(shardInfo.getFailed(), equalTo(reportedFailures.size()));
    assertThat(shardInfo.getFailures(), arrayWithSize(reportedFailures.size()));
    assertThat(shardInfo.getSuccessful(), equalTo(1 + expectedReplicas.size() - simulatedFailures.size()));
    final List<ShardRouting> unassignedShards = indexShardRoutingTable.shardsWithState(ShardRoutingState.UNASSIGNED);
    final int totalShards = 1 + expectedReplicas.size() + unassignedShards.size() + untrackedShards.size();
    assertThat(replicationGroup.toString(), shardInfo.getTotal(), equalTo(totalShards));
    assertThat(primary.knownLocalCheckpoints.remove(primaryShard.allocationId().getId()), equalTo(primary.localCheckpoint));
    assertThat(primary.knownLocalCheckpoints, equalTo(replicasProxy.generatedLocalCheckpoints));
    assertThat(primary.knownGlobalCheckpoints.remove(primaryShard.allocationId().getId()), equalTo(primary.globalCheckpoint));
    assertThat(primary.knownGlobalCheckpoints, equalTo(replicasProxy.generatedGlobalCheckpoints));
}