Examples with OperationRouting org.elasticsearch.cluster.routing.OperationRouting used on opensource projects

Example 1 with OperationRouting

use of org.elasticsearch.cluster.routing.OperationRouting in project crate by crate.

the class BulkShardProcessorTest method testThatAddAfterFailureBlocksDueToRetry.

@Test
public void testThatAddAfterFailureBlocksDueToRetry() throws Exception {
    ClusterService clusterService = mock(ClusterService.class);
    OperationRouting operationRouting = mock(OperationRouting.class);
    mockShard(operationRouting, 1);
    mockShard(operationRouting, 2);
    mockShard(operationRouting, 3);
    when(clusterService.operationRouting()).thenReturn(operationRouting);
    // listener will be executed 2 times, once for the successfully added row and once for the failure
    final CountDownLatch listenerLatch = new CountDownLatch(2);
    final AtomicReference<ActionListener<ShardResponse>> ref = new AtomicReference<>();
    BulkRequestExecutor<ShardUpsertRequest> transportShardBulkAction = (request, listener) -> {
        ref.set(listener);
        listenerLatch.countDown();
    };
    BulkRetryCoordinator bulkRetryCoordinator = new BulkRetryCoordinator(threadPool);
    BulkRetryCoordinatorPool coordinatorPool = mock(BulkRetryCoordinatorPool.class);
    when(coordinatorPool.coordinator(any(ShardId.class))).thenReturn(bulkRetryCoordinator);
    ShardUpsertRequest.Builder builder = new ShardUpsertRequest.Builder(TimeValue.timeValueMillis(10), false, false, null, new Reference[] { fooRef }, UUID.randomUUID());
    final BulkShardProcessor<ShardUpsertRequest> bulkShardProcessor = new BulkShardProcessor<>(clusterService, mock(TransportBulkCreateIndicesAction.class), new IndexNameExpressionResolver(Settings.EMPTY), Settings.EMPTY, coordinatorPool, false, 1, builder, transportShardBulkAction, UUID.randomUUID());
    bulkShardProcessor.add("foo", new ShardUpsertRequest.Item("1", null, new Object[] { "bar1" }, null), null);
    final ActionListener<ShardResponse> listener = ref.get();
    listener.onFailure(new EsRejectedExecutionException());
    // wait, failure retry lock is done in decoupled thread
    listenerLatch.await(10, TimeUnit.SECONDS);
    final ScheduledExecutorService scheduledExecutorService = Executors.newScheduledThreadPool(2);
    try {
        final AtomicBoolean hadBlocked = new AtomicBoolean(false);
        final AtomicBoolean hasBlocked = new AtomicBoolean(true);
        final CountDownLatch latch = new CountDownLatch(1);
        scheduledExecutorService.execute(new Runnable() {

            @Override
            public void run() {
                scheduledExecutorService.schedule(new Runnable() {

                    @Override
                    public void run() {
                        hadBlocked.set(hasBlocked.get());
                        latch.countDown();
                    }
                }, 10, TimeUnit.MILLISECONDS);
                bulkShardProcessor.add("foo", new ShardUpsertRequest.Item("2", null, new Object[] { "bar2" }, null), null);
                hasBlocked.set(false);
            }
        });
        latch.await();
        assertTrue(hadBlocked.get());
    } finally {
        scheduledExecutorService.shutdownNow();
    }
}

Example 2 with OperationRouting

use of org.elasticsearch.cluster.routing.OperationRouting in project crate by crate.

the class BulkShardProcessorTest method testKill.

@Test
public void testKill() throws Exception {
    ClusterService clusterService = mock(ClusterService.class);
    OperationRouting operationRouting = mock(OperationRouting.class);
    mockShard(operationRouting, 1);
    mockShard(operationRouting, 2);
    mockShard(operationRouting, 3);
    when(clusterService.operationRouting()).thenReturn(operationRouting);
    final AtomicReference<ActionListener<ShardResponse>> ref = new AtomicReference<>();
    BulkRequestExecutor<ShardUpsertRequest> transportShardBulkAction = (request, listener) -> ref.set(listener);
    BulkRetryCoordinator bulkRetryCoordinator = new BulkRetryCoordinator(threadPool);
    BulkRetryCoordinatorPool coordinatorPool = mock(BulkRetryCoordinatorPool.class);
    when(coordinatorPool.coordinator(any(ShardId.class))).thenReturn(bulkRetryCoordinator);
    ShardUpsertRequest.Builder builder = new ShardUpsertRequest.Builder(TimeValue.timeValueMillis(10), false, false, null, new Reference[] { fooRef }, UUID.randomUUID());
    final BulkShardProcessor<ShardUpsertRequest> bulkShardProcessor = new BulkShardProcessor<>(clusterService, mock(TransportBulkCreateIndicesAction.class), new IndexNameExpressionResolver(Settings.EMPTY), Settings.EMPTY, coordinatorPool, false, 1, builder, transportShardBulkAction, UUID.randomUUID());
    assertThat(bulkShardProcessor.add("foo", new ShardUpsertRequest.Item("1", null, new Object[] { "bar1" }, null), null), is(true));
    bulkShardProcessor.kill(new InterruptedException());
    // A InterruptedException is thrown
    expectedException.expect(ExecutionException.class);
    expectedException.expectCause(isA(InterruptedException.class));
    bulkShardProcessor.result().get();
    // it's not possible to add more
    assertThat(bulkShardProcessor.add("foo", new ShardUpsertRequest.Item("1", null, new Object[] { "bar1" }, null), null), is(false));
}

Example 3 with OperationRouting

use of org.elasticsearch.cluster.routing.OperationRouting in project crate by crate.

the class InternalTestCluster method routingKeyForShard.

synchronized String routingKeyForShard(Index index, int shard, Random random) {
    assertThat(shard, greaterThanOrEqualTo(0));
    assertThat(shard, greaterThanOrEqualTo(0));
    for (NodeAndClient n : nodes.values()) {
        Node node = n.node;
        IndicesService indicesService = getInstanceFromNode(IndicesService.class, node);
        ClusterService clusterService = getInstanceFromNode(ClusterService.class, node);
        IndexService indexService = indicesService.indexService(index);
        if (indexService != null) {
            assertThat(indexService.getIndexSettings().getSettings().getAsInt(IndexMetadata.SETTING_NUMBER_OF_SHARDS, -1), greaterThan(shard));
            OperationRouting operationRouting = clusterService.operationRouting();
            while (true) {
                String routing = RandomStrings.randomAsciiLettersOfLength(random, 10);
                final int targetShard = operationRouting.indexShards(clusterService.state(), index.getName(), null, routing).shardId().getId();
                if (shard == targetShard) {
                    return routing;
                }
            }
        }
    }
    fail("Could not find a node that holds " + index);
    return null;
}

Example 4 with OperationRouting

use of org.elasticsearch.cluster.routing.OperationRouting in project elasticsearch by elastic.

the class RoutingIteratorTests method testReplicaShardPreferenceIters.

public void testReplicaShardPreferenceIters() throws Exception {
    AllocationService strategy = createAllocationService(Settings.builder().put("cluster.routing.allocation.node_concurrent_recoveries", 10).build());
    OperationRouting operationRouting = new OperationRouting(Settings.EMPTY, new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS));
    MetaData metaData = MetaData.builder().put(IndexMetaData.builder("test").settings(settings(Version.CURRENT)).numberOfShards(2).numberOfReplicas(2)).build();
    RoutingTable routingTable = RoutingTable.builder().addAsNew(metaData.index("test")).build();
    ClusterState clusterState = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metaData(metaData).routingTable(routingTable).build();
    clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().add(newNode("node1")).add(newNode("node2")).add(newNode("node3")).localNodeId("node1")).build();
    clusterState = strategy.reroute(clusterState, "reroute");
    clusterState = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
    // When replicas haven't initialized, it comes back with the primary first, then initializing replicas
    GroupShardsIterator shardIterators = operationRouting.searchShards(clusterState, new String[] { "test" }, null, "_replica_first");
    // two potential shards
    assertThat(shardIterators.size(), equalTo(2));
    ShardIterator iter = shardIterators.iterator().next();
    // three potential candidates for the shard
    assertThat(iter.size(), equalTo(3));
    ShardRouting routing = iter.nextOrNull();
    assertNotNull(routing);
    assertThat(routing.shardId().id(), anyOf(equalTo(0), equalTo(1)));
    // replicas haven't initialized yet, so primary is first
    assertTrue(routing.primary());
    assertTrue(routing.started());
    routing = iter.nextOrNull();
    assertThat(routing.shardId().id(), anyOf(equalTo(0), equalTo(1)));
    assertFalse(routing.primary());
    assertTrue(routing.initializing());
    routing = iter.nextOrNull();
    assertThat(routing.shardId().id(), anyOf(equalTo(0), equalTo(1)));
    assertFalse(routing.primary());
    assertTrue(routing.initializing());
    clusterState = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
    clusterState = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
    shardIterators = operationRouting.searchShards(clusterState, new String[] { "test" }, null, "_replica");
    // two potential shards
    assertThat(shardIterators.size(), equalTo(2));
    iter = shardIterators.iterator().next();
    // two potential replicas for the shard
    assertThat(iter.size(), equalTo(2));
    routing = iter.nextOrNull();
    assertNotNull(routing);
    assertThat(routing.shardId().id(), anyOf(equalTo(0), equalTo(1)));
    assertFalse(routing.primary());
    routing = iter.nextOrNull();
    assertThat(routing.shardId().id(), anyOf(equalTo(0), equalTo(1)));
    assertFalse(routing.primary());
    shardIterators = operationRouting.searchShards(clusterState, new String[] { "test" }, null, "_replica_first");
    // two potential shards
    assertThat(shardIterators.size(), equalTo(2));
    iter = shardIterators.iterator().next();
    // three potential candidates for the shard
    assertThat(iter.size(), equalTo(3));
    routing = iter.nextOrNull();
    assertNotNull(routing);
    assertThat(routing.shardId().id(), anyOf(equalTo(0), equalTo(1)));
    assertFalse(routing.primary());
    routing = iter.nextOrNull();
    assertThat(routing.shardId().id(), anyOf(equalTo(0), equalTo(1)));
    assertFalse(routing.primary());
    // finally the primary
    routing = iter.nextOrNull();
    assertThat(routing.shardId().id(), anyOf(equalTo(0), equalTo(1)));
    assertTrue(routing.primary());
}

Example 5 with OperationRouting

use of org.elasticsearch.cluster.routing.OperationRouting in project elasticsearch by elastic.

the class RoutingIteratorTests method testShardsAndPreferNodeRouting.

public void testShardsAndPreferNodeRouting() {
    AllocationService strategy = createAllocationService(Settings.builder().put("cluster.routing.allocation.node_concurrent_recoveries", 10).build());
    MetaData metaData = MetaData.builder().put(IndexMetaData.builder("test").settings(settings(Version.CURRENT)).numberOfShards(5).numberOfReplicas(1)).build();
    RoutingTable routingTable = RoutingTable.builder().addAsNew(metaData.index("test")).build();
    ClusterState clusterState = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metaData(metaData).routingTable(routingTable).build();
    clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().add(newNode("node1")).add(newNode("node2")).localNodeId("node1")).build();
    clusterState = strategy.reroute(clusterState, "reroute");
    clusterState = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
    clusterState = strategy.applyStartedShards(clusterState, clusterState.getRoutingNodes().shardsWithState(INITIALIZING));
    OperationRouting operationRouting = new OperationRouting(Settings.EMPTY, new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS));
    GroupShardsIterator shardIterators = operationRouting.searchShards(clusterState, new String[] { "test" }, null, "_shards:0");
    assertThat(shardIterators.size(), equalTo(1));
    assertThat(shardIterators.iterator().next().shardId().id(), equalTo(0));
    shardIterators = operationRouting.searchShards(clusterState, new String[] { "test" }, null, "_shards:1");
    assertThat(shardIterators.size(), equalTo(1));
    assertThat(shardIterators.iterator().next().shardId().id(), equalTo(1));
    //check node preference, first without preference to see they switch
    shardIterators = operationRouting.searchShards(clusterState, new String[] { "test" }, null, "_shards:0|");
    assertThat(shardIterators.size(), equalTo(1));
    assertThat(shardIterators.iterator().next().shardId().id(), equalTo(0));
    String firstRoundNodeId = shardIterators.iterator().next().nextOrNull().currentNodeId();
    shardIterators = operationRouting.searchShards(clusterState, new String[] { "test" }, null, "_shards:0");
    assertThat(shardIterators.size(), equalTo(1));
    assertThat(shardIterators.iterator().next().shardId().id(), equalTo(0));
    assertThat(shardIterators.iterator().next().nextOrNull().currentNodeId(), not(equalTo(firstRoundNodeId)));
    shardIterators = operationRouting.searchShards(clusterState, new String[] { "test" }, null, "_shards:0|_prefer_nodes:node1");
    assertThat(shardIterators.size(), equalTo(1));
    assertThat(shardIterators.iterator().next().shardId().id(), equalTo(0));
    assertThat(shardIterators.iterator().next().nextOrNull().currentNodeId(), equalTo("node1"));
    shardIterators = operationRouting.searchShards(clusterState, new String[] { "test" }, null, "_shards:0|_prefer_nodes:node1,node2");
    assertThat(shardIterators.size(), equalTo(1));
    Iterator<ShardIterator> iterator = shardIterators.iterator();
    final ShardIterator it = iterator.next();
    assertThat(it.shardId().id(), equalTo(0));
    final String firstNodeId = it.nextOrNull().currentNodeId();
    assertThat(firstNodeId, anyOf(equalTo("node1"), equalTo("node2")));
    if ("node1".equals(firstNodeId)) {
        assertThat(it.nextOrNull().currentNodeId(), equalTo("node2"));
    } else {
        assertThat(it.nextOrNull().currentNodeId(), equalTo("node1"));
    }
}