Examples with UnassignedInfo org.opensearch.cluster.routing.UnassignedInfo used on opensource projects

Example 46 with UnassignedInfo

use of org.opensearch.cluster.routing.UnassignedInfo in project OpenSearch by opensearch-project.

the class AllocationDecidersTests method verifyDebugMode.

private void verifyDebugMode(RoutingAllocation.DebugMode mode, Matcher<Collection<? extends Decision>> matcher) {
    AllocationDeciders deciders = new AllocationDeciders(Collections.singleton(new AllocationDecider() {

        @Override
        public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) {
            return Decision.YES;
        }

        @Override
        public Decision canRebalance(ShardRouting shardRouting, RoutingAllocation allocation) {
            return Decision.YES;
        }

        @Override
        public Decision canRemain(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) {
            return Decision.YES;
        }

        @Override
        public Decision canAllocate(ShardRouting shardRouting, RoutingAllocation allocation) {
            return Decision.YES;
        }

        @Override
        public Decision canAllocate(IndexMetadata indexMetadata, RoutingNode node, RoutingAllocation allocation) {
            return Decision.YES;
        }

        @Override
        public Decision shouldAutoExpandToNode(IndexMetadata indexMetadata, DiscoveryNode node, RoutingAllocation allocation) {
            return Decision.YES;
        }

        @Override
        public Decision canRebalance(RoutingAllocation allocation) {
            return Decision.YES;
        }

        public Decision canMoveAway(ShardRouting shardRouting, RoutingAllocation allocation) {
            return Decision.YES;
        }

        public Decision canMoveAnyShard(RoutingAllocation allocation) {
            return Decision.YES;
        }

        public Decision canAllocateAnyShardToNode(RoutingNode node, RoutingAllocation allocation) {
            return Decision.YES;
        }
    }));
    ClusterState clusterState = ClusterState.builder(new ClusterName("test")).build();
    final RoutingAllocation allocation = new RoutingAllocation(deciders, clusterState.getRoutingNodes(), clusterState, null, null, 0L);
    allocation.setDebugMode(mode);
    final UnassignedInfo unassignedInfo = new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "_message");
    final ShardRouting shardRouting = ShardRouting.newUnassigned(new ShardId("test", "testUUID", 0), true, RecoverySource.ExistingStoreRecoverySource.INSTANCE, unassignedInfo);
    IndexMetadata idx = IndexMetadata.builder("idx").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(0).build();
    RoutingNode routingNode = new RoutingNode("testNode", null);
    verify(deciders.canAllocate(shardRouting, routingNode, allocation), matcher);
    verify(deciders.canAllocate(idx, routingNode, allocation), matcher);
    verify(deciders.canAllocate(shardRouting, allocation), matcher);
    verify(deciders.canRebalance(shardRouting, allocation), matcher);
    verify(deciders.canRebalance(allocation), matcher);
    verify(deciders.canRemain(shardRouting, routingNode, allocation), matcher);
    verify(deciders.canForceAllocatePrimary(shardRouting, routingNode, allocation), matcher);
    verify(deciders.shouldAutoExpandToNode(idx, null, allocation), matcher);
    verify(deciders.canMoveAway(shardRouting, allocation), matcher);
    verify(deciders.canMoveAnyShard(allocation), matcher);
    verify(deciders.canAllocateAnyShardToNode(routingNode, allocation), matcher);
}

Example 47 with UnassignedInfo

use of org.opensearch.cluster.routing.UnassignedInfo in project OpenSearch by opensearch-project.

the class SearchAsyncActionTests method getShardsIter.

static GroupShardsIterator<SearchShardIterator> getShardsIter(String index, OriginalIndices originalIndices, int numShards, boolean doReplicas, DiscoveryNode primaryNode, DiscoveryNode replicaNode) {
    ArrayList<SearchShardIterator> list = new ArrayList<>();
    for (int i = 0; i < numShards; i++) {
        ArrayList<ShardRouting> started = new ArrayList<>();
        ArrayList<ShardRouting> initializing = new ArrayList<>();
        ArrayList<ShardRouting> unassigned = new ArrayList<>();
        ShardRouting routing = ShardRouting.newUnassigned(new ShardId(new Index(index, "_na_"), i), true, RecoverySource.EmptyStoreRecoverySource.INSTANCE, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foobar"));
        routing = routing.initialize(primaryNode.getId(), i + "p", 0);
        routing.started();
        started.add(routing);
        if (doReplicas) {
            routing = ShardRouting.newUnassigned(new ShardId(new Index(index, "_na_"), i), false, RecoverySource.PeerRecoverySource.INSTANCE, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foobar"));
            if (replicaNode != null) {
                routing = routing.initialize(replicaNode.getId(), i + "r", 0);
                if (randomBoolean()) {
                    routing.started();
                    started.add(routing);
                } else {
                    initializing.add(routing);
                }
            } else {
                // unused yet
                unassigned.add(routing);
            }
        }
        Collections.shuffle(started, random());
        started.addAll(initializing);
        list.add(new SearchShardIterator(null, new ShardId(new Index(index, "_na_"), i), started, originalIndices));
    }
    return new GroupShardsIterator<>(list);
}

Example 48 with UnassignedInfo

use of org.opensearch.cluster.routing.UnassignedInfo in project OpenSearch by opensearch-project.

the class ClusterStateCreationUtils method state.

/**
 * Creates cluster state with and index that has one shard and #(replicaStates) replicas
 *
 * @param index              name of the index
 * @param activePrimaryLocal if active primary should coincide with the local node in the cluster state
 * @param primaryState       state of primary
 * @param replicaStates      states of the replicas. length of this array determines also the number of replicas
 */
public static ClusterState state(String index, boolean activePrimaryLocal, ShardRoutingState primaryState, ShardRoutingState... replicaStates) {
    final int numberOfReplicas = replicaStates.length;
    int numberOfNodes = numberOfReplicas + 1;
    if (primaryState == ShardRoutingState.RELOCATING) {
        numberOfNodes++;
    }
    for (ShardRoutingState state : replicaStates) {
        if (state == ShardRoutingState.RELOCATING) {
            numberOfNodes++;
        }
    }
    // we need a non-local master to test shard failures
    numberOfNodes = Math.max(2, numberOfNodes);
    final ShardId shardId = new ShardId(index, "_na_", 0);
    DiscoveryNodes.Builder discoBuilder = DiscoveryNodes.builder();
    Set<String> unassignedNodes = new HashSet<>();
    for (int i = 0; i < numberOfNodes + 1; i++) {
        final DiscoveryNode node = newNode(i);
        discoBuilder = discoBuilder.add(node);
        unassignedNodes.add(node.getId());
    }
    discoBuilder.localNodeId(newNode(0).getId());
    // we need a non-local master to test shard failures
    discoBuilder.masterNodeId(newNode(1).getId());
    final int primaryTerm = 1 + randomInt(200);
    IndexMetadata indexMetadata = IndexMetadata.builder(index).settings(Settings.builder().put(SETTING_VERSION_CREATED, Version.CURRENT).put(SETTING_NUMBER_OF_SHARDS, 1).put(SETTING_NUMBER_OF_REPLICAS, numberOfReplicas).put(SETTING_CREATION_DATE, System.currentTimeMillis())).primaryTerm(0, primaryTerm).build();
    IndexShardRoutingTable.Builder indexShardRoutingBuilder = new IndexShardRoutingTable.Builder(shardId);
    String primaryNode = null;
    String relocatingNode = null;
    UnassignedInfo unassignedInfo = null;
    if (primaryState != ShardRoutingState.UNASSIGNED) {
        if (activePrimaryLocal) {
            primaryNode = newNode(0).getId();
            unassignedNodes.remove(primaryNode);
        } else {
            Set<String> unassignedNodesExecludingPrimary = new HashSet<>(unassignedNodes);
            unassignedNodesExecludingPrimary.remove(newNode(0).getId());
            primaryNode = selectAndRemove(unassignedNodesExecludingPrimary);
            unassignedNodes.remove(primaryNode);
        }
        if (primaryState == ShardRoutingState.RELOCATING) {
            relocatingNode = selectAndRemove(unassignedNodes);
        } else if (primaryState == ShardRoutingState.INITIALIZING) {
            unassignedInfo = new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, null);
        }
    } else {
        unassignedInfo = new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, null);
    }
    indexShardRoutingBuilder.addShard(TestShardRouting.newShardRouting(index, 0, primaryNode, relocatingNode, true, primaryState, unassignedInfo));
    for (ShardRoutingState replicaState : replicaStates) {
        String replicaNode = null;
        relocatingNode = null;
        unassignedInfo = null;
        if (replicaState != ShardRoutingState.UNASSIGNED) {
            assert primaryNode != null : "a replica is assigned but the primary isn't";
            replicaNode = selectAndRemove(unassignedNodes);
            if (replicaState == ShardRoutingState.RELOCATING) {
                relocatingNode = selectAndRemove(unassignedNodes);
            }
        } else {
            unassignedInfo = new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, null);
        }
        indexShardRoutingBuilder.addShard(TestShardRouting.newShardRouting(index, shardId.id(), replicaNode, relocatingNode, false, replicaState, unassignedInfo));
    }
    final IndexShardRoutingTable indexShardRoutingTable = indexShardRoutingBuilder.build();
    IndexMetadata.Builder indexMetadataBuilder = new IndexMetadata.Builder(indexMetadata);
    indexMetadataBuilder.putInSyncAllocationIds(0, indexShardRoutingTable.activeShards().stream().map(ShardRouting::allocationId).map(AllocationId::getId).collect(Collectors.toSet()));
    ClusterState.Builder state = ClusterState.builder(new ClusterName("test"));
    state.nodes(discoBuilder);
    state.metadata(Metadata.builder().put(indexMetadataBuilder.build(), false).generateClusterUuidIfNeeded());
    state.routingTable(RoutingTable.builder().add(IndexRoutingTable.builder(indexMetadata.getIndex()).addIndexShard(indexShardRoutingTable)).build());
    return state.build();
}

Example 49 with UnassignedInfo

use of org.opensearch.cluster.routing.UnassignedInfo in project OpenSearch by opensearch-project.

the class IndicesStoreTests method testShardCanBeDeletedNoShardStarted.

public void testShardCanBeDeletedNoShardStarted() throws Exception {
    int numShards = randomIntBetween(1, 7);
    int numReplicas = randomInt(2);
    IndexShardRoutingTable.Builder routingTable = new IndexShardRoutingTable.Builder(new ShardId("test", "_na_", 1));
    for (int i = 0; i < numShards; i++) {
        int unStartedShard = randomInt(numReplicas);
        for (int j = 0; j <= numReplicas; j++) {
            ShardRoutingState state;
            if (j == unStartedShard) {
                state = randomFrom(NOT_STARTED_STATES);
            } else {
                state = randomFrom(ShardRoutingState.values());
            }
            UnassignedInfo unassignedInfo = null;
            if (state == ShardRoutingState.UNASSIGNED) {
                unassignedInfo = new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, null);
            }
            String currentNodeId = state == ShardRoutingState.UNASSIGNED ? null : randomAlphaOfLength(10);
            String relocatingNodeId = state == ShardRoutingState.RELOCATING ? randomAlphaOfLength(10) : null;
            routingTable.addShard(TestShardRouting.newShardRouting("test", i, currentNodeId, relocatingNodeId, j == 0, state, unassignedInfo));
        }
    }
    assertFalse(IndicesStore.shardCanBeDeleted(localNode.getId(), routingTable.build()));
}

Example 50 with UnassignedInfo

use of org.opensearch.cluster.routing.UnassignedInfo in project OpenSearch by opensearch-project.

the class FsRepositoryTests method testSnapshotAndRestore.

public void testSnapshotAndRestore() throws IOException, InterruptedException {
    ThreadPool threadPool = new TestThreadPool(getClass().getSimpleName());
    try (Directory directory = newDirectory()) {
        Path repo = createTempDir();
        Settings settings = Settings.builder().put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toAbsolutePath()).put(Environment.PATH_REPO_SETTING.getKey(), repo.toAbsolutePath()).putList(Environment.PATH_DATA_SETTING.getKey(), tmpPaths()).put("location", repo).put("compress", randomBoolean()).put("chunk_size", randomIntBetween(100, 1000), ByteSizeUnit.BYTES).build();
        int numDocs = indexDocs(directory);
        RepositoryMetadata metadata = new RepositoryMetadata("test", "fs", settings);
        FsRepository repository = new FsRepository(metadata, new Environment(settings, null), NamedXContentRegistry.EMPTY, BlobStoreTestUtil.mockClusterService(), new RecoverySettings(settings, new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)));
        repository.start();
        final Settings indexSettings = Settings.builder().put(IndexMetadata.SETTING_INDEX_UUID, "myindexUUID").build();
        IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("myindex", indexSettings);
        ShardId shardId = new ShardId(idxSettings.getIndex(), 1);
        Store store = new Store(shardId, idxSettings, directory, new DummyShardLock(shardId));
        SnapshotId snapshotId = new SnapshotId("test", "test");
        IndexId indexId = new IndexId(idxSettings.getIndex().getName(), idxSettings.getUUID());
        IndexCommit indexCommit = Lucene.getIndexCommit(Lucene.readSegmentInfos(store.directory()), store.directory());
        final PlainActionFuture<String> future1 = PlainActionFuture.newFuture();
        runGeneric(threadPool, () -> {
            IndexShardSnapshotStatus snapshotStatus = IndexShardSnapshotStatus.newInitializing(null);
            repository.snapshotShard(store, null, snapshotId, indexId, indexCommit, null, snapshotStatus, Version.CURRENT, Collections.emptyMap(), future1);
            future1.actionGet();
            IndexShardSnapshotStatus.Copy copy = snapshotStatus.asCopy();
            assertEquals(copy.getTotalFileCount(), copy.getIncrementalFileCount());
        });
        final String shardGeneration = future1.actionGet();
        Lucene.cleanLuceneIndex(directory);
        expectThrows(org.apache.lucene.index.IndexNotFoundException.class, () -> Lucene.readSegmentInfos(directory));
        DiscoveryNode localNode = new DiscoveryNode("foo", buildNewFakeTransportAddress(), emptyMap(), emptySet(), Version.CURRENT);
        ShardRouting routing = ShardRouting.newUnassigned(shardId, true, new RecoverySource.SnapshotRecoverySource("test", new Snapshot("foo", snapshotId), Version.CURRENT, indexId), new UnassignedInfo(UnassignedInfo.Reason.EXISTING_INDEX_RESTORED, ""));
        routing = ShardRoutingHelper.initialize(routing, localNode.getId(), 0);
        RecoveryState state = new RecoveryState(routing, localNode, null);
        final PlainActionFuture<Void> futureA = PlainActionFuture.newFuture();
        runGeneric(threadPool, () -> repository.restoreShard(store, snapshotId, indexId, shardId, state, futureA));
        futureA.actionGet();
        assertTrue(state.getIndex().recoveredBytes() > 0);
        assertEquals(0, state.getIndex().reusedFileCount());
        assertEquals(indexCommit.getFileNames().size(), state.getIndex().recoveredFileCount());
        assertEquals(numDocs, Lucene.readSegmentInfos(directory).totalMaxDoc());
        deleteRandomDoc(store.directory());
        SnapshotId incSnapshotId = new SnapshotId("test1", "test1");
        IndexCommit incIndexCommit = Lucene.getIndexCommit(Lucene.readSegmentInfos(store.directory()), store.directory());
        Collection<String> commitFileNames = incIndexCommit.getFileNames();
        final PlainActionFuture<String> future2 = PlainActionFuture.newFuture();
        runGeneric(threadPool, () -> {
            IndexShardSnapshotStatus snapshotStatus = IndexShardSnapshotStatus.newInitializing(shardGeneration);
            repository.snapshotShard(store, null, incSnapshotId, indexId, incIndexCommit, null, snapshotStatus, Version.CURRENT, Collections.emptyMap(), future2);
            future2.actionGet();
            IndexShardSnapshotStatus.Copy copy = snapshotStatus.asCopy();
            assertEquals(2, copy.getIncrementalFileCount());
            assertEquals(commitFileNames.size(), copy.getTotalFileCount());
        });
        // roll back to the first snap and then incrementally restore
        RecoveryState firstState = new RecoveryState(routing, localNode, null);
        final PlainActionFuture<Void> futureB = PlainActionFuture.newFuture();
        runGeneric(threadPool, () -> repository.restoreShard(store, snapshotId, indexId, shardId, firstState, futureB));
        futureB.actionGet();
        assertEquals("should reuse everything except of .liv and .si", commitFileNames.size() - 2, firstState.getIndex().reusedFileCount());
        RecoveryState secondState = new RecoveryState(routing, localNode, null);
        final PlainActionFuture<Void> futureC = PlainActionFuture.newFuture();
        runGeneric(threadPool, () -> repository.restoreShard(store, incSnapshotId, indexId, shardId, secondState, futureC));
        futureC.actionGet();
        assertEquals(secondState.getIndex().reusedFileCount(), commitFileNames.size() - 2);
        assertEquals(secondState.getIndex().recoveredFileCount(), 2);
        List<RecoveryState.FileDetail> recoveredFiles = secondState.getIndex().fileDetails().stream().filter(f -> f.reused() == false).collect(Collectors.toList());
        Collections.sort(recoveredFiles, Comparator.comparing(RecoveryState.FileDetail::name));
        assertTrue(recoveredFiles.get(0).name(), recoveredFiles.get(0).name().endsWith(".liv"));
        assertTrue(recoveredFiles.get(1).name(), recoveredFiles.get(1).name().endsWith("segments_" + incIndexCommit.getGeneration()));
    } finally {
        terminate(threadPool);
    }
}