Examples with RestartCallback org.opensearch.test.InternalTestCluster.RestartCallback used on opensource projects

Example 1 with RestartCallback

use of org.opensearch.test.InternalTestCluster.RestartCallback in project OpenSearch by opensearch-project.

the class GatewayIndexStateIT method restartNodesOnBrokenClusterState.

private void restartNodesOnBrokenClusterState(ClusterState.Builder clusterStateBuilder) throws Exception {
    Map<String, PersistedClusterStateService> lucenePersistedStateFactories = Stream.of(internalCluster().getNodeNames()).collect(Collectors.toMap(Function.identity(), nodeName -> internalCluster().getInstance(PersistedClusterStateService.class, nodeName)));
    final ClusterState clusterState = clusterStateBuilder.build();
    internalCluster().fullRestart(new RestartCallback() {

        @Override
        public Settings onNodeStopped(String nodeName) throws Exception {
            final PersistedClusterStateService lucenePersistedStateFactory = lucenePersistedStateFactories.get(nodeName);
            try (PersistedClusterStateService.Writer writer = lucenePersistedStateFactory.createWriter()) {
                writer.writeFullStateAndCommit(clusterState.term(), clusterState);
            }
            return super.onNodeStopped(nodeName);
        }
    });
}

Example 2 with RestartCallback

use of org.opensearch.test.InternalTestCluster.RestartCallback in project OpenSearch by opensearch-project.

the class GatewayIndexStateIT method testIndexDeletionWhenNodeRejoins.

/**
 * This test ensures that when an index deletion takes place while a node is offline, when that
 * node rejoins the cluster, it deletes the index locally instead of importing it as a dangling index.
 */
public void testIndexDeletionWhenNodeRejoins() throws Exception {
    final String indexName = "test-index-del-on-node-rejoin-idx";
    final int numNodes = 2;
    final List<String> nodes;
    logger.info("--> starting a cluster with " + numNodes + " nodes");
    nodes = internalCluster().startNodes(numNodes, Settings.builder().put(IndexGraveyard.SETTING_MAX_TOMBSTONES.getKey(), randomIntBetween(10, 100)).build());
    logger.info("--> create an index");
    createIndex(indexName);
    logger.info("--> waiting for green status");
    ensureGreen();
    final String indexUUID = resolveIndex(indexName).getUUID();
    logger.info("--> restart a random date node, deleting the index in between stopping and restarting");
    internalCluster().restartRandomDataNode(new RestartCallback() {

        @Override
        public Settings onNodeStopped(final String nodeName) throws Exception {
            nodes.remove(nodeName);
            logger.info("--> stopped node[{}], remaining nodes {}", nodeName, nodes);
            assert nodes.size() > 0;
            final String otherNode = nodes.get(0);
            logger.info("--> delete index and verify it is deleted");
            final Client client = client(otherNode);
            client.admin().indices().prepareDelete(indexName).execute().actionGet();
            assertFalse(client.admin().indices().prepareExists(indexName).execute().actionGet().isExists());
            logger.info("--> index deleted");
            return super.onNodeStopped(nodeName);
        }
    });
    logger.info("--> wait until all nodes are back online");
    client().admin().cluster().health(Requests.clusterHealthRequest().waitForEvents(Priority.LANGUID).waitForNodes(Integer.toString(numNodes))).actionGet();
    logger.info("--> waiting for green status");
    ensureGreen();
    logger.info("--> verify that the deleted index is removed from the cluster and not reimported as dangling by the restarted node");
    assertFalse(client().admin().indices().prepareExists(indexName).execute().actionGet().isExists());
    assertBusy(() -> {
        final NodeEnvironment nodeEnv = internalCluster().getInstance(NodeEnvironment.class);
        try {
            assertFalse("index folder " + indexUUID + " should be deleted", nodeEnv.availableIndexFolders().contains(indexUUID));
        } catch (IOException e) {
            logger.error("Unable to retrieve available index folders from the node", e);
            fail("Unable to retrieve available index folders from the node");
        }
    });
}

Example 3 with RestartCallback

use of org.opensearch.test.InternalTestCluster.RestartCallback in project OpenSearch by opensearch-project.

the class MetadataNodesIT method testMetaWrittenWhenIndexIsClosedAndMetaUpdated.

@SuppressWarnings("unchecked")
public void testMetaWrittenWhenIndexIsClosedAndMetaUpdated() throws Exception {
    String masterNode = internalCluster().startMasterOnlyNode(Settings.EMPTY);
    final String dataNode = internalCluster().startDataOnlyNode(Settings.EMPTY);
    final String index = "index";
    assertAcked(prepareCreate(index).setSettings(Settings.builder().put("index.number_of_replicas", 0)));
    logger.info("--> wait for green index");
    ensureGreen();
    logger.info("--> wait for meta state written for index");
    assertIndexInMetaState(dataNode, index);
    assertIndexInMetaState(masterNode, index);
    logger.info("--> close index");
    client().admin().indices().prepareClose(index).get();
    // close the index
    ClusterStateResponse clusterStateResponse = client().admin().cluster().prepareState().get();
    assertThat(clusterStateResponse.getState().getMetadata().index(index).getState().name(), equalTo(IndexMetadata.State.CLOSE.name()));
    // update the mapping. this should cause the new meta data to be written although index is closed
    client().admin().indices().preparePutMapping(index).setSource(jsonBuilder().startObject().startObject("properties").startObject("integer_field").field("type", "integer").endObject().endObject().endObject()).get();
    GetMappingsResponse getMappingsResponse = client().admin().indices().prepareGetMappings(index).get();
    assertNotNull(((Map<String, ?>) (getMappingsResponse.getMappings().get(index).getSourceAsMap().get("properties"))).get("integer_field"));
    // make sure it was also written on red node although index is closed
    ImmutableOpenMap<String, IndexMetadata> indicesMetadata = getIndicesMetadataOnNode(dataNode);
    assertNotNull(((Map<String, ?>) (indicesMetadata.get(index).getMappings().get("_doc").getSourceAsMap().get("properties"))).get("integer_field"));
    assertThat(indicesMetadata.get(index).getState(), equalTo(IndexMetadata.State.CLOSE));
    /* Try the same and see if this also works if node was just restarted.
         * Each node holds an array of indices it knows of and checks if it should
         * write new meta data by looking up in this array. We need it because if an
         * index is closed it will not appear in the shard routing and we therefore
         * need to keep track of what we wrote before. However, when the node is
         * restarted this array is empty and we have to fill it before we decide
         * what we write. This is why we explicitly test for it.
         */
    internalCluster().restartNode(dataNode, new RestartCallback());
    client().admin().indices().preparePutMapping(index).setSource(jsonBuilder().startObject().startObject("properties").startObject("float_field").field("type", "float").endObject().endObject().endObject()).get();
    getMappingsResponse = client().admin().indices().prepareGetMappings(index).get();
    assertNotNull(((Map<String, ?>) (getMappingsResponse.getMappings().get(index).getSourceAsMap().get("properties"))).get("float_field"));
    // make sure it was also written on red node although index is closed
    indicesMetadata = getIndicesMetadataOnNode(dataNode);
    assertNotNull(((Map<String, ?>) (indicesMetadata.get(index).getMappings().get("_doc").getSourceAsMap().get("properties"))).get("float_field"));
    assertThat(indicesMetadata.get(index).getState(), equalTo(IndexMetadata.State.CLOSE));
    // finally check that meta data is also written of index opened again
    assertAcked(client().admin().indices().prepareOpen(index).get());
    // make sure index is fully initialized and nothing is changed anymore
    ensureGreen();
    indicesMetadata = getIndicesMetadataOnNode(dataNode);
    assertThat(indicesMetadata.get(index).getState(), equalTo(IndexMetadata.State.OPEN));
}

Example 4 with RestartCallback

use of org.opensearch.test.InternalTestCluster.RestartCallback in project OpenSearch by opensearch-project.

the class QuorumGatewayIT method testQuorumRecovery.

public void testQuorumRecovery() throws Exception {
    logger.info("--> starting 3 nodes");
    // we are shutting down nodes - make sure we don't have 2 clusters if we test network
    internalCluster().startNodes(3);
    createIndex("test");
    ensureGreen();
    final NumShards test = getNumShards("test");
    logger.info("--> indexing...");
    client().prepareIndex("test").setId("1").setSource(jsonBuilder().startObject().field("field", "value1").endObject()).get();
    // We don't check for failures in the flush response: if we do we might get the following:
    // FlushNotAllowedEngineException[[test][1] recovery is in progress, flush [COMMIT_TRANSLOG] is not allowed]
    flush();
    client().prepareIndex("test").setId("2").setSource(jsonBuilder().startObject().field("field", "value2").endObject()).get();
    refresh();
    for (int i = 0; i < 10; i++) {
        assertHitCount(client().prepareSearch().setSize(0).setQuery(matchAllQuery()).get(), 2L);
    }
    logger.info("--> restart all nodes");
    internalCluster().fullRestart(new RestartCallback() {

        @Override
        public void doAfterNodes(int numNodes, final Client activeClient) throws Exception {
            if (numNodes == 1) {
                assertBusy(() -> {
                    logger.info("--> running cluster_health (wait for the shards to startup)");
                    ClusterHealthResponse clusterHealth = activeClient.admin().cluster().health(clusterHealthRequest().waitForYellowStatus().waitForNodes("2").waitForActiveShards(test.numPrimaries * 2)).actionGet();
                    logger.info("--> done cluster_health, status {}", clusterHealth.getStatus());
                    assertFalse(clusterHealth.isTimedOut());
                    assertEquals(ClusterHealthStatus.YELLOW, clusterHealth.getStatus());
                }, 30, TimeUnit.SECONDS);
                logger.info("--> one node is closed -- index 1 document into the remaining nodes");
                activeClient.prepareIndex("test").setId("3").setSource(jsonBuilder().startObject().field("field", "value3").endObject()).get();
                assertNoFailures(activeClient.admin().indices().prepareRefresh().get());
                for (int i = 0; i < 10; i++) {
                    assertHitCount(activeClient.prepareSearch().setSize(0).setQuery(matchAllQuery()).get(), 3L);
                }
            }
        }
    });
    logger.info("--> all nodes are started back, verifying we got the latest version");
    logger.info("--> running cluster_health (wait for the shards to startup)");
    ensureGreen();
    for (int i = 0; i < 10; i++) {
        assertHitCount(client().prepareSearch().setSize(0).setQuery(matchAllQuery()).get(), 3L);
    }
}

Example 5 with RestartCallback

use of org.opensearch.test.InternalTestCluster.RestartCallback in project OpenSearch by opensearch-project.

the class RecoveryFromGatewayIT method testTwoNodeFirstNodeCleared.

public void testTwoNodeFirstNodeCleared() throws Exception {
    final String firstNode = internalCluster().startNode();
    internalCluster().startNode();
    client().prepareIndex("test").setId("1").setSource(jsonBuilder().startObject().field("field", "value1").endObject()).execute().actionGet();
    flush();
    client().prepareIndex("test").setId("2").setSource(jsonBuilder().startObject().field("field", "value2").endObject()).execute().actionGet();
    refresh();
    logger.info("Running Cluster Health (wait for the shards to startup)");
    ensureGreen();
    for (int i = 0; i < 10; i++) {
        assertHitCount(client().prepareSearch().setSize(0).setQuery(matchAllQuery()).execute().actionGet(), 2);
    }
    Map<String, long[]> primaryTerms = assertAndCapturePrimaryTerms(null);
    client().execute(AddVotingConfigExclusionsAction.INSTANCE, new AddVotingConfigExclusionsRequest(firstNode)).get();
    internalCluster().fullRestart(new RestartCallback() {

        @Override
        public Settings onNodeStopped(String nodeName) {
            return Settings.builder().put(RECOVER_AFTER_NODES_SETTING.getKey(), 2).putList(// disable bootstrapping
            INITIAL_MASTER_NODES_SETTING.getKey()).build();
        }

        @Override
        public boolean clearData(String nodeName) {
            return firstNode.equals(nodeName);
        }
    });
    logger.info("Running Cluster Health (wait for the shards to startup)");
    ensureGreen();
    primaryTerms = assertAndCapturePrimaryTerms(primaryTerms);
    for (int i = 0; i < 10; i++) {
        assertHitCount(client().prepareSearch().setSize(0).setQuery(matchAllQuery()).execute().actionGet(), 2);
    }
    client().execute(ClearVotingConfigExclusionsAction.INSTANCE, new ClearVotingConfigExclusionsRequest()).get();
}