Examples with RepositoriesMetadata org.opensearch.cluster.metadata.RepositoriesMetadata used on opensource projects

Example 1 with RepositoriesMetadata

use of org.opensearch.cluster.metadata.RepositoriesMetadata in project OpenSearch by opensearch-project.

the class ClusterModule method getNamedWriteables.

public static List<Entry> getNamedWriteables() {
    List<Entry> entries = new ArrayList<>();
    // Cluster State
    registerClusterCustom(entries, SnapshotsInProgress.TYPE, SnapshotsInProgress::new, SnapshotsInProgress::readDiffFrom);
    registerClusterCustom(entries, RestoreInProgress.TYPE, RestoreInProgress::new, RestoreInProgress::readDiffFrom);
    registerClusterCustom(entries, SnapshotDeletionsInProgress.TYPE, SnapshotDeletionsInProgress::new, SnapshotDeletionsInProgress::readDiffFrom);
    registerClusterCustom(entries, RepositoryCleanupInProgress.TYPE, RepositoryCleanupInProgress::new, RepositoryCleanupInProgress::readDiffFrom);
    // Metadata
    registerMetadataCustom(entries, RepositoriesMetadata.TYPE, RepositoriesMetadata::new, RepositoriesMetadata::readDiffFrom);
    registerMetadataCustom(entries, IngestMetadata.TYPE, IngestMetadata::new, IngestMetadata::readDiffFrom);
    registerMetadataCustom(entries, ScriptMetadata.TYPE, ScriptMetadata::new, ScriptMetadata::readDiffFrom);
    registerMetadataCustom(entries, IndexGraveyard.TYPE, IndexGraveyard::new, IndexGraveyard::readDiffFrom);
    registerMetadataCustom(entries, PersistentTasksCustomMetadata.TYPE, PersistentTasksCustomMetadata::new, PersistentTasksCustomMetadata::readDiffFrom);
    registerMetadataCustom(entries, ComponentTemplateMetadata.TYPE, ComponentTemplateMetadata::new, ComponentTemplateMetadata::readDiffFrom);
    registerMetadataCustom(entries, ComposableIndexTemplateMetadata.TYPE, ComposableIndexTemplateMetadata::new, ComposableIndexTemplateMetadata::readDiffFrom);
    registerMetadataCustom(entries, DataStreamMetadata.TYPE, DataStreamMetadata::new, DataStreamMetadata::readDiffFrom);
    // Task Status (not Diffable)
    entries.add(new Entry(Task.Status.class, PersistentTasksNodeService.Status.NAME, PersistentTasksNodeService.Status::new));
    return entries;
}

Example 2 with RepositoriesMetadata

use of org.opensearch.cluster.metadata.RepositoriesMetadata in project OpenSearch by opensearch-project.

the class RepositoriesIT method testRepositoryCreation.

public void testRepositoryCreation() throws Exception {
    Client client = client();
    Path location = randomRepoPath();
    createRepository("test-repo-1", "fs", location);
    logger.info("--> verify the repository");
    int numberOfFiles = FileSystemUtils.files(location).length;
    VerifyRepositoryResponse verifyRepositoryResponse = client.admin().cluster().prepareVerifyRepository("test-repo-1").get();
    assertThat(verifyRepositoryResponse.getNodes().size(), equalTo(cluster().numDataAndMasterNodes()));
    logger.info("--> verify that we didn't leave any files as a result of verification");
    assertThat(FileSystemUtils.files(location).length, equalTo(numberOfFiles));
    logger.info("--> check that repository is really there");
    ClusterStateResponse clusterStateResponse = client.admin().cluster().prepareState().clear().setMetadata(true).get();
    Metadata metadata = clusterStateResponse.getState().getMetadata();
    RepositoriesMetadata repositoriesMetadata = metadata.custom(RepositoriesMetadata.TYPE);
    assertThat(repositoriesMetadata, notNullValue());
    assertThat(repositoriesMetadata.repository("test-repo-1"), notNullValue());
    assertThat(repositoriesMetadata.repository("test-repo-1").type(), equalTo("fs"));
    logger.info("-->  creating another repository");
    createRepository("test-repo-2", "fs");
    logger.info("--> check that both repositories are in cluster state");
    clusterStateResponse = client.admin().cluster().prepareState().clear().setMetadata(true).get();
    metadata = clusterStateResponse.getState().getMetadata();
    repositoriesMetadata = metadata.custom(RepositoriesMetadata.TYPE);
    assertThat(repositoriesMetadata, notNullValue());
    assertThat(repositoriesMetadata.repositories().size(), equalTo(2));
    assertThat(repositoriesMetadata.repository("test-repo-1"), notNullValue());
    assertThat(repositoriesMetadata.repository("test-repo-1").type(), equalTo("fs"));
    assertThat(repositoriesMetadata.repository("test-repo-2"), notNullValue());
    assertThat(repositoriesMetadata.repository("test-repo-2").type(), equalTo("fs"));
    logger.info("--> check that both repositories can be retrieved by getRepositories query");
    GetRepositoriesResponse repositoriesResponse = client.admin().cluster().prepareGetRepositories(randomFrom("_all", "*", "test-repo-*")).get();
    assertThat(repositoriesResponse.repositories().size(), equalTo(2));
    assertThat(findRepository(repositoriesResponse.repositories(), "test-repo-1"), notNullValue());
    assertThat(findRepository(repositoriesResponse.repositories(), "test-repo-2"), notNullValue());
    logger.info("--> check that trying to create a repository with the same settings repeatedly does not update cluster state");
    String beforeStateUuid = clusterStateResponse.getState().stateUUID();
    assertThat(client.admin().cluster().preparePutRepository("test-repo-1").setType("fs").setSettings(Settings.builder().put("location", location)).get().isAcknowledged(), equalTo(true));
    assertEquals(beforeStateUuid, client.admin().cluster().prepareState().clear().get().getState().stateUUID());
    logger.info("--> delete repository test-repo-1");
    client.admin().cluster().prepareDeleteRepository("test-repo-1").get();
    repositoriesResponse = client.admin().cluster().prepareGetRepositories().get();
    assertThat(repositoriesResponse.repositories().size(), equalTo(1));
    assertThat(findRepository(repositoriesResponse.repositories(), "test-repo-2"), notNullValue());
    logger.info("--> delete repository test-repo-2");
    client.admin().cluster().prepareDeleteRepository("test-repo-2").get();
    repositoriesResponse = client.admin().cluster().prepareGetRepositories().get();
    assertThat(repositoriesResponse.repositories().size(), equalTo(0));
}

Example 3 with RepositoriesMetadata

use of org.opensearch.cluster.metadata.RepositoriesMetadata in project OpenSearch by opensearch-project.

the class CorruptedBlobStoreRepositoryIT method testFindDanglingLatestGeneration.

public void testFindDanglingLatestGeneration() throws Exception {
    Path repo = randomRepoPath();
    final String repoName = "test-repo";
    createRepository(repoName, "fs", Settings.builder().put("location", repo).put("compress", false).put("chunk_size", randomIntBetween(100, 1000), ByteSizeUnit.BYTES));
    createIndex("test-idx-1", "test-idx-2");
    logger.info("--> indexing some data");
    indexRandom(true, client().prepareIndex().setIndex("test-idx-1").setSource("foo", "bar"), client().prepareIndex().setIndex("test-idx-2").setSource("foo", "bar"));
    final String snapshot = "test-snap";
    logger.info("--> creating snapshot");
    CreateSnapshotResponse createSnapshotResponse = client().admin().cluster().prepareCreateSnapshot(repoName, snapshot).setWaitForCompletion(true).setIndices("test-idx-*").get();
    assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), greaterThan(0));
    assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), equalTo(createSnapshotResponse.getSnapshotInfo().totalShards()));
    final Repository repository = internalCluster().getCurrentMasterNodeInstance(RepositoriesService.class).repository(repoName);
    logger.info("--> move index-N blob to next generation");
    final RepositoryData repositoryData = getRepositoryData(repoName);
    final long beforeMoveGen = repositoryData.getGenId();
    Files.move(repo.resolve("index-" + beforeMoveGen), repo.resolve("index-" + (beforeMoveGen + 1)));
    logger.info("--> set next generation as pending in the cluster state");
    updateClusterState(currentState -> ClusterState.builder(currentState).metadata(Metadata.builder(currentState.getMetadata()).putCustom(RepositoriesMetadata.TYPE, currentState.metadata().<RepositoriesMetadata>custom(RepositoriesMetadata.TYPE).withUpdatedGeneration(repository.getMetadata().name(), beforeMoveGen, beforeMoveGen + 1)).build()).build());
    logger.info("--> full cluster restart");
    internalCluster().fullRestart();
    ensureGreen();
    logger.info("--> verify index-N blob is found at the new location");
    assertThat(getRepositoryData(repoName).getGenId(), is(beforeMoveGen + 1));
    startDeleteSnapshot(repoName, snapshot).get();
    logger.info("--> verify index-N blob is found at the expected location");
    assertThat(getRepositoryData(repoName).getGenId(), is(beforeMoveGen + 2));
    logger.info("--> make sure snapshot doesn't exist");
    expectThrows(SnapshotMissingException.class, () -> client().admin().cluster().prepareGetSnapshots(repoName).addSnapshots(snapshot).get());
}

Example 4 with RepositoriesMetadata

use of org.opensearch.cluster.metadata.RepositoriesMetadata in project OpenSearch by opensearch-project.

the class SnapshotsService method validate.

/**
 * Validates snapshot request
 *
 * @param repositoryName repository name
 * @param snapshotName snapshot name
 * @param state   current cluster state
 */
private static void validate(String repositoryName, String snapshotName, ClusterState state) {
    RepositoriesMetadata repositoriesMetadata = state.getMetadata().custom(RepositoriesMetadata.TYPE);
    if (repositoriesMetadata == null || repositoriesMetadata.repository(repositoryName) == null) {
        throw new RepositoryMissingException(repositoryName);
    }
    validate(repositoryName, snapshotName);
}

Example 5 with RepositoriesMetadata

use of org.opensearch.cluster.metadata.RepositoriesMetadata in project OpenSearch by opensearch-project.

the class RepositoriesService method registerRepository.

/**
 * Registers new repository in the cluster
 * <p>
 * This method can be only called on the master node. It tries to create a new repository on the master
 * and if it was successful it adds new repository to cluster metadata.
 *
 * @param request  register repository request
 * @param listener register repository listener
 */
public void registerRepository(final PutRepositoryRequest request, final ActionListener<ClusterStateUpdateResponse> listener) {
    assert lifecycle.started() : "Trying to register new repository but service is in state [" + lifecycle.state() + "]";
    final RepositoryMetadata newRepositoryMetadata = new RepositoryMetadata(request.name(), request.type(), request.settings());
    validate(request.name());
    final ActionListener<ClusterStateUpdateResponse> registrationListener;
    if (request.verify()) {
        registrationListener = ActionListener.delegateFailure(listener, (delegatedListener, clusterStateUpdateResponse) -> {
            if (clusterStateUpdateResponse.isAcknowledged()) {
                // The response was acknowledged - all nodes should know about the new repository, let's verify them
                verifyRepository(request.name(), ActionListener.delegateFailure(delegatedListener, (innerDelegatedListener, discoveryNodes) -> innerDelegatedListener.onResponse(clusterStateUpdateResponse)));
            } else {
                delegatedListener.onResponse(clusterStateUpdateResponse);
            }
        });
    } else {
        registrationListener = listener;
    }
    // Trying to create the new repository on master to make sure it works
    try {
        closeRepository(createRepository(newRepositoryMetadata, typesRegistry));
    } catch (Exception e) {
        registrationListener.onFailure(e);
        return;
    }
    clusterService.submitStateUpdateTask("put_repository [" + request.name() + "]", new AckedClusterStateUpdateTask<ClusterStateUpdateResponse>(request, registrationListener) {

        @Override
        protected ClusterStateUpdateResponse newResponse(boolean acknowledged) {
            return new ClusterStateUpdateResponse(acknowledged);
        }

        @Override
        public ClusterState execute(ClusterState currentState) {
            ensureRepositoryNotInUse(currentState, request.name());
            Metadata metadata = currentState.metadata();
            Metadata.Builder mdBuilder = Metadata.builder(currentState.metadata());
            RepositoriesMetadata repositories = metadata.custom(RepositoriesMetadata.TYPE);
            if (repositories == null) {
                logger.info("put repository [{}]", request.name());
                repositories = new RepositoriesMetadata(Collections.singletonList(new RepositoryMetadata(request.name(), request.type(), request.settings())));
            } else {
                boolean found = false;
                List<RepositoryMetadata> repositoriesMetadata = new ArrayList<>(repositories.repositories().size() + 1);
                for (RepositoryMetadata repositoryMetadata : repositories.repositories()) {
                    if (repositoryMetadata.name().equals(newRepositoryMetadata.name())) {
                        if (newRepositoryMetadata.equalsIgnoreGenerations(repositoryMetadata)) {
                            // Previous version is the same as this one no update is needed.
                            return currentState;
                        }
                        found = true;
                        repositoriesMetadata.add(newRepositoryMetadata);
                    } else {
                        repositoriesMetadata.add(repositoryMetadata);
                    }
                }
                if (!found) {
                    logger.info("put repository [{}]", request.name());
                    repositoriesMetadata.add(new RepositoryMetadata(request.name(), request.type(), request.settings()));
                } else {
                    logger.info("update repository [{}]", request.name());
                }
                repositories = new RepositoriesMetadata(repositoriesMetadata);
            }
            mdBuilder.putCustom(RepositoriesMetadata.TYPE, repositories);
            return ClusterState.builder(currentState).metadata(mdBuilder).build();
        }

        @Override
        public void onFailure(String source, Exception e) {
            logger.warn(() -> new ParameterizedMessage("failed to create repository [{}]", request.name()), e);
            super.onFailure(source, e);
        }

        @Override
        public boolean mustAck(DiscoveryNode discoveryNode) {
            // repository is created on both master and data nodes
            return discoveryNode.isMasterNode() || discoveryNode.isDataNode();
        }
    });
}