Examples with RepositoryData org.elasticsearch.repositories.RepositoryData used on opensource projects

Example 1 with RepositoryData

use of org.elasticsearch.repositories.RepositoryData in project elasticsearch by elastic.

the class SharedClusterSnapshotRestoreIT method testDeleteOrphanSnapshot.

public void testDeleteOrphanSnapshot() throws Exception {
    Client client = client();
    logger.info("-->  creating repository");
    final String repositoryName = "test-repo";
    assertAcked(client.admin().cluster().preparePutRepository(repositoryName).setType("mock").setSettings(Settings.builder().put("location", randomRepoPath()).put("compress", randomBoolean()).put("chunk_size", randomIntBetween(100, 1000), ByteSizeUnit.BYTES)));
    logger.info("--> create the index");
    final String idxName = "test-idx";
    createIndex(idxName);
    ensureGreen();
    ClusterService clusterService = internalCluster().getInstance(ClusterService.class, internalCluster().getMasterName());
    final CountDownLatch countDownLatch = new CountDownLatch(1);
    logger.info("--> snapshot");
    final String snapshotName = "test-snap";
    CreateSnapshotResponse createSnapshotResponse = client.admin().cluster().prepareCreateSnapshot(repositoryName, snapshotName).setWaitForCompletion(true).setIndices(idxName).get();
    assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), greaterThan(0));
    assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), equalTo(createSnapshotResponse.getSnapshotInfo().totalShards()));
    logger.info("--> emulate an orphan snapshot");
    RepositoriesService repositoriesService = internalCluster().getInstance(RepositoriesService.class, internalCluster().getMasterName());
    final RepositoryData repositoryData = repositoriesService.repository(repositoryName).getRepositoryData();
    final IndexId indexId = repositoryData.resolveIndexId(idxName);
    clusterService.submitStateUpdateTask("orphan snapshot test", new ClusterStateUpdateTask() {

        @Override
        public ClusterState execute(ClusterState currentState) {
            // Simulate orphan snapshot
            ImmutableOpenMap.Builder<ShardId, ShardSnapshotStatus> shards = ImmutableOpenMap.builder();
            shards.put(new ShardId(idxName, "_na_", 0), new ShardSnapshotStatus("unknown-node", State.ABORTED));
            shards.put(new ShardId(idxName, "_na_", 1), new ShardSnapshotStatus("unknown-node", State.ABORTED));
            shards.put(new ShardId(idxName, "_na_", 2), new ShardSnapshotStatus("unknown-node", State.ABORTED));
            List<Entry> entries = new ArrayList<>();
            entries.add(new Entry(new Snapshot(repositoryName, createSnapshotResponse.getSnapshotInfo().snapshotId()), true, false, State.ABORTED, Collections.singletonList(indexId), System.currentTimeMillis(), repositoryData.getGenId(), shards.build()));
            return ClusterState.builder(currentState).putCustom(SnapshotsInProgress.TYPE, new SnapshotsInProgress(Collections.unmodifiableList(entries))).build();
        }

        @Override
        public void onFailure(String source, Exception e) {
            fail();
        }

        @Override
        public void clusterStateProcessed(String source, ClusterState oldState, final ClusterState newState) {
            countDownLatch.countDown();
        }
    });
    countDownLatch.await();
    logger.info("--> try deleting the orphan snapshot");
    assertAcked(client.admin().cluster().prepareDeleteSnapshot(repositoryName, snapshotName).get("10s"));
}

Example 2 with RepositoryData

use of org.elasticsearch.repositories.RepositoryData in project elasticsearch by elastic.

the class BlobStoreRepository method getRepositoryData.

@Override
public RepositoryData getRepositoryData() {
    try {
        final long indexGen = latestIndexBlobId();
        final String snapshotsIndexBlobName = INDEX_FILE_PREFIX + Long.toString(indexGen);
        RepositoryData repositoryData;
        try (InputStream blob = snapshotsBlobContainer.readBlob(snapshotsIndexBlobName)) {
            BytesStreamOutput out = new BytesStreamOutput();
            Streams.copy(blob, out);
            // EMPTY is safe here because RepositoryData#fromXContent calls namedObject
            try (XContentParser parser = XContentHelper.createParser(NamedXContentRegistry.EMPTY, out.bytes())) {
                repositoryData = RepositoryData.snapshotsFromXContent(parser, indexGen);
            } catch (NotXContentException e) {
                logger.warn("[{}] index blob is not valid x-content [{} bytes]", snapshotsIndexBlobName, out.bytes().length());
                throw e;
            }
        }
        // now load the incompatible snapshot ids, if they exist
        try (InputStream blob = snapshotsBlobContainer.readBlob(INCOMPATIBLE_SNAPSHOTS_BLOB)) {
            BytesStreamOutput out = new BytesStreamOutput();
            Streams.copy(blob, out);
            try (XContentParser parser = XContentHelper.createParser(NamedXContentRegistry.EMPTY, out.bytes())) {
                repositoryData = repositoryData.incompatibleSnapshotsFromXContent(parser);
            }
        } catch (NoSuchFileException e) {
            logger.debug("[{}] Incompatible snapshots blob [{}] does not exist, the likely reason is that " + "there are no incompatible snapshots in the repository", metadata.name(), INCOMPATIBLE_SNAPSHOTS_BLOB);
        }
        return repositoryData;
    } catch (NoSuchFileException ex) {
        // repository doesn't have an index blob, its a new blank repo
        return RepositoryData.EMPTY;
    } catch (IOException ioe) {
        throw new RepositoryException(metadata.name(), "could not read repository data from index blob", ioe);
    }
}

Example 3 with RepositoryData

use of org.elasticsearch.repositories.RepositoryData in project elasticsearch by elastic.

the class BlobStoreRepository method initializeSnapshot.

@Override
public void initializeSnapshot(SnapshotId snapshotId, List<IndexId> indices, MetaData clusterMetaData) {
    if (isReadOnly()) {
        throw new RepositoryException(metadata.name(), "cannot create snapshot in a readonly repository");
    }
    try {
        final String snapshotName = snapshotId.getName();
        // check if the snapshot name already exists in the repository
        final RepositoryData repositoryData = getRepositoryData();
        if (repositoryData.getAllSnapshotIds().stream().anyMatch(s -> s.getName().equals(snapshotName))) {
            throw new InvalidSnapshotNameException(metadata.name(), snapshotId.getName(), "snapshot with the same name already exists");
        }
        if (snapshotFormat.exists(snapshotsBlobContainer, snapshotId.getUUID())) {
            throw new InvalidSnapshotNameException(metadata.name(), snapshotId.getName(), "snapshot with the same name already exists");
        }
        // Write Global MetaData
        globalMetaDataFormat.write(clusterMetaData, snapshotsBlobContainer, snapshotId.getUUID());
        // write the index metadata for each index in the snapshot
        for (IndexId index : indices) {
            final IndexMetaData indexMetaData = clusterMetaData.index(index.getName());
            final BlobPath indexPath = basePath().add("indices").add(index.getId());
            final BlobContainer indexMetaDataBlobContainer = blobStore().blobContainer(indexPath);
            indexMetaDataFormat.write(indexMetaData, indexMetaDataBlobContainer, snapshotId.getUUID());
        }
    } catch (IOException ex) {
        throw new SnapshotCreationException(metadata.name(), snapshotId, ex);
    }
}

Example 4 with RepositoryData

use of org.elasticsearch.repositories.RepositoryData in project elasticsearch by elastic.

the class BlobStoreRepositoryTests method testReadAndWriteSnapshotsThroughIndexFile.

public void testReadAndWriteSnapshotsThroughIndexFile() throws Exception {
    final BlobStoreRepository repository = setupRepo();
    // write to and read from a index file with no entries
    assertThat(repository.getRepositoryData().getSnapshotIds().size(), equalTo(0));
    final RepositoryData emptyData = RepositoryData.EMPTY;
    repository.writeIndexGen(emptyData, emptyData.getGenId());
    RepositoryData repoData = repository.getRepositoryData();
    assertEquals(repoData, emptyData);
    assertEquals(repoData.getIndices().size(), 0);
    assertEquals(repoData.getSnapshotIds().size(), 0);
    assertEquals(0L, repoData.getGenId());
    // write to and read from an index file with snapshots but no indices
    repoData = addRandomSnapshotsToRepoData(repoData, false);
    repository.writeIndexGen(repoData, repoData.getGenId());
    assertEquals(repoData, repository.getRepositoryData());
    // write to and read from a index file with random repository data
    repoData = addRandomSnapshotsToRepoData(repository.getRepositoryData(), true);
    repository.writeIndexGen(repoData, repoData.getGenId());
    assertEquals(repoData, repository.getRepositoryData());
}

Example 5 with RepositoryData

use of org.elasticsearch.repositories.RepositoryData in project elasticsearch by elastic.

the class BlobStoreRepositoryTests method testReadAndWriteIncompatibleSnapshots.

public void testReadAndWriteIncompatibleSnapshots() throws Exception {
    final BlobStoreRepository repository = setupRepo();
    // write to and read from incompatible snapshots file with no entries
    assertEquals(0, repository.getRepositoryData().getIncompatibleSnapshotIds().size());
    RepositoryData emptyData = RepositoryData.EMPTY;
    repository.writeIndexGen(emptyData, emptyData.getGenId());
    repository.writeIncompatibleSnapshots(emptyData);
    RepositoryData readData = repository.getRepositoryData();
    assertEquals(emptyData, readData);
    assertEquals(0, readData.getIndices().size());
    assertEquals(0, readData.getSnapshotIds().size());
    // write to and read from incompatible snapshots with some number of entries
    final int numSnapshots = randomIntBetween(1, 20);
    final List<SnapshotId> snapshotIds = new ArrayList<>(numSnapshots);
    for (int i = 0; i < numSnapshots; i++) {
        snapshotIds.add(new SnapshotId(randomAsciiOfLength(8), UUIDs.randomBase64UUID()));
    }
    RepositoryData repositoryData = new RepositoryData(readData.getGenId(), Collections.emptyList(), Collections.emptyMap(), snapshotIds);
    repository.blobContainer().deleteBlob("incompatible-snapshots");
    repository.writeIncompatibleSnapshots(repositoryData);
    readData = repository.getRepositoryData();
    assertEquals(repositoryData.getIncompatibleSnapshotIds(), readData.getIncompatibleSnapshotIds());
}