Examples with IndexId org.elasticsearch.repositories.IndexId used on opensource projects

Example 26 with IndexId

use of org.elasticsearch.repositories.IndexId in project elasticsearch by elastic.

the class SnapshotShardsService method processIndexShardSnapshots.

/**
     * Checks if any new shards should be snapshotted on this node
     *
     * @param event cluster state changed event
     */
private void processIndexShardSnapshots(ClusterChangedEvent event) {
    SnapshotsInProgress snapshotsInProgress = event.state().custom(SnapshotsInProgress.TYPE);
    Map<Snapshot, SnapshotShards> survivors = new HashMap<>();
    // First, remove snapshots that are no longer there
    for (Map.Entry<Snapshot, SnapshotShards> entry : shardSnapshots.entrySet()) {
        final Snapshot snapshot = entry.getKey();
        if (snapshotsInProgress != null && snapshotsInProgress.snapshot(snapshot) != null) {
            survivors.put(entry.getKey(), entry.getValue());
        } else {
            // state update, which is being processed here
            for (IndexShardSnapshotStatus snapshotStatus : entry.getValue().shards.values()) {
                if (snapshotStatus.stage() == Stage.INIT || snapshotStatus.stage() == Stage.STARTED) {
                    snapshotStatus.abort();
                }
            }
        }
    }
    // For now we will be mostly dealing with a single snapshot at a time but might have multiple simultaneously running
    // snapshots in the future
    Map<Snapshot, Map<ShardId, IndexShardSnapshotStatus>> newSnapshots = new HashMap<>();
    // Now go through all snapshots and update existing or create missing
    final String localNodeId = event.state().nodes().getLocalNodeId();
    final DiscoveryNode masterNode = event.state().nodes().getMasterNode();
    final Map<Snapshot, Map<String, IndexId>> snapshotIndices = new HashMap<>();
    if (snapshotsInProgress != null) {
        for (SnapshotsInProgress.Entry entry : snapshotsInProgress.entries()) {
            snapshotIndices.put(entry.snapshot(), entry.indices().stream().collect(Collectors.toMap(IndexId::getName, Function.identity())));
            if (entry.state() == State.STARTED) {
                Map<ShardId, IndexShardSnapshotStatus> startedShards = new HashMap<>();
                SnapshotShards snapshotShards = shardSnapshots.get(entry.snapshot());
                for (ObjectObjectCursor<ShardId, ShardSnapshotStatus> shard : entry.shards()) {
                    // Add all new shards to start processing on
                    if (localNodeId.equals(shard.value.nodeId())) {
                        if (shard.value.state() == State.INIT && (snapshotShards == null || !snapshotShards.shards.containsKey(shard.key))) {
                            logger.trace("[{}] - Adding shard to the queue", shard.key);
                            startedShards.put(shard.key, new IndexShardSnapshotStatus());
                        }
                    }
                }
                if (!startedShards.isEmpty()) {
                    newSnapshots.put(entry.snapshot(), startedShards);
                    if (snapshotShards != null) {
                        // We already saw this snapshot but we need to add more started shards
                        Map<ShardId, IndexShardSnapshotStatus> shards = new HashMap<>();
                        // Put all shards that were already running on this node
                        shards.putAll(snapshotShards.shards);
                        // Put all newly started shards
                        shards.putAll(startedShards);
                        survivors.put(entry.snapshot(), new SnapshotShards(unmodifiableMap(shards)));
                    } else {
                        // Brand new snapshot that we haven't seen before
                        survivors.put(entry.snapshot(), new SnapshotShards(unmodifiableMap(startedShards)));
                    }
                }
            } else if (entry.state() == State.ABORTED) {
                // Abort all running shards for this snapshot
                SnapshotShards snapshotShards = shardSnapshots.get(entry.snapshot());
                if (snapshotShards != null) {
                    for (ObjectObjectCursor<ShardId, ShardSnapshotStatus> shard : entry.shards()) {
                        IndexShardSnapshotStatus snapshotStatus = snapshotShards.shards.get(shard.key);
                        if (snapshotStatus != null) {
                            switch(snapshotStatus.stage()) {
                                case INIT:
                                case STARTED:
                                    snapshotStatus.abort();
                                    break;
                                case FINALIZE:
                                    logger.debug("[{}] trying to cancel snapshot on shard [{}] that is finalizing, letting it finish", entry.snapshot(), shard.key);
                                    break;
                                case DONE:
                                    logger.debug("[{}] trying to cancel snapshot on the shard [{}] that is already done, updating status on the master", entry.snapshot(), shard.key);
                                    updateIndexShardSnapshotStatus(entry.snapshot(), shard.key, new ShardSnapshotStatus(localNodeId, State.SUCCESS), masterNode);
                                    break;
                                case FAILURE:
                                    logger.debug("[{}] trying to cancel snapshot on the shard [{}] that has already failed, updating status on the master", entry.snapshot(), shard.key);
                                    updateIndexShardSnapshotStatus(entry.snapshot(), shard.key, new ShardSnapshotStatus(localNodeId, State.FAILED, snapshotStatus.failure()), masterNode);
                                    break;
                                default:
                                    throw new IllegalStateException("Unknown snapshot shard stage " + snapshotStatus.stage());
                            }
                        }
                    }
                }
            }
        }
    }
    // Update the list of snapshots that we saw and tried to started
    // If startup of these shards fails later, we don't want to try starting these shards again
    shutdownLock.lock();
    try {
        shardSnapshots = unmodifiableMap(survivors);
        if (shardSnapshots.isEmpty()) {
            // Notify all waiting threads that no more snapshots
            shutdownCondition.signalAll();
        }
    } finally {
        shutdownLock.unlock();
    }
    // We have new shards to starts
    if (newSnapshots.isEmpty() == false) {
        Executor executor = threadPool.executor(ThreadPool.Names.SNAPSHOT);
        for (final Map.Entry<Snapshot, Map<ShardId, IndexShardSnapshotStatus>> entry : newSnapshots.entrySet()) {
            Map<String, IndexId> indicesMap = snapshotIndices.get(entry.getKey());
            assert indicesMap != null;
            for (final Map.Entry<ShardId, IndexShardSnapshotStatus> shardEntry : entry.getValue().entrySet()) {
                final ShardId shardId = shardEntry.getKey();
                try {
                    final IndexShard indexShard = indicesService.indexServiceSafe(shardId.getIndex()).getShardOrNull(shardId.id());
                    final IndexId indexId = indicesMap.get(shardId.getIndexName());
                    assert indexId != null;
                    executor.execute(new AbstractRunnable() {

                        @Override
                        public void doRun() {
                            snapshot(indexShard, entry.getKey(), indexId, shardEntry.getValue());
                            updateIndexShardSnapshotStatus(entry.getKey(), shardId, new ShardSnapshotStatus(localNodeId, State.SUCCESS), masterNode);
                        }

                        @Override
                        public void onFailure(Exception e) {
                            logger.warn((Supplier<?>) () -> new ParameterizedMessage("[{}] [{}] failed to create snapshot", shardId, entry.getKey()), e);
                            updateIndexShardSnapshotStatus(entry.getKey(), shardId, new ShardSnapshotStatus(localNodeId, State.FAILED, ExceptionsHelper.detailedMessage(e)), masterNode);
                        }
                    });
                } catch (Exception e) {
                    updateIndexShardSnapshotStatus(entry.getKey(), shardId, new ShardSnapshotStatus(localNodeId, State.FAILED, ExceptionsHelper.detailedMessage(e)), masterNode);
                }
            }
        }
    }
}

Example 27 with IndexId

use of org.elasticsearch.repositories.IndexId in project elasticsearch by elastic.

the class BlobStoreRepositoryTests method addRandomSnapshotsToRepoData.

private RepositoryData addRandomSnapshotsToRepoData(RepositoryData repoData, boolean inclIndices) {
    int numSnapshots = randomIntBetween(1, 20);
    for (int i = 0; i < numSnapshots; i++) {
        SnapshotId snapshotId = new SnapshotId(randomAsciiOfLength(8), UUIDs.randomBase64UUID());
        int numIndices = inclIndices ? randomIntBetween(0, 20) : 0;
        List<IndexId> indexIds = new ArrayList<>(numIndices);
        for (int j = 0; j < numIndices; j++) {
            indexIds.add(new IndexId(randomAsciiOfLength(8), UUIDs.randomBase64UUID()));
        }
        repoData = repoData.addSnapshot(snapshotId, indexIds);
    }
    return repoData;
}

Example 28 with IndexId

use of org.elasticsearch.repositories.IndexId in project elasticsearch by elastic.

the class IndexShardTests method testRestoreShard.

public void testRestoreShard() throws IOException {
    final IndexShard source = newStartedShard(true);
    IndexShard target = newStartedShard(true);
    indexDoc(source, "test", "0");
    if (randomBoolean()) {
        source.refresh("test");
    }
    indexDoc(target, "test", "1");
    target.refresh("test");
    assertDocs(target, new Uid("test", "1"));
    // only flush source
    flushShard(source);
    final ShardRouting origRouting = target.routingEntry();
    ShardRouting routing = ShardRoutingHelper.reinitPrimary(origRouting);
    final Snapshot snapshot = new Snapshot("foo", new SnapshotId("bar", UUIDs.randomBase64UUID()));
    routing = ShardRoutingHelper.newWithRestoreSource(routing, new RecoverySource.SnapshotRecoverySource(snapshot, Version.CURRENT, "test"));
    target = reinitShard(target, routing);
    Store sourceStore = source.store();
    Store targetStore = target.store();
    DiscoveryNode localNode = new DiscoveryNode("foo", buildNewFakeTransportAddress(), emptyMap(), emptySet(), Version.CURRENT);
    target.markAsRecovering("store", new RecoveryState(routing, localNode, null));
    assertTrue(target.restoreFromRepository(new RestoreOnlyRepository("test") {

        @Override
        public void restoreShard(IndexShard shard, SnapshotId snapshotId, Version version, IndexId indexId, ShardId snapshotShardId, RecoveryState recoveryState) {
            try {
                cleanLuceneIndex(targetStore.directory());
                for (String file : sourceStore.directory().listAll()) {
                    if (file.equals("write.lock") || file.startsWith("extra")) {
                        continue;
                    }
                    targetStore.directory().copyFrom(sourceStore.directory(), file, file, IOContext.DEFAULT);
                }
            } catch (Exception ex) {
                throw new RuntimeException(ex);
            }
        }
    }));
    target.updateRoutingEntry(routing.moveToStarted());
    assertDocs(target, new Uid("test", "0"));
    closeShards(source, target);
}

Example 29 with IndexId

use of org.elasticsearch.repositories.IndexId in project elasticsearch by elastic.

the class ESBlobStoreRepositoryIntegTestCase method testIndicesDeletedFromRepository.

public void testIndicesDeletedFromRepository() throws Exception {
    Client client = client();
    logger.info("-->  creating repository");
    final String repoName = "test-repo";
    createTestRepository(repoName);
    createIndex("test-idx-1", "test-idx-2", "test-idx-3");
    ensureGreen();
    logger.info("--> indexing some data");
    for (int i = 0; i < 20; i++) {
        index("test-idx-1", "doc", Integer.toString(i), "foo", "bar" + i);
        index("test-idx-2", "doc", Integer.toString(i), "foo", "baz" + i);
        index("test-idx-3", "doc", Integer.toString(i), "foo", "baz" + i);
    }
    refresh();
    logger.info("--> take a snapshot");
    CreateSnapshotResponse createSnapshotResponse = client.admin().cluster().prepareCreateSnapshot(repoName, "test-snap").setWaitForCompletion(true).get();
    assertEquals(createSnapshotResponse.getSnapshotInfo().successfulShards(), createSnapshotResponse.getSnapshotInfo().totalShards());
    logger.info("--> indexing more data");
    for (int i = 20; i < 40; i++) {
        index("test-idx-1", "doc", Integer.toString(i), "foo", "bar" + i);
        index("test-idx-2", "doc", Integer.toString(i), "foo", "baz" + i);
        index("test-idx-3", "doc", Integer.toString(i), "foo", "baz" + i);
    }
    logger.info("--> take another snapshot with only 2 of the 3 indices");
    createSnapshotResponse = client.admin().cluster().prepareCreateSnapshot(repoName, "test-snap2").setWaitForCompletion(true).setIndices("test-idx-1", "test-idx-2").get();
    assertEquals(createSnapshotResponse.getSnapshotInfo().successfulShards(), createSnapshotResponse.getSnapshotInfo().totalShards());
    logger.info("--> delete a snapshot");
    assertAcked(client().admin().cluster().prepareDeleteSnapshot(repoName, "test-snap").get());
    logger.info("--> verify index folder deleted from blob container");
    RepositoriesService repositoriesSvc = internalCluster().getInstance(RepositoriesService.class, internalCluster().getMasterName());
    @SuppressWarnings("unchecked") BlobStoreRepository repository = (BlobStoreRepository) repositoriesSvc.repository(repoName);
    BlobContainer indicesBlobContainer = repository.blobStore().blobContainer(repository.basePath().add("indices"));
    RepositoryData repositoryData = repository.getRepositoryData();
    for (IndexId indexId : repositoryData.getIndices().values()) {
        if (indexId.getName().equals("test-idx-3")) {
            // deleted index
            assertFalse(indicesBlobContainer.blobExists(indexId.getId()));
        }
    }
}

Example 30 with IndexId

use of org.elasticsearch.repositories.IndexId in project elasticsearch by elastic.

the class SnapshotsService method indicesToFailForCloseOrDeletion.

private static Set<Index> indicesToFailForCloseOrDeletion(ClusterState currentState, Set<IndexMetaData> indices) {
    SnapshotsInProgress snapshots = currentState.custom(SnapshotsInProgress.TYPE);
    Set<Index> indicesToFail = null;
    if (snapshots != null) {
        for (final SnapshotsInProgress.Entry entry : snapshots.entries()) {
            if (entry.partial() == false) {
                if (entry.state() == State.INIT) {
                    for (IndexId index : entry.indices()) {
                        IndexMetaData indexMetaData = currentState.metaData().index(index.getName());
                        if (indexMetaData != null && indices.contains(indexMetaData)) {
                            if (indicesToFail == null) {
                                indicesToFail = new HashSet<>();
                            }
                            indicesToFail.add(indexMetaData.getIndex());
                        }
                    }
                } else {
                    for (ObjectObjectCursor<ShardId, SnapshotsInProgress.ShardSnapshotStatus> shard : entry.shards()) {
                        if (!shard.value.state().completed()) {
                            IndexMetaData indexMetaData = currentState.metaData().index(shard.key.getIndex());
                            if (indexMetaData != null && indices.contains(indexMetaData)) {
                                if (indicesToFail == null) {
                                    indicesToFail = new HashSet<>();
                                }
                                indicesToFail.add(shard.key.getIndex());
                            }
                        }
                    }
                }
            }
        }
    }
    return indicesToFail;
}