Examples with ImmutableOpenMap org.elasticsearch.common.collect.ImmutableOpenMap used on opensource projects

Example 11 with ImmutableOpenMap

use of org.elasticsearch.common.collect.ImmutableOpenMap in project elasticsearch by elastic.

the class SnapshotsService method processSnapshotsOnRemovedNodes.

/**
     * Cleans up shard snapshots that were running on removed nodes
     *
     * @param event cluster changed event
     */
private void processSnapshotsOnRemovedNodes(ClusterChangedEvent event) {
    if (removedNodesCleanupNeeded(event)) {
        // Check if we just became the master
        final boolean newMaster = !event.previousState().nodes().isLocalNodeElectedMaster();
        clusterService.submitStateUpdateTask("update snapshot state after node removal", new ClusterStateUpdateTask() {

            @Override
            public ClusterState execute(ClusterState currentState) throws Exception {
                DiscoveryNodes nodes = currentState.nodes();
                SnapshotsInProgress snapshots = currentState.custom(SnapshotsInProgress.TYPE);
                if (snapshots == null) {
                    return currentState;
                }
                boolean changed = false;
                ArrayList<SnapshotsInProgress.Entry> entries = new ArrayList<>();
                for (final SnapshotsInProgress.Entry snapshot : snapshots.entries()) {
                    SnapshotsInProgress.Entry updatedSnapshot = snapshot;
                    boolean snapshotChanged = false;
                    if (snapshot.state() == State.STARTED || snapshot.state() == State.ABORTED) {
                        ImmutableOpenMap.Builder<ShardId, ShardSnapshotStatus> shards = ImmutableOpenMap.builder();
                        for (ObjectObjectCursor<ShardId, ShardSnapshotStatus> shardEntry : snapshot.shards()) {
                            ShardSnapshotStatus shardStatus = shardEntry.value;
                            if (!shardStatus.state().completed() && shardStatus.nodeId() != null) {
                                if (nodes.nodeExists(shardStatus.nodeId())) {
                                    shards.put(shardEntry.key, shardEntry.value);
                                } else {
                                    // TODO: Restart snapshot on another node?
                                    snapshotChanged = true;
                                    logger.warn("failing snapshot of shard [{}] on closed node [{}]", shardEntry.key, shardStatus.nodeId());
                                    shards.put(shardEntry.key, new ShardSnapshotStatus(shardStatus.nodeId(), State.FAILED, "node shutdown"));
                                }
                            }
                        }
                        if (snapshotChanged) {
                            changed = true;
                            ImmutableOpenMap<ShardId, ShardSnapshotStatus> shardsMap = shards.build();
                            if (!snapshot.state().completed() && completed(shardsMap.values())) {
                                updatedSnapshot = new SnapshotsInProgress.Entry(snapshot, State.SUCCESS, shardsMap);
                                endSnapshot(updatedSnapshot);
                            } else {
                                updatedSnapshot = new SnapshotsInProgress.Entry(snapshot, snapshot.state(), shardsMap);
                            }
                        }
                        entries.add(updatedSnapshot);
                    } else if (snapshot.state() == State.INIT && newMaster) {
                        // Clean up the snapshot that failed to start from the old master
                        deleteSnapshot(snapshot.snapshot(), new DeleteSnapshotListener() {

                            @Override
                            public void onResponse() {
                                logger.debug("cleaned up abandoned snapshot {} in INIT state", snapshot.snapshot());
                            }

                            @Override
                            public void onFailure(Exception e) {
                                logger.warn("failed to clean up abandoned snapshot {} in INIT state", snapshot.snapshot());
                            }
                        }, updatedSnapshot.getRepositoryStateId(), false);
                    } else if (snapshot.state() == State.SUCCESS && newMaster) {
                        // Finalize the snapshot
                        endSnapshot(snapshot);
                    }
                }
                if (changed) {
                    snapshots = new SnapshotsInProgress(entries.toArray(new SnapshotsInProgress.Entry[entries.size()]));
                    return ClusterState.builder(currentState).putCustom(SnapshotsInProgress.TYPE, snapshots).build();
                }
                return currentState;
            }

            @Override
            public void onFailure(String source, Exception e) {
                logger.warn("failed to update snapshot state after node removal");
            }
        });
    }
}

Example 12 with ImmutableOpenMap

use of org.elasticsearch.common.collect.ImmutableOpenMap in project elasticsearch by elastic.

the class SnapshotsService method deleteSnapshot.

/**
     * Deletes snapshot from repository.
     * <p>
     * If the snapshot is still running cancels the snapshot first and then deletes it from the repository.
     *
     * @param snapshot snapshot
     * @param listener listener
     * @param repositoryStateId the unique id for the state of the repository
     */
private void deleteSnapshot(final Snapshot snapshot, final DeleteSnapshotListener listener, final long repositoryStateId, final boolean immediatePriority) {
    Priority priority = immediatePriority ? Priority.IMMEDIATE : Priority.NORMAL;
    clusterService.submitStateUpdateTask("delete snapshot", new ClusterStateUpdateTask(priority) {

        boolean waitForSnapshot = false;

        @Override
        public ClusterState execute(ClusterState currentState) throws Exception {
            SnapshotDeletionsInProgress deletionsInProgress = currentState.custom(SnapshotDeletionsInProgress.TYPE);
            if (deletionsInProgress != null && deletionsInProgress.hasDeletionsInProgress()) {
                throw new ConcurrentSnapshotExecutionException(snapshot, "cannot delete - another snapshot is currently being deleted");
            }
            RestoreInProgress restoreInProgress = currentState.custom(RestoreInProgress.TYPE);
            if (restoreInProgress != null) {
                // and the files the restore depends on would all be gone
                if (restoreInProgress.entries().isEmpty() == false) {
                    throw new ConcurrentSnapshotExecutionException(snapshot, "cannot delete snapshot during a restore");
                }
            }
            ClusterState.Builder clusterStateBuilder = ClusterState.builder(currentState);
            SnapshotsInProgress snapshots = currentState.custom(SnapshotsInProgress.TYPE);
            SnapshotsInProgress.Entry snapshotEntry = snapshots != null ? snapshots.snapshot(snapshot) : null;
            if (snapshotEntry == null) {
                // This snapshot is not running - delete
                if (snapshots != null && !snapshots.entries().isEmpty()) {
                    // However other snapshots are running - cannot continue
                    throw new ConcurrentSnapshotExecutionException(snapshot, "another snapshot is currently running cannot delete");
                }
                // add the snapshot deletion to the cluster state
                SnapshotDeletionsInProgress.Entry entry = new SnapshotDeletionsInProgress.Entry(snapshot, System.currentTimeMillis(), repositoryStateId);
                if (deletionsInProgress != null) {
                    deletionsInProgress = deletionsInProgress.withAddedEntry(entry);
                } else {
                    deletionsInProgress = SnapshotDeletionsInProgress.newInstance(entry);
                }
                clusterStateBuilder.putCustom(SnapshotDeletionsInProgress.TYPE, deletionsInProgress);
            } else {
                // This snapshot is currently running - stopping shards first
                waitForSnapshot = true;
                ImmutableOpenMap<ShardId, ShardSnapshotStatus> shards;
                if (snapshotEntry.state() == State.STARTED && snapshotEntry.shards() != null) {
                    // snapshot is currently running - stop started shards
                    ImmutableOpenMap.Builder<ShardId, ShardSnapshotStatus> shardsBuilder = ImmutableOpenMap.builder();
                    for (ObjectObjectCursor<ShardId, ShardSnapshotStatus> shardEntry : snapshotEntry.shards()) {
                        ShardSnapshotStatus status = shardEntry.value;
                        if (!status.state().completed()) {
                            shardsBuilder.put(shardEntry.key, new ShardSnapshotStatus(status.nodeId(), State.ABORTED));
                        } else {
                            shardsBuilder.put(shardEntry.key, status);
                        }
                    }
                    shards = shardsBuilder.build();
                } else if (snapshotEntry.state() == State.INIT) {
                    // snapshot hasn't started yet - end it
                    shards = snapshotEntry.shards();
                    endSnapshot(snapshotEntry);
                } else {
                    boolean hasUncompletedShards = false;
                    // Cleanup in case a node gone missing and snapshot wasn't updated for some reason
                    for (ObjectCursor<ShardSnapshotStatus> shardStatus : snapshotEntry.shards().values()) {
                        // Check if we still have shard running on existing nodes
                        if (shardStatus.value.state().completed() == false && shardStatus.value.nodeId() != null && currentState.nodes().get(shardStatus.value.nodeId()) != null) {
                            hasUncompletedShards = true;
                            break;
                        }
                    }
                    if (hasUncompletedShards) {
                        // snapshot is being finalized - wait for shards to complete finalization process
                        logger.debug("trying to delete completed snapshot - should wait for shards to finalize on all nodes");
                        return currentState;
                    } else {
                        // no shards to wait for - finish the snapshot
                        logger.debug("trying to delete completed snapshot with no finalizing shards - can delete immediately");
                        shards = snapshotEntry.shards();
                        endSnapshot(snapshotEntry);
                    }
                }
                SnapshotsInProgress.Entry newSnapshot = new SnapshotsInProgress.Entry(snapshotEntry, State.ABORTED, shards);
                snapshots = new SnapshotsInProgress(newSnapshot);
                clusterStateBuilder.putCustom(SnapshotsInProgress.TYPE, snapshots);
            }
            return clusterStateBuilder.build();
        }

        @Override
        public void onFailure(String source, Exception e) {
            listener.onFailure(e);
        }

        @Override
        public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
            if (waitForSnapshot) {
                logger.trace("adding snapshot completion listener to wait for deleted snapshot to finish");
                addListener(new SnapshotCompletionListener() {

                    @Override
                    public void onSnapshotCompletion(Snapshot completedSnapshot, SnapshotInfo snapshotInfo) {
                        if (completedSnapshot.equals(snapshot)) {
                            logger.trace("deleted snapshot completed - deleting files");
                            removeListener(this);
                            threadPool.executor(ThreadPool.Names.SNAPSHOT).execute(() -> deleteSnapshot(completedSnapshot.getRepository(), completedSnapshot.getSnapshotId().getName(), listener, true));
                        }
                    }

                    @Override
                    public void onSnapshotFailure(Snapshot failedSnapshot, Exception e) {
                        if (failedSnapshot.equals(snapshot)) {
                            logger.trace("deleted snapshot failed - deleting files", e);
                            removeListener(this);
                            threadPool.executor(ThreadPool.Names.SNAPSHOT).execute(() -> deleteSnapshot(failedSnapshot.getRepository(), failedSnapshot.getSnapshotId().getName(), listener, true));
                        }
                    }
                });
            } else {
                logger.trace("deleted snapshot is not running - deleting files");
                deleteSnapshotFromRepository(snapshot, listener, repositoryStateId);
            }
        }
    });
}

Example 13 with ImmutableOpenMap

use of org.elasticsearch.common.collect.ImmutableOpenMap in project elasticsearch by elastic.

the class SnapshotsService method beginSnapshot.

/**
     * Starts snapshot.
     * <p>
     * Creates snapshot in repository and updates snapshot metadata record with list of shards that needs to be processed.
     *
     * @param clusterState               cluster state
     * @param snapshot                   snapshot meta data
     * @param partial                    allow partial snapshots
     * @param userCreateSnapshotListener listener
     */
private void beginSnapshot(final ClusterState clusterState, final SnapshotsInProgress.Entry snapshot, final boolean partial, final CreateSnapshotListener userCreateSnapshotListener) {
    boolean snapshotCreated = false;
    try {
        Repository repository = repositoriesService.repository(snapshot.snapshot().getRepository());
        MetaData metaData = clusterState.metaData();
        if (!snapshot.includeGlobalState()) {
            // Remove global state from the cluster state
            MetaData.Builder builder = MetaData.builder();
            for (IndexId index : snapshot.indices()) {
                builder.put(metaData.index(index.getName()), false);
            }
            metaData = builder.build();
        }
        repository.initializeSnapshot(snapshot.snapshot().getSnapshotId(), snapshot.indices(), metaData);
        snapshotCreated = true;
        if (snapshot.indices().isEmpty()) {
            // No indices in this snapshot - we are done
            userCreateSnapshotListener.onResponse();
            endSnapshot(snapshot);
            return;
        }
        clusterService.submitStateUpdateTask("update_snapshot [" + snapshot.snapshot() + "]", new ClusterStateUpdateTask() {

            boolean accepted = false;

            SnapshotsInProgress.Entry updatedSnapshot;

            String failure = null;

            @Override
            public ClusterState execute(ClusterState currentState) {
                SnapshotsInProgress snapshots = currentState.custom(SnapshotsInProgress.TYPE);
                List<SnapshotsInProgress.Entry> entries = new ArrayList<>();
                for (SnapshotsInProgress.Entry entry : snapshots.entries()) {
                    if (entry.snapshot().equals(snapshot.snapshot())) {
                        // Replace the snapshot that was just created
                        ImmutableOpenMap<ShardId, SnapshotsInProgress.ShardSnapshotStatus> shards = shards(currentState, entry.indices());
                        if (!partial) {
                            Tuple<Set<String>, Set<String>> indicesWithMissingShards = indicesWithMissingShards(shards, currentState.metaData());
                            Set<String> missing = indicesWithMissingShards.v1();
                            Set<String> closed = indicesWithMissingShards.v2();
                            if (missing.isEmpty() == false || closed.isEmpty() == false) {
                                StringBuilder failureMessage = new StringBuilder();
                                updatedSnapshot = new SnapshotsInProgress.Entry(entry, State.FAILED, shards);
                                entries.add(updatedSnapshot);
                                if (missing.isEmpty() == false) {
                                    failureMessage.append("Indices don't have primary shards ");
                                    failureMessage.append(missing);
                                }
                                if (closed.isEmpty() == false) {
                                    if (failureMessage.length() > 0) {
                                        failureMessage.append("; ");
                                    }
                                    failureMessage.append("Indices are closed ");
                                    failureMessage.append(closed);
                                }
                                failure = failureMessage.toString();
                                continue;
                            }
                        }
                        updatedSnapshot = new SnapshotsInProgress.Entry(entry, State.STARTED, shards);
                        entries.add(updatedSnapshot);
                        if (!completed(shards.values())) {
                            accepted = true;
                        }
                    } else {
                        entries.add(entry);
                    }
                }
                return ClusterState.builder(currentState).putCustom(SnapshotsInProgress.TYPE, new SnapshotsInProgress(Collections.unmodifiableList(entries))).build();
            }

            @Override
            public void onFailure(String source, Exception e) {
                logger.warn((Supplier<?>) () -> new ParameterizedMessage("[{}] failed to create snapshot", snapshot.snapshot().getSnapshotId()), e);
                removeSnapshotFromClusterState(snapshot.snapshot(), null, e, new CleanupAfterErrorListener(snapshot, true, userCreateSnapshotListener, e));
            }

            @Override
            public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
                // The userCreateSnapshotListener.onResponse() notifies caller that the snapshot was accepted
                // for processing. If client wants to wait for the snapshot completion, it can register snapshot
                // completion listener in this method. For the snapshot completion to work properly, the snapshot
                // should still exist when listener is registered.
                userCreateSnapshotListener.onResponse();
                // go ahead and continue working on this snapshot rather then end here.
                if (!accepted && updatedSnapshot != null) {
                    endSnapshot(updatedSnapshot, failure);
                }
            }
        });
    } catch (Exception e) {
        logger.warn((Supplier<?>) () -> new ParameterizedMessage("failed to create snapshot [{}]", snapshot.snapshot().getSnapshotId()), e);
        removeSnapshotFromClusterState(snapshot.snapshot(), null, e, new CleanupAfterErrorListener(snapshot, snapshotCreated, userCreateSnapshotListener, e));
    }
}

Example 14 with ImmutableOpenMap

use of org.elasticsearch.common.collect.ImmutableOpenMap in project elasticsearch by elastic.

the class MetaDataDeleteIndexService method deleteIndices.

/**
     * Delete some indices from the cluster state.
     */
public ClusterState deleteIndices(ClusterState currentState, Set<Index> indices) {
    final MetaData meta = currentState.metaData();
    final Set<IndexMetaData> metaDatas = indices.stream().map(i -> meta.getIndexSafe(i)).collect(toSet());
    // Check if index deletion conflicts with any running snapshots
    SnapshotsService.checkIndexDeletion(currentState, metaDatas);
    RoutingTable.Builder routingTableBuilder = RoutingTable.builder(currentState.routingTable());
    MetaData.Builder metaDataBuilder = MetaData.builder(meta);
    ClusterBlocks.Builder clusterBlocksBuilder = ClusterBlocks.builder().blocks(currentState.blocks());
    final IndexGraveyard.Builder graveyardBuilder = IndexGraveyard.builder(metaDataBuilder.indexGraveyard());
    final int previousGraveyardSize = graveyardBuilder.tombstones().size();
    for (final Index index : indices) {
        String indexName = index.getName();
        logger.info("{} deleting index", index);
        routingTableBuilder.remove(indexName);
        clusterBlocksBuilder.removeIndexBlocks(indexName);
        metaDataBuilder.remove(indexName);
    }
    // add tombstones to the cluster state for each deleted index
    final IndexGraveyard currentGraveyard = graveyardBuilder.addTombstones(indices).build(settings);
    // the new graveyard set on the metadata
    metaDataBuilder.indexGraveyard(currentGraveyard);
    logger.trace("{} tombstones purged from the cluster state. Previous tombstone size: {}. Current tombstone size: {}.", graveyardBuilder.getNumPurged(), previousGraveyardSize, currentGraveyard.getTombstones().size());
    MetaData newMetaData = metaDataBuilder.build();
    ClusterBlocks blocks = clusterBlocksBuilder.build();
    // update snapshot restore entries
    ImmutableOpenMap<String, ClusterState.Custom> customs = currentState.getCustoms();
    final RestoreInProgress restoreInProgress = currentState.custom(RestoreInProgress.TYPE);
    if (restoreInProgress != null) {
        RestoreInProgress updatedRestoreInProgress = RestoreService.updateRestoreStateWithDeletedIndices(restoreInProgress, indices);
        if (updatedRestoreInProgress != restoreInProgress) {
            ImmutableOpenMap.Builder<String, ClusterState.Custom> builder = ImmutableOpenMap.builder(customs);
            builder.put(RestoreInProgress.TYPE, updatedRestoreInProgress);
            customs = builder.build();
        }
    }
    return allocationService.reroute(ClusterState.builder(currentState).routingTable(routingTableBuilder.build()).metaData(newMetaData).blocks(blocks).customs(customs).build(), "deleted indices [" + indices + "]");
}

Example 15 with ImmutableOpenMap

use of org.elasticsearch.common.collect.ImmutableOpenMap in project elasticsearch by elastic.

the class GetIndexIT method assertMappings.

private void assertMappings(GetIndexResponse response, String indexName) {
    ImmutableOpenMap<String, ImmutableOpenMap<String, MappingMetaData>> mappings = response.mappings();
    assertThat(mappings, notNullValue());
    assertThat(mappings.size(), equalTo(1));
    ImmutableOpenMap<String, MappingMetaData> indexMappings = mappings.get(indexName);
    assertThat(indexMappings, notNullValue());
    assertThat(indexMappings.size(), anyOf(equalTo(1), equalTo(2)));
    if (indexMappings.size() == 2) {
        MappingMetaData mapping = indexMappings.get("_default_");
        assertThat(mapping, notNullValue());
    }
    MappingMetaData mapping = indexMappings.get("type1");
    assertThat(mapping, notNullValue());
    assertThat(mapping.type(), equalTo("type1"));
}