Examples with ObjectObjectCursor com.carrotsearch.hppc.cursors.ObjectObjectCursor used on opensource projects

Example 6 with ObjectObjectCursor

use of com.carrotsearch.hppc.cursors.ObjectObjectCursor in project elasticsearch by elastic.

the class SnapshotsService method deleteSnapshot.

/**
     * Deletes snapshot from repository.
     * <p>
     * If the snapshot is still running cancels the snapshot first and then deletes it from the repository.
     *
     * @param snapshot snapshot
     * @param listener listener
     * @param repositoryStateId the unique id for the state of the repository
     */
private void deleteSnapshot(final Snapshot snapshot, final DeleteSnapshotListener listener, final long repositoryStateId, final boolean immediatePriority) {
    Priority priority = immediatePriority ? Priority.IMMEDIATE : Priority.NORMAL;
    clusterService.submitStateUpdateTask("delete snapshot", new ClusterStateUpdateTask(priority) {

        boolean waitForSnapshot = false;

        @Override
        public ClusterState execute(ClusterState currentState) throws Exception {
            SnapshotDeletionsInProgress deletionsInProgress = currentState.custom(SnapshotDeletionsInProgress.TYPE);
            if (deletionsInProgress != null && deletionsInProgress.hasDeletionsInProgress()) {
                throw new ConcurrentSnapshotExecutionException(snapshot, "cannot delete - another snapshot is currently being deleted");
            }
            RestoreInProgress restoreInProgress = currentState.custom(RestoreInProgress.TYPE);
            if (restoreInProgress != null) {
                // and the files the restore depends on would all be gone
                if (restoreInProgress.entries().isEmpty() == false) {
                    throw new ConcurrentSnapshotExecutionException(snapshot, "cannot delete snapshot during a restore");
                }
            }
            ClusterState.Builder clusterStateBuilder = ClusterState.builder(currentState);
            SnapshotsInProgress snapshots = currentState.custom(SnapshotsInProgress.TYPE);
            SnapshotsInProgress.Entry snapshotEntry = snapshots != null ? snapshots.snapshot(snapshot) : null;
            if (snapshotEntry == null) {
                // This snapshot is not running - delete
                if (snapshots != null && !snapshots.entries().isEmpty()) {
                    // However other snapshots are running - cannot continue
                    throw new ConcurrentSnapshotExecutionException(snapshot, "another snapshot is currently running cannot delete");
                }
                // add the snapshot deletion to the cluster state
                SnapshotDeletionsInProgress.Entry entry = new SnapshotDeletionsInProgress.Entry(snapshot, System.currentTimeMillis(), repositoryStateId);
                if (deletionsInProgress != null) {
                    deletionsInProgress = deletionsInProgress.withAddedEntry(entry);
                } else {
                    deletionsInProgress = SnapshotDeletionsInProgress.newInstance(entry);
                }
                clusterStateBuilder.putCustom(SnapshotDeletionsInProgress.TYPE, deletionsInProgress);
            } else {
                // This snapshot is currently running - stopping shards first
                waitForSnapshot = true;
                ImmutableOpenMap<ShardId, ShardSnapshotStatus> shards;
                if (snapshotEntry.state() == State.STARTED && snapshotEntry.shards() != null) {
                    // snapshot is currently running - stop started shards
                    ImmutableOpenMap.Builder<ShardId, ShardSnapshotStatus> shardsBuilder = ImmutableOpenMap.builder();
                    for (ObjectObjectCursor<ShardId, ShardSnapshotStatus> shardEntry : snapshotEntry.shards()) {
                        ShardSnapshotStatus status = shardEntry.value;
                        if (!status.state().completed()) {
                            shardsBuilder.put(shardEntry.key, new ShardSnapshotStatus(status.nodeId(), State.ABORTED));
                        } else {
                            shardsBuilder.put(shardEntry.key, status);
                        }
                    }
                    shards = shardsBuilder.build();
                } else if (snapshotEntry.state() == State.INIT) {
                    // snapshot hasn't started yet - end it
                    shards = snapshotEntry.shards();
                    endSnapshot(snapshotEntry);
                } else {
                    boolean hasUncompletedShards = false;
                    // Cleanup in case a node gone missing and snapshot wasn't updated for some reason
                    for (ObjectCursor<ShardSnapshotStatus> shardStatus : snapshotEntry.shards().values()) {
                        // Check if we still have shard running on existing nodes
                        if (shardStatus.value.state().completed() == false && shardStatus.value.nodeId() != null && currentState.nodes().get(shardStatus.value.nodeId()) != null) {
                            hasUncompletedShards = true;
                            break;
                        }
                    }
                    if (hasUncompletedShards) {
                        // snapshot is being finalized - wait for shards to complete finalization process
                        logger.debug("trying to delete completed snapshot - should wait for shards to finalize on all nodes");
                        return currentState;
                    } else {
                        // no shards to wait for - finish the snapshot
                        logger.debug("trying to delete completed snapshot with no finalizing shards - can delete immediately");
                        shards = snapshotEntry.shards();
                        endSnapshot(snapshotEntry);
                    }
                }
                SnapshotsInProgress.Entry newSnapshot = new SnapshotsInProgress.Entry(snapshotEntry, State.ABORTED, shards);
                snapshots = new SnapshotsInProgress(newSnapshot);
                clusterStateBuilder.putCustom(SnapshotsInProgress.TYPE, snapshots);
            }
            return clusterStateBuilder.build();
        }

        @Override
        public void onFailure(String source, Exception e) {
            listener.onFailure(e);
        }

        @Override
        public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
            if (waitForSnapshot) {
                logger.trace("adding snapshot completion listener to wait for deleted snapshot to finish");
                addListener(new SnapshotCompletionListener() {

                    @Override
                    public void onSnapshotCompletion(Snapshot completedSnapshot, SnapshotInfo snapshotInfo) {
                        if (completedSnapshot.equals(snapshot)) {
                            logger.trace("deleted snapshot completed - deleting files");
                            removeListener(this);
                            threadPool.executor(ThreadPool.Names.SNAPSHOT).execute(() -> deleteSnapshot(completedSnapshot.getRepository(), completedSnapshot.getSnapshotId().getName(), listener, true));
                        }
                    }

                    @Override
                    public void onSnapshotFailure(Snapshot failedSnapshot, Exception e) {
                        if (failedSnapshot.equals(snapshot)) {
                            logger.trace("deleted snapshot failed - deleting files", e);
                            removeListener(this);
                            threadPool.executor(ThreadPool.Names.SNAPSHOT).execute(() -> deleteSnapshot(failedSnapshot.getRepository(), failedSnapshot.getSnapshotId().getName(), listener, true));
                        }
                    }
                });
            } else {
                logger.trace("deleted snapshot is not running - deleting files");
                deleteSnapshotFromRepository(snapshot, listener, repositoryStateId);
            }
        }
    });
}

Example 7 with ObjectObjectCursor

use of com.carrotsearch.hppc.cursors.ObjectObjectCursor in project elasticsearch by elastic.

the class SnapshotsService method endSnapshot.

/**
     * Finalizes the shard in repository and then removes it from cluster state
     * <p>
     * This is non-blocking method that runs on a thread from SNAPSHOT thread pool
     *
     * @param entry   snapshot
     * @param failure failure reason or null if snapshot was successful
     */
private void endSnapshot(final SnapshotsInProgress.Entry entry, final String failure) {
    threadPool.executor(ThreadPool.Names.SNAPSHOT).execute(new Runnable() {

        @Override
        public void run() {
            final Snapshot snapshot = entry.snapshot();
            try {
                final Repository repository = repositoriesService.repository(snapshot.getRepository());
                logger.trace("[{}] finalizing snapshot in repository, state: [{}], failure[{}]", snapshot, entry.state(), failure);
                ArrayList<ShardSearchFailure> failures = new ArrayList<>();
                ArrayList<SnapshotShardFailure> shardFailures = new ArrayList<>();
                for (ObjectObjectCursor<ShardId, ShardSnapshotStatus> shardStatus : entry.shards()) {
                    ShardId shardId = shardStatus.key;
                    ShardSnapshotStatus status = shardStatus.value;
                    if (status.state().failed()) {
                        failures.add(new ShardSearchFailure(status.reason(), new SearchShardTarget(status.nodeId(), shardId)));
                        shardFailures.add(new SnapshotShardFailure(status.nodeId(), shardId, status.reason()));
                    }
                }
                SnapshotInfo snapshotInfo = repository.finalizeSnapshot(snapshot.getSnapshotId(), entry.indices(), entry.startTime(), failure, entry.shards().size(), Collections.unmodifiableList(shardFailures), entry.getRepositoryStateId());
                removeSnapshotFromClusterState(snapshot, snapshotInfo, null);
            } catch (Exception e) {
                logger.warn((Supplier<?>) () -> new ParameterizedMessage("[{}] failed to finalize snapshot", snapshot), e);
                removeSnapshotFromClusterState(snapshot, null, e);
            }
        }
    });
}

Example 8 with ObjectObjectCursor

use of com.carrotsearch.hppc.cursors.ObjectObjectCursor in project elasticsearch by elastic.

the class StoreRecovery method recoverFromLocalShards.

boolean recoverFromLocalShards(BiConsumer<String, MappingMetaData> mappingUpdateConsumer, final IndexShard indexShard, final List<LocalShardSnapshot> shards) throws IOException {
    if (canRecover(indexShard)) {
        RecoverySource.Type recoveryType = indexShard.recoveryState().getRecoverySource().getType();
        assert recoveryType == RecoverySource.Type.LOCAL_SHARDS : "expected local shards recovery type: " + recoveryType;
        if (shards.isEmpty()) {
            throw new IllegalArgumentException("shards must not be empty");
        }
        Set<Index> indices = shards.stream().map((s) -> s.getIndex()).collect(Collectors.toSet());
        if (indices.size() > 1) {
            throw new IllegalArgumentException("can't add shards from more than one index");
        }
        IndexMetaData indexMetaData = shards.get(0).getIndexMetaData();
        for (ObjectObjectCursor<String, MappingMetaData> mapping : indexMetaData.getMappings()) {
            mappingUpdateConsumer.accept(mapping.key, mapping.value);
        }
        indexShard.mapperService().merge(indexMetaData, MapperService.MergeReason.MAPPING_RECOVERY, true);
        return executeRecovery(indexShard, () -> {
            logger.debug("starting recovery from local shards {}", shards);
            try {
                // don't close this directory!!
                final Directory directory = indexShard.store().directory();
                addIndices(indexShard.recoveryState().getIndex(), directory, shards.stream().map(s -> s.getSnapshotDirectory()).collect(Collectors.toList()).toArray(new Directory[shards.size()]));
                internalRecoverFromStore(indexShard);
                // just trigger a merge to do housekeeping on the
                // copied segments - we will also see them in stats etc.
                indexShard.getEngine().forceMerge(false, -1, false, false, false);
            } catch (IOException ex) {
                throw new IndexShardRecoveryException(indexShard.shardId(), "failed to recover from local shards", ex);
            }
        });
    }
    return false;
}

Example 9 with ObjectObjectCursor

use of com.carrotsearch.hppc.cursors.ObjectObjectCursor in project crate by crate.

the class DocSchemaInfo method update.

@Override
public void update(ClusterChangedEvent event) {
    assert event.metadataChanged() : "metadataChanged must be true if update is called";
    // search for aliases of deleted and created indices, they must be invalidated also
    Metadata prevMetadata = event.previousState().metadata();
    for (Index index : event.indicesDeleted()) {
        invalidateFromIndex(index, prevMetadata);
    }
    Metadata newMetadata = event.state().metadata();
    for (String index : event.indicesCreated()) {
        invalidateAliases(newMetadata.index(index).getAliases());
    }
    // search for templates with changed meta data => invalidate template aliases
    ImmutableOpenMap<String, IndexTemplateMetadata> newTemplates = newMetadata.templates();
    ImmutableOpenMap<String, IndexTemplateMetadata> prevTemplates = prevMetadata.templates();
    if (!newTemplates.equals(prevTemplates)) {
        for (ObjectCursor<IndexTemplateMetadata> cursor : newTemplates.values()) {
            invalidateAliases(cursor.value.aliases());
        }
        for (ObjectCursor<IndexTemplateMetadata> cursor : prevTemplates.values()) {
            invalidateAliases(cursor.value.aliases());
        }
    }
    // search indices with changed meta data
    Iterator<String> currentTablesIt = docTableByName.keySet().iterator();
    ObjectLookupContainer<String> templates = newTemplates.keys();
    ImmutableOpenMap<String, IndexMetadata> indices = newMetadata.indices();
    while (currentTablesIt.hasNext()) {
        String tableName = currentTablesIt.next();
        String indexName = getIndexName(tableName);
        IndexMetadata newIndexMetadata = newMetadata.index(indexName);
        if (newIndexMetadata == null) {
            docTableByName.remove(tableName);
        } else {
            IndexMetadata oldIndexMetadata = prevMetadata.index(indexName);
            if (oldIndexMetadata != null && ClusterChangedEvent.indexMetadataChanged(oldIndexMetadata, newIndexMetadata)) {
                docTableByName.remove(tableName);
                // invalidate aliases of changed indices
                invalidateAliases(newIndexMetadata.getAliases());
                invalidateAliases(oldIndexMetadata.getAliases());
            } else {
                // this is the case if a single partition has been modified using alter table <t> partition (...)
                String possibleTemplateName = PartitionName.templateName(name(), tableName);
                if (templates.contains(possibleTemplateName)) {
                    for (ObjectObjectCursor<String, IndexMetadata> indexEntry : indices) {
                        if (IndexParts.isPartitioned(indexEntry.key)) {
                            docTableByName.remove(tableName);
                            break;
                        }
                    }
                }
            }
        }
    }
    // re register UDFs for this schema
    UserDefinedFunctionsMetadata udfMetadata = newMetadata.custom(UserDefinedFunctionsMetadata.TYPE);
    if (udfMetadata != null) {
        udfService.updateImplementations(schemaName, udfMetadata.functionsMetadata().stream().filter(f -> schemaName.equals(f.schema())));
    }
}

Example 10 with ObjectObjectCursor

use of com.carrotsearch.hppc.cursors.ObjectObjectCursor in project crate by crate.

the class SnapshotsService method processSnapshotsOnRemovedNodes.

/**
 * Cleans up shard snapshots that were running on removed nodes
 */
private void processSnapshotsOnRemovedNodes() {
    clusterService.submitStateUpdateTask("update snapshot state after node removal", new ClusterStateUpdateTask() {

        @Override
        public ClusterState execute(ClusterState currentState) {
            DiscoveryNodes nodes = currentState.nodes();
            SnapshotsInProgress snapshots = currentState.custom(SnapshotsInProgress.TYPE);
            if (snapshots == null) {
                return currentState;
            }
            boolean changed = false;
            ArrayList<SnapshotsInProgress.Entry> entries = new ArrayList<>();
            for (final SnapshotsInProgress.Entry snapshot : snapshots.entries()) {
                SnapshotsInProgress.Entry updatedSnapshot = snapshot;
                if (snapshot.state() == State.STARTED || snapshot.state() == State.ABORTED) {
                    ImmutableOpenMap.Builder<ShardId, ShardSnapshotStatus> shards = ImmutableOpenMap.builder();
                    boolean snapshotChanged = false;
                    for (ObjectObjectCursor<ShardId, ShardSnapshotStatus> shardEntry : snapshot.shards()) {
                        final ShardSnapshotStatus shardStatus = shardEntry.value;
                        final ShardId shardId = shardEntry.key;
                        if (!shardStatus.state().completed() && shardStatus.nodeId() != null) {
                            if (nodes.nodeExists(shardStatus.nodeId())) {
                                shards.put(shardId, shardStatus);
                            } else {
                                // TODO: Restart snapshot on another node?
                                snapshotChanged = true;
                                LOGGER.warn("failing snapshot of shard [{}] on closed node [{}]", shardId, shardStatus.nodeId());
                                shards.put(shardId, new ShardSnapshotStatus(shardStatus.nodeId(), ShardState.FAILED, "node shutdown", shardStatus.generation()));
                            }
                        } else {
                            shards.put(shardId, shardStatus);
                        }
                    }
                    if (snapshotChanged) {
                        changed = true;
                        ImmutableOpenMap<ShardId, ShardSnapshotStatus> shardsMap = shards.build();
                        if (!snapshot.state().completed() && completed(shardsMap.values())) {
                            updatedSnapshot = new SnapshotsInProgress.Entry(snapshot, State.SUCCESS, shardsMap);
                        } else {
                            updatedSnapshot = new SnapshotsInProgress.Entry(snapshot, snapshot.state(), shardsMap);
                        }
                    }
                    entries.add(updatedSnapshot);
                } else if (snapshot.state() == State.INIT && initializingSnapshots.contains(snapshot.snapshot()) == false) {
                    changed = true;
                    // Mark the snapshot as aborted as it failed to start from the previous master
                    updatedSnapshot = new SnapshotsInProgress.Entry(snapshot, State.ABORTED, snapshot.shards());
                    entries.add(updatedSnapshot);
                    // Clean up the snapshot that failed to start from the old master
                    deleteSnapshot(snapshot.snapshot(), new ActionListener<Void>() {

                        @Override
                        public void onResponse(Void aVoid) {
                            LOGGER.debug("cleaned up abandoned snapshot {} in INIT state", snapshot.snapshot());
                        }

                        @Override
                        public void onFailure(Exception e) {
                            LOGGER.warn("failed to clean up abandoned snapshot {} in INIT state", snapshot.snapshot());
                        }
                    }, updatedSnapshot.repositoryStateId(), false);
                }
                assert updatedSnapshot.shards().size() == snapshot.shards().size() : "Shard count changed during snapshot status update from [" + snapshot + "] to [" + updatedSnapshot + "]";
            }
            if (changed) {
                return ClusterState.builder(currentState).putCustom(SnapshotsInProgress.TYPE, new SnapshotsInProgress(unmodifiableList(entries))).build();
            }
            return currentState;
        }

        @Override
        public void onFailure(String source, Exception e) {
            LOGGER.warn("failed to update snapshot state after node removal");
        }
    });
}