Examples with ObjectLookupContainer com.carrotsearch.hppc.ObjectLookupContainer used on opensource projects

Example 1 with ObjectLookupContainer

use of com.carrotsearch.hppc.ObjectLookupContainer in project crate by crate.

the class DocSchemaInfo method update.

@Override
public void update(ClusterChangedEvent event) {
    assert event.metadataChanged() : "metadataChanged must be true if update is called";
    // search for aliases of deleted and created indices, they must be invalidated also
    Metadata prevMetadata = event.previousState().metadata();
    for (Index index : event.indicesDeleted()) {
        invalidateFromIndex(index, prevMetadata);
    }
    Metadata newMetadata = event.state().metadata();
    for (String index : event.indicesCreated()) {
        invalidateAliases(newMetadata.index(index).getAliases());
    }
    // search for templates with changed meta data => invalidate template aliases
    ImmutableOpenMap<String, IndexTemplateMetadata> newTemplates = newMetadata.templates();
    ImmutableOpenMap<String, IndexTemplateMetadata> prevTemplates = prevMetadata.templates();
    if (!newTemplates.equals(prevTemplates)) {
        for (ObjectCursor<IndexTemplateMetadata> cursor : newTemplates.values()) {
            invalidateAliases(cursor.value.aliases());
        }
        for (ObjectCursor<IndexTemplateMetadata> cursor : prevTemplates.values()) {
            invalidateAliases(cursor.value.aliases());
        }
    }
    // search indices with changed meta data
    Iterator<String> currentTablesIt = docTableByName.keySet().iterator();
    ObjectLookupContainer<String> templates = newTemplates.keys();
    ImmutableOpenMap<String, IndexMetadata> indices = newMetadata.indices();
    while (currentTablesIt.hasNext()) {
        String tableName = currentTablesIt.next();
        String indexName = getIndexName(tableName);
        IndexMetadata newIndexMetadata = newMetadata.index(indexName);
        if (newIndexMetadata == null) {
            docTableByName.remove(tableName);
        } else {
            IndexMetadata oldIndexMetadata = prevMetadata.index(indexName);
            if (oldIndexMetadata != null && ClusterChangedEvent.indexMetadataChanged(oldIndexMetadata, newIndexMetadata)) {
                docTableByName.remove(tableName);
                // invalidate aliases of changed indices
                invalidateAliases(newIndexMetadata.getAliases());
                invalidateAliases(oldIndexMetadata.getAliases());
            } else {
                // this is the case if a single partition has been modified using alter table <t> partition (...)
                String possibleTemplateName = PartitionName.templateName(name(), tableName);
                if (templates.contains(possibleTemplateName)) {
                    for (ObjectObjectCursor<String, IndexMetadata> indexEntry : indices) {
                        if (IndexParts.isPartitioned(indexEntry.key)) {
                            docTableByName.remove(tableName);
                            break;
                        }
                    }
                }
            }
        }
    }
    // re register UDFs for this schema
    UserDefinedFunctionsMetadata udfMetadata = newMetadata.custom(UserDefinedFunctionsMetadata.TYPE);
    if (udfMetadata != null) {
        udfService.updateImplementations(schemaName, udfMetadata.functionsMetadata().stream().filter(f -> schemaName.equals(f.schema())));
    }
}

Example 2 with ObjectLookupContainer

use of com.carrotsearch.hppc.ObjectLookupContainer in project crate by crate.

the class DiskThresholdMonitor method onNewInfo.

public void onNewInfo(ClusterInfo info) {
    if (checkInProgress.compareAndSet(false, true) == false) {
        LOGGER.info("skipping monitor as a check is already in progress");
        return;
    }
    final ImmutableOpenMap<String, DiskUsage> usages = info.getNodeLeastAvailableDiskUsages();
    if (usages == null) {
        checkFinished();
        return;
    }
    boolean reroute = false;
    String explanation = "";
    final long currentTimeMillis = currentTimeMillisSupplier.getAsLong();
    // Garbage collect nodes that have been removed from the cluster
    // from the map that tracks watermark crossing
    final ObjectLookupContainer<String> nodes = usages.keys();
    for (String node : nodeHasPassedWatermark) {
        if (nodes.contains(node) == false) {
            nodeHasPassedWatermark.remove(node);
        }
    }
    final ClusterState state = clusterStateSupplier.get();
    final Set<String> indicesToMarkReadOnly = new HashSet<>();
    for (final ObjectObjectCursor<String, DiskUsage> entry : usages) {
        final String node = entry.key;
        final DiskUsage usage = entry.value;
        warnAboutDiskIfNeeded(usage);
        if (usage.getFreeBytes() < diskThresholdSettings.getFreeBytesThresholdFloodStage().getBytes() || usage.getFreeDiskAsPercentage() < diskThresholdSettings.getFreeDiskThresholdFloodStage()) {
            final RoutingNode routingNode = state.getRoutingNodes().node(node);
            if (routingNode != null) {
                // this might happen if we haven't got the full cluster-state yet?!
                for (ShardRouting routing : routingNode) {
                    indicesToMarkReadOnly.add(routing.index().getName());
                }
            }
        } else if (usage.getFreeBytes() < diskThresholdSettings.getFreeBytesThresholdHigh().getBytes() || usage.getFreeDiskAsPercentage() < diskThresholdSettings.getFreeDiskThresholdHigh()) {
            if (lastRunTimeMillis.get() < currentTimeMillis - diskThresholdSettings.getRerouteInterval().millis()) {
                reroute = true;
                explanation = "high disk watermark exceeded on one or more nodes";
            } else {
                LOGGER.debug("high disk watermark exceeded on {} but an automatic reroute has occurred " + "in the last [{}], skipping reroute", node, diskThresholdSettings.getRerouteInterval());
            }
            nodeHasPassedWatermark.add(node);
        } else if (usage.getFreeBytes() < diskThresholdSettings.getFreeBytesThresholdLow().getBytes() || usage.getFreeDiskAsPercentage() < diskThresholdSettings.getFreeDiskThresholdLow()) {
            nodeHasPassedWatermark.add(node);
        } else {
            if (nodeHasPassedWatermark.contains(node)) {
                // if they are able to be
                if (lastRunTimeMillis.get() < currentTimeMillis - diskThresholdSettings.getRerouteInterval().millis()) {
                    reroute = true;
                    explanation = "one or more nodes has gone under the high or low watermark";
                    nodeHasPassedWatermark.remove(node);
                } else {
                    LOGGER.debug("{} has gone below a disk threshold, but an automatic reroute has occurred " + "in the last [{}], skipping reroute", node, diskThresholdSettings.getRerouteInterval());
                }
            }
        }
    }
    final ActionListener<Void> listener = new GroupedActionListener<>(ActionListener.wrap(this::checkFinished), 2);
    if (reroute) {
        LOGGER.info("rerouting shards: [{}]", explanation);
        rerouteService.reroute("disk threshold monitor", Priority.HIGH, ActionListener.wrap(r -> {
            setLastRunTimeMillis();
            listener.onResponse(r);
        }, e -> {
            LOGGER.debug("reroute failed", e);
            setLastRunTimeMillis();
            listener.onFailure(e);
        }));
    } else {
        listener.onResponse(null);
    }
    indicesToMarkReadOnly.removeIf(index -> state.getBlocks().indexBlocked(ClusterBlockLevel.WRITE, index));
    if (indicesToMarkReadOnly.isEmpty() == false) {
        markIndicesReadOnly(indicesToMarkReadOnly, ActionListener.wrap(r -> {
            setLastRunTimeMillis();
            listener.onResponse(r);
        }, e -> {
            LOGGER.debug("marking indices readonly failed", e);
            setLastRunTimeMillis();
            listener.onFailure(e);
        }));
    } else {
        listener.onResponse(null);
    }
}