Examples with Cluster org.graylog2.indexer.cluster.Cluster used on opensource projects

Example 11 with Cluster

use of org.graylog2.indexer.cluster.Cluster in project graylog2-server by Graylog2.

the class Indices method getAllMessageFieldsForIndices.

public Map<String, Set<String>> getAllMessageFieldsForIndices(final String[] writeIndexWildcards) {
    final Map<String, Set<String>> fields = new HashMap<>();
    final ClusterStateRequest csr = new ClusterStateRequest().blocks(true).nodes(true).indices(writeIndexWildcards);
    final ClusterState cs = c.admin().cluster().state(csr).actionGet().getState();
    for (ObjectObjectCursor<String, IndexMetaData> m : cs.getMetaData().indices()) {
        try {
            MappingMetaData mmd = m.value.mapping(IndexMapping.TYPE_MESSAGE);
            if (mmd == null) {
                // There is no mapping if there are no messages in the index.
                continue;
            }
            @SuppressWarnings("unchecked") final Map<String, Object> mapping = (Map<String, Object>) mmd.getSourceAsMap().get("properties");
            if (mapping != null) {
                fields.put(m.key, mapping.keySet());
            }
        } catch (Exception e) {
            LOG.error("Error while trying to get fields of <" + m.index + ">", e);
        }
    }
    return fields;
}

Example 12 with Cluster

use of org.graylog2.indexer.cluster.Cluster in project graylog2-server by Graylog2.

the class V20161130141500_DefaultStreamRecalcIndexRanges method upgrade.

@Override
public void upgrade() {
    IndexSet indexSet;
    try {
        indexSet = indexSetRegistry.getDefault();
    } catch (IllegalStateException e) {
        // Try to find the default index set manually if the index set registry cannot find it.
        // This is needed if you run this migration with a 2.2.0-beta.2 state before commit 460cac6af.
        final IndexSetConfig indexSetConfig = indexSetService.findOne(DBQuery.is("default", true)).orElseThrow(() -> new IllegalStateException("No default index set configured! This is a bug!"));
        indexSet = indexSetFactory.create(indexSetConfig);
    }
    final IndexSet defaultIndexSet = indexSet;
    if (!cluster.isConnected()) {
        LOG.info("Cluster not connected yet, delaying migration until it is reachable.");
        while (true) {
            try {
                cluster.waitForConnectedAndDeflectorHealthy();
                break;
            } catch (InterruptedException | TimeoutException e) {
                LOG.warn("Interrupted or timed out waiting for Elasticsearch cluster, checking again.");
            }
        }
    }
    final Set<String> indexRangesWithoutStreams = indexRangeService.findAll().stream().filter(indexRange -> defaultIndexSet.isManagedIndex(indexRange.indexName())).filter(indexRange -> indexRange.streamIds() == null).map(IndexRange::indexName).collect(Collectors.toSet());
    if (indexRangesWithoutStreams.size() == 0) {
        // all ranges have a stream list, even if it is empty, nothing more to do
        return;
    }
    final String currentWriteTarget;
    try {
        currentWriteTarget = defaultIndexSet.getActiveWriteIndex();
    } catch (TooManyAliasesException e) {
        LOG.error("Multiple write targets found for write alias. Cannot continue to assign streams to older indices", e);
        return;
    }
    for (String indexName : defaultIndexSet.getManagedIndices()) {
        if (indexName.equals(currentWriteTarget)) {
            // do not recalculate for current write target
            continue;
        }
        if (!indexRangesWithoutStreams.contains(indexName)) {
            // already computed streams for this index
            continue;
        }
        LOG.info("Recalculating streams in index {}", indexName);
        final CreateNewSingleIndexRangeJob createNewSingleIndexRangeJob = rebuildIndexRangeJobFactory.create(indexSetRegistry.getAll(), indexName);
        createNewSingleIndexRangeJob.execute();
    }
}

Example 13 with Cluster

use of org.graylog2.indexer.cluster.Cluster in project graylog2-server by Graylog2.

the class ClusterIdGeneratorPeriodical method doRun.

@Override
public void doRun() {
    if (clusterConfigService.get(ClusterId.class) == null) {
        ClusterId clusterId = ClusterId.create(UUID.randomUUID().toString());
        clusterConfigService.write(clusterId);
        LOG.debug("Generated cluster ID {}", clusterId.clusterId());
    }
}

Example 14 with Cluster

use of org.graylog2.indexer.cluster.Cluster in project graylog2-server by Graylog2.

the class IndexRangesCleanupPeriodical method doRun.

@Override
public void doRun() {
    if (!cluster.isConnected() || !cluster.isHealthy()) {
        LOG.info("Skipping index range cleanup because the Elasticsearch cluster is unreachable or unhealthy");
        return;
    }
    final Set<String> indexNames = ImmutableSet.copyOf(indexSetRegistry.getManagedIndices());
    final SortedSet<IndexRange> indexRanges = indexRangeService.findAll();
    final Set<String> removedIndices = new HashSet<>();
    for (IndexRange indexRange : indexRanges) {
        if (!indexNames.contains(indexRange.indexName())) {
            removedIndices.add(indexRange.indexName());
        }
    }
    if (!removedIndices.isEmpty()) {
        LOG.info("Removing index range information for unavailable indices: {}", removedIndices);
        eventBus.post(IndicesDeletedEvent.create(removedIndices));
    }
}

Example 15 with Cluster

use of org.graylog2.indexer.cluster.Cluster in project graylog2-server by Graylog2.

the class IndexerClusterCheckerThread method doRun.

@Override
public void doRun() {
    if (!notificationService.isFirst(Notification.Type.ES_OPEN_FILES)) {
        return;
    }
    try {
        cluster.health().getStatus();
    } catch (Exception e) {
        LOG.info("Indexer not fully initialized yet. Skipping periodic cluster check.");
        return;
    }
    boolean allHigher = true;
    final Map<String, NodeInfo> nodesInfos = cluster.getDataNodes();
    final Map<String, NodeStats> nodesStats = cluster.getNodesStats(nodesInfos.keySet().toArray(new String[nodesInfos.size()]));
    for (Map.Entry<String, NodeStats> entry : nodesStats.entrySet()) {
        final String nodeId = entry.getKey();
        final NodeStats nodeStats = entry.getValue();
        final NodeInfo nodeInfo = nodesInfos.get(nodeId);
        final String nodeName = nodeInfo.getNode().getName();
        // Check number of maximum open files.
        final ProcessStats processStats = nodeStats.getProcess();
        if (processStats == null) {
            LOG.debug("Couldn't read process stats of Elasticsearch node {}", nodeName);
            return;
        }
        final long maxFileDescriptors = processStats.getMaxFileDescriptors();
        final JvmInfo jvmInfo = nodeInfo.getJvm();
        if (jvmInfo == null) {
            LOG.debug("Couldn't read JVM info of Elasticsearch node {}", nodeName);
            return;
        }
        final String osName = jvmInfo.getSystemProperties().getOrDefault("os.name", "");
        if (osName.startsWith("Windows")) {
            LOG.debug("Skipping open file limit check for Indexer node <{}> on Windows", nodeName);
        } else if (maxFileDescriptors != -1 && maxFileDescriptors < MINIMUM_OPEN_FILES_LIMIT) {
            // Write notification.
            final Notification notification = notificationService.buildNow().addType(Notification.Type.ES_OPEN_FILES).addSeverity(Notification.Severity.URGENT).addDetail("hostname", nodeInfo.getHostname()).addDetail("max_file_descriptors", maxFileDescriptors);
            if (notificationService.publishIfFirst(notification)) {
                LOG.warn("Indexer node <{}> open file limit is too low: [{}]. Set it to at least {}.", nodeName, maxFileDescriptors, MINIMUM_OPEN_FILES_LIMIT);
            }
            allHigher = false;
        }
    }
    if (allHigher) {
        Notification notification = notificationService.build().addType(Notification.Type.ES_OPEN_FILES);
        notificationService.fixed(notification);
    }
}