Examples with ServerStatus org.graylog2.plugin.ServerStatus used on opensource projects

Example 1 with ServerStatus

use of org.graylog2.plugin.ServerStatus in project graylog2-server by Graylog2.

the class DecodingProcessor method postProcessMessage.

@Nullable
private Message postProcessMessage(RawMessage raw, Codec codec, String inputIdOnCurrentNode, String baseMetricName, Message message, long decodeTime) {
    if (message == null) {
        metricRegistry.meter(name(baseMetricName, "failures")).mark();
        return null;
    }
    if (!message.isComplete()) {
        metricRegistry.meter(name(baseMetricName, "incomplete")).mark();
        if (LOG.isDebugEnabled()) {
            LOG.debug("Dropping incomplete message {} on input <{}>. Parsed fields: [{}]", raw, inputIdOnCurrentNode, message.getFields());
        }
        return null;
    }
    message.setMessageQueueId(raw.getMessageQueueId());
    message.recordTiming(serverStatus, "parse", decodeTime);
    metricRegistry.timer(name(baseMetricName, "parseTime")).update(decodeTime, TimeUnit.NANOSECONDS);
    for (final RawMessage.SourceNode node : raw.getSourceNodes()) {
        switch(node.type) {
            case SERVER:
                // Always use the last source node.
                if (message.getField(Message.FIELD_GL2_SOURCE_INPUT) != null) {
                    LOG.debug("Multiple server nodes ({} {}) set for message id {}", message.getField(Message.FIELD_GL2_SOURCE_INPUT), node.nodeId, message.getId());
                }
                message.addField(Message.FIELD_GL2_SOURCE_INPUT, node.inputId);
                message.addField(Message.FIELD_GL2_SOURCE_NODE, node.nodeId);
                break;
            // TODO Due to be removed in Graylog 3.x
            case RADIO:
                // Always use the last source node.
                if (message.getField(Message.FIELD_GL2_SOURCE_RADIO_INPUT) != null) {
                    LOG.debug("Multiple radio nodes ({} {}) set for message id {}", message.getField(Message.FIELD_GL2_SOURCE_RADIO_INPUT), node.nodeId, message.getId());
                }
                message.addField(Message.FIELD_GL2_SOURCE_RADIO_INPUT, node.inputId);
                message.addField(Message.FIELD_GL2_SOURCE_RADIO, node.nodeId);
                break;
        }
    }
    if (inputIdOnCurrentNode != null) {
        try {
            message.setSourceInputId(inputIdOnCurrentNode);
        } catch (RuntimeException e) {
            LOG.warn("Unable to find input with id " + inputIdOnCurrentNode + ", not setting input id in this message.", e);
        }
    }
    final ResolvableInetSocketAddress remoteAddress = raw.getRemoteAddress();
    if (remoteAddress != null) {
        final String addrString = InetAddresses.toAddrString(remoteAddress.getAddress());
        message.addField(Message.FIELD_GL2_REMOTE_IP, addrString);
        if (remoteAddress.getPort() > 0) {
            message.addField(Message.FIELD_GL2_REMOTE_PORT, remoteAddress.getPort());
        }
        if (remoteAddress.isReverseLookedUp()) {
            // avoid reverse lookup if the hostname is available
            message.addField(Message.FIELD_GL2_REMOTE_HOSTNAME, remoteAddress.getHostName());
        }
        if (Strings.isNullOrEmpty(message.getSource())) {
            message.setSource(addrString);
        }
    }
    if (codec.getConfiguration() != null && codec.getConfiguration().stringIsSet(Codec.Config.CK_OVERRIDE_SOURCE)) {
        message.setSource(codec.getConfiguration().getString(Codec.Config.CK_OVERRIDE_SOURCE));
    }
    // Make sure that there is a value for the source field.
    if (Strings.isNullOrEmpty(message.getSource())) {
        message.setSource("unknown");
    }
    // The raw message timestamp is the receive time of the message. It has been created before writing the raw
    // message to the journal.
    message.setReceiveTime(raw.getTimestamp());
    metricRegistry.meter(name(baseMetricName, "processedMessages")).mark();
    decodedTrafficCounter.inc(message.getSize());
    return message;
}

Example 2 with ServerStatus

use of org.graylog2.plugin.ServerStatus in project graylog2-server by Graylog2.

the class ServerBootstrap method startCommand.

@Override
protected void startCommand() {
    final AuditEventSender auditEventSender = injector.getInstance(AuditEventSender.class);
    final NodeId nodeId = injector.getInstance(NodeId.class);
    final String systemInformation = Tools.getSystemInformation();
    final Map<String, Object> auditEventContext = ImmutableMap.of("version", version.toString(), "java", systemInformation, "node_id", nodeId.toString());
    auditEventSender.success(AuditActor.system(nodeId), NODE_STARTUP_INITIATE, auditEventContext);
    final OS os = OS.getOs();
    LOG.info("Graylog {} {} starting up", commandName, version);
    LOG.info("JRE: {}", systemInformation);
    LOG.info("Deployment: {}", configuration.getInstallationSource());
    LOG.info("OS: {}", os.getPlatformName());
    LOG.info("Arch: {}", os.getArch());
    try {
        if (configuration.isLeader() && configuration.runMigrations()) {
            runMigrations();
        }
    } catch (Exception e) {
        LOG.warn("Exception while running migrations", e);
        System.exit(1);
    }
    final ServerStatus serverStatus = injector.getInstance(ServerStatus.class);
    serverStatus.initialize();
    startNodeRegistration(injector);
    final ActivityWriter activityWriter;
    final ServiceManager serviceManager;
    final Service leaderElectionService;
    try {
        activityWriter = injector.getInstance(ActivityWriter.class);
        serviceManager = injector.getInstance(ServiceManager.class);
        leaderElectionService = injector.getInstance(Key.get(Service.class, Names.named("LeaderElectionService")));
    } catch (ProvisionException e) {
        LOG.error("Guice error", e);
        annotateProvisionException(e);
        auditEventSender.failure(AuditActor.system(nodeId), NODE_STARTUP_INITIATE, auditEventContext);
        System.exit(-1);
        return;
    } catch (Exception e) {
        LOG.error("Unexpected exception", e);
        auditEventSender.failure(AuditActor.system(nodeId), NODE_STARTUP_INITIATE, auditEventContext);
        System.exit(-1);
        return;
    }
    Runtime.getRuntime().addShutdownHook(new Thread(injector.getInstance(shutdownHook())));
    // propagate default size to input plugins
    MessageInput.setDefaultRecvBufferSize(configuration.getUdpRecvBufferSizes());
    // Start services.
    final ServiceManagerListener serviceManagerListener = injector.getInstance(ServiceManagerListener.class);
    serviceManager.addListener(serviceManagerListener, MoreExecutors.directExecutor());
    try {
        leaderElectionService.startAsync().awaitRunning();
        serviceManager.startAsync().awaitHealthy();
    } catch (Exception e) {
        try {
            serviceManager.stopAsync().awaitStopped(configuration.getShutdownTimeout(), TimeUnit.MILLISECONDS);
        } catch (TimeoutException timeoutException) {
            LOG.error("Unable to shutdown properly on time. {}", serviceManager.servicesByState());
        }
        LOG.error("Graylog startup failed. Exiting. Exception was:", e);
        auditEventSender.failure(AuditActor.system(nodeId), NODE_STARTUP_INITIATE, auditEventContext);
        System.exit(-1);
    }
    LOG.info("Services started, startup times in ms: {}", serviceManager.startupTimes());
    activityWriter.write(new Activity("Started up.", Main.class));
    LOG.info("Graylog " + commandName + " up and running.");
    auditEventSender.success(AuditActor.system(nodeId), NODE_STARTUP_COMPLETE, auditEventContext);
    // Block forever.
    try {
        Thread.currentThread().join();
    } catch (InterruptedException e) {
        return;
    }
}

Example 3 with ServerStatus

use of org.graylog2.plugin.ServerStatus in project graylog2-server by Graylog2.

the class OutputBufferProcessor method onEvent.

/**
 * Each message will be written to one or more outputs.
 * <p>
 * The default output is always being used for every message, but optionally the message can be routed to additional
 * outputs, currently based on the stream outputs that are configured in the system.
 * </p>
 * <p>
 * The stream outputs are time limited so one bad output does not impact throughput too much. Essentially this means
 * that the work of writing to the outputs is performed, but the writer threads will not wait forever for stream
 * outputs to finish their work. <b>This might lead to increased memory usage!</b>
 * </p>
 * <p>
 * The default output, however, is allowed to block and is not subject to time limiting. This is important because it
 * can exert back pressure on the processing pipeline this way, making sure we don't run into excessive heap usage.
 * </p>
 *
 * @param event the message to write to outputs
 * @throws Exception
 */
@Override
public void onEvent(MessageEvent event) throws Exception {
    incomingMessages.mark();
    final Message msg = event.getMessage();
    if (msg == null) {
        LOG.debug("Skipping null message.");
        return;
    }
    LOG.debug("Processing message <{}> from OutputBuffer.", msg.getId());
    final Set<MessageOutput> messageOutputs = outputRouter.getStreamOutputsForMessage(msg);
    msg.recordCounter(serverStatus, "matched-outputs", messageOutputs.size());
    final Future<?> defaultOutputCompletion = processMessage(msg, defaultMessageOutput);
    final CountDownLatch streamOutputsDoneSignal = new CountDownLatch(messageOutputs.size());
    for (final MessageOutput output : messageOutputs) {
        processMessage(msg, output, streamOutputsDoneSignal);
    }
    // Wait until all writer threads for stream outputs have finished or timeout is reached.
    if (!streamOutputsDoneSignal.await(configuration.getOutputModuleTimeout(), TimeUnit.MILLISECONDS)) {
        LOG.warn("Timeout reached. Not waiting any longer for stream output writer threads to complete.");
    }
    // this exerts the back pressure to the system
    if (defaultOutputCompletion != null) {
        Uninterruptibles.getUninterruptibly(defaultOutputCompletion);
    } else {
        LOG.error("The default output future was null, this is a bug!");
    }
    if (msg.hasRecordings()) {
        LOG.debug("Message event trace: {}", msg.recordingsAsString());
    }
    outputThroughput.inc();
    LOG.debug("Wrote message <{}> to all outputs. Finished handling.", msg.getId());
    event.clearMessages();
}

Example 4 with ServerStatus

use of org.graylog2.plugin.ServerStatus in project graylog2-server by Graylog2.

the class MessageFilterChainProcessor method process.

@Override
public Messages process(Messages messages) {
    for (final MessageFilter filter : filterRegistry) {
        for (Message msg : messages) {
            final String timerName = name(filter.getClass(), "executionTime");
            final Timer timer = metricRegistry.timer(timerName);
            final Timer.Context timerContext = timer.time();
            try {
                LOG.debug("Applying filter [{}] on message <{}>.", filter.getName(), msg.getId());
                if (filter.filter(msg)) {
                    LOG.debug("Filter [{}] marked message <{}> to be discarded. Dropping message.", filter.getName(), msg.getId());
                    msg.setFilterOut(true);
                    filteredOutMessages.mark();
                    messageQueueAcknowledger.acknowledge(msg);
                }
            } catch (Exception e) {
                final String shortError = String.format(Locale.US, "Could not apply filter [%s] on message <%s>", filter.getName(), msg.getId());
                if (LOG.isDebugEnabled()) {
                    LOG.error("{}:", shortError, e);
                } else {
                    LOG.error("{}:\n{}", shortError, ExceptionUtils.getShortenedStackTrace(e));
                }
                msg.addProcessingError(new Message.ProcessingError(ProcessingFailureCause.MessageFilterException, shortError, ExceptionUtils.getRootCauseMessage(e)));
            } finally {
                final long elapsedNanos = timerContext.stop();
                msg.recordTiming(serverStatus, timerName, elapsedNanos);
            }
        }
    }
    return messages;
}

Example 5 with ServerStatus

use of org.graylog2.plugin.ServerStatus in project graylog2-server by Graylog2.

the class InputFacadeTest method setUp.

@Before
@SuppressForbidden("Using Executors.newSingleThreadExecutor() is okay in tests")
public void setUp() throws Exception {
    final MetricRegistry metricRegistry = new MetricRegistry();
    final ClusterEventBus clusterEventBus = new ClusterEventBus("cluster-event-bus", Executors.newSingleThreadExecutor());
    final GrokPatternService grokPatternService = new InMemoryGrokPatternService(clusterEventBus);
    grokPatternService.save(GrokPattern.create("GREEDY", ".*"));
    final EventBus clusterBus = new EventBus();
    final GrokPatternRegistry grokPatternRegistry = new GrokPatternRegistry(clusterBus, grokPatternService, Executors.newScheduledThreadPool(1));
    final ExtractorFactory extractorFactory = new ExtractorFactory(metricRegistry, grokPatternRegistry, lookupTableService);
    final ConverterFactory converterFactory = new ConverterFactory(lookupTableService);
    inputService = new InputServiceImpl(mongodb.mongoConnection(), extractorFactory, converterFactory, messageInputFactory, clusterEventBus);
    final InputRegistry inputRegistry = new InputRegistry();
    Set<PluginMetaData> pluginMetaData = new HashSet<>();
    Map<String, MessageInput.Factory<? extends MessageInput>> inputFactories = new HashMap<>();
    final FakeHttpMessageInput.Factory fakeHttpMessageInputFactory = mock(FakeHttpMessageInput.Factory.class);
    final FakeHttpMessageInput.Descriptor fakeHttpMessageInputDescriptor = mock(FakeHttpMessageInput.Descriptor.class);
    when(fakeHttpMessageInputFactory.getDescriptor()).thenReturn(fakeHttpMessageInputDescriptor);
    final RawUDPInput.Factory rawUDPInputFactory = mock(RawUDPInput.Factory.class);
    final RawUDPInput.Descriptor rawUDPInputDescriptor = mock(RawUDPInput.Descriptor.class);
    when(rawUDPInputFactory.getDescriptor()).thenReturn(rawUDPInputDescriptor);
    inputFactories.put("org.graylog2.inputs.random.FakeHttpMessageInput", fakeHttpMessageInputFactory);
    inputFactories.put("org.graylog2.inputs.raw.udp.RawUDPInput", rawUDPInputFactory);
    facade = new InputFacade(objectMapper, inputService, inputRegistry, dbLookupTableService, grokPatternService, messageInputFactory, extractorFactory, converterFactory, serverStatus, pluginMetaData, inputFactories);
}