Examples with Lifecycle org.graylog2.plugin.lifecycles.Lifecycle used on opensource projects

Example 1 with Lifecycle

use of org.graylog2.plugin.lifecycles.Lifecycle in project graylog2-server by Graylog2.

the class HttpPollTransport method doLaunch.

@Override
public void doLaunch(final MessageInput input) throws MisfireException {
    serverStatus.awaitRunning(() -> lifecycleStateChange(Lifecycle.RUNNING));
    // listen for lifecycle changes
    serverEventBus.register(this);
    final Map<String, String> headers = parseHeaders(configuration.getString(CK_HEADERS));
    // figure out a reasonable remote address
    final String url = configuration.getString(CK_URL);
    final InetSocketAddress remoteAddress;
    InetSocketAddress remoteAddress1;
    try {
        final URL url1 = new URL(url);
        final int port = url1.getPort();
        remoteAddress1 = new InetSocketAddress(url1.getHost(), port != -1 ? port : 80);
    } catch (MalformedURLException e) {
        remoteAddress1 = null;
    }
    remoteAddress = remoteAddress1;
    final Runnable task = () -> {
        if (paused) {
            LOG.debug("Message processing paused, not polling HTTP resource {}.", url);
            return;
        }
        if (isThrottled()) {
            // this transport won't block, but we can simply skip this iteration
            LOG.debug("Not polling HTTP resource {} because we are throttled.", url);
            return;
        }
        final Request.Builder requestBuilder = new Request.Builder().get().url(url).headers(Headers.of(headers));
        try (final Response r = httpClient.newCall(requestBuilder.build()).execute()) {
            if (!r.isSuccessful()) {
                LOG.error("Expected successful HTTP status code [2xx], got " + r.code());
                return;
            }
            input.processRawMessage(new RawMessage(r.body().bytes(), remoteAddress));
        } catch (IOException e) {
            LOG.error("Could not fetch HTTP resource at " + url, e);
        }
    };
    scheduledFuture = scheduler.scheduleAtFixedRate(task, 0, configuration.getInt(CK_INTERVAL), TimeUnit.valueOf(configuration.getString(CK_TIMEUNIT)));
}

Example 2 with Lifecycle

use of org.graylog2.plugin.lifecycles.Lifecycle in project graylog2-server by Graylog2.

the class KafkaTransport method doLaunch.

@Override
public void doLaunch(final MessageInput input) throws MisfireException {
    serverStatus.awaitRunning(new Runnable() {

        @Override
        public void run() {
            lifecycleStateChange(Lifecycle.RUNNING);
        }
    });
    // listen for lifecycle changes
    serverEventBus.register(this);
    final Properties props = new Properties();
    props.put("group.id", GROUP_ID);
    props.put("client.id", "gl2-" + nodeId + "-" + input.getId());
    props.put("fetch.min.bytes", String.valueOf(configuration.getInt(CK_FETCH_MIN_BYTES)));
    props.put("fetch.wait.max.ms", String.valueOf(configuration.getInt(CK_FETCH_WAIT_MAX)));
    props.put("zookeeper.connect", configuration.getString(CK_ZOOKEEPER));
    // Default auto commit interval is 60 seconds. Reduce to 1 second to minimize message duplication
    // if something breaks.
    props.put("auto.commit.interval.ms", "1000");
    // Set a consumer timeout to avoid blocking on the consumer iterator.
    props.put("consumer.timeout.ms", "1000");
    final int numThreads = configuration.getInt(CK_THREADS);
    final ConsumerConfig consumerConfig = new ConsumerConfig(props);
    cc = Consumer.createJavaConsumerConnector(consumerConfig);
    final TopicFilter filter = new Whitelist(configuration.getString(CK_TOPIC_FILTER));
    final List<KafkaStream<byte[], byte[]>> streams = cc.createMessageStreamsByFilter(filter, numThreads);
    final ExecutorService executor = executorService(numThreads);
    // this is being used during shutdown to first stop all submitted jobs before committing the offsets back to zookeeper
    // and then shutting down the connection.
    // this is to avoid yanking away the connection from the consumer runnables
    stopLatch = new CountDownLatch(streams.size());
    for (final KafkaStream<byte[], byte[]> stream : streams) {
        executor.submit(new Runnable() {

            @Override
            public void run() {
                final ConsumerIterator<byte[], byte[]> consumerIterator = stream.iterator();
                boolean retry;
                do {
                    retry = false;
                    try {
                        // noinspection WhileLoopReplaceableByForEach
                        while (consumerIterator.hasNext()) {
                            if (paused) {
                                // we try not to spin here, so we wait until the lifecycle goes back to running.
                                LOG.debug("Message processing is paused, blocking until message processing is turned back on.");
                                Uninterruptibles.awaitUninterruptibly(pausedLatch);
                            }
                            // check for being stopped before actually getting the message, otherwise we could end up losing that message
                            if (stopped) {
                                break;
                            }
                            if (isThrottled()) {
                                blockUntilUnthrottled();
                            }
                            // process the message, this will immediately mark the message as having been processed. this gets tricky
                            // if we get an exception about processing it down below.
                            final MessageAndMetadata<byte[], byte[]> message = consumerIterator.next();
                            final byte[] bytes = message.message();
                            // it is possible that the message is null
                            if (bytes == null) {
                                continue;
                            }
                            totalBytesRead.addAndGet(bytes.length);
                            lastSecBytesReadTmp.addAndGet(bytes.length);
                            final RawMessage rawMessage = new RawMessage(bytes);
                            // TODO implement throttling
                            input.processRawMessage(rawMessage);
                        }
                    } catch (ConsumerTimeoutException e) {
                        // Happens when there is nothing to consume, retry to check again.
                        retry = true;
                    } catch (Exception e) {
                        LOG.error("Kafka consumer error, stopping consumer thread.", e);
                    }
                } while (retry && !stopped);
                // explicitly commit our offsets when stopping.
                // this might trigger a couple of times, but it won't hurt
                cc.commitOffsets();
                stopLatch.countDown();
            }
        });
    }
    scheduler.scheduleAtFixedRate(new Runnable() {

        @Override
        public void run() {
            lastSecBytesRead.set(lastSecBytesReadTmp.getAndSet(0));
        }
    }, 1, 1, TimeUnit.SECONDS);
}

Example 3 with Lifecycle

use of org.graylog2.plugin.lifecycles.Lifecycle in project graylog2-server by Graylog2.

the class KafkaTransport method doLaunchLegacy.

private void doLaunchLegacy(final MessageInput input) {
    final Properties props = new Properties();
    props.put("group.id", configuration.getString(CK_GROUP_ID, DEFAULT_GROUP_ID));
    props.put("client.id", "gl2-" + nodeId.getShortNodeId() + "-" + input.getId());
    props.put("fetch.min.bytes", String.valueOf(configuration.getInt(CK_FETCH_MIN_BYTES)));
    props.put("fetch.wait.max.ms", String.valueOf(configuration.getInt(CK_FETCH_WAIT_MAX)));
    props.put("zookeeper.connect", configuration.getString(CK_ZOOKEEPER));
    props.put("auto.offset.reset", configuration.getString(CK_OFFSET_RESET, DEFAULT_OFFSET_RESET));
    // Default auto commit interval is 60 seconds. Reduce to 1 second to minimize message duplication
    // if something breaks.
    props.put("auto.commit.interval.ms", "1000");
    // Set a consumer timeout to avoid blocking on the consumer iterator.
    props.put("consumer.timeout.ms", "1000");
    insertCustomProperties(props);
    final int numThreads = configuration.getInt(CK_THREADS);
    final ConsumerConfig consumerConfig = new ConsumerConfig(props);
    cc = Consumer.createJavaConsumerConnector(consumerConfig);
    final TopicFilter filter = new Whitelist(configuration.getString(CK_TOPIC_FILTER));
    final List<KafkaStream<byte[], byte[]>> streams = cc.createMessageStreamsByFilter(filter, numThreads);
    // this is being used during shutdown to first stop all submitted jobs before committing the offsets back to zookeeper
    // and then shutting down the connection.
    // this is to avoid yanking away the connection from the consumer runnables
    stopLatch = new CountDownLatch(streams.size());
    for (final KafkaStream<byte[], byte[]> stream : streams) {
        executor.submit(new Runnable() {

            @Override
            public void run() {
                final ConsumerIterator<byte[], byte[]> consumerIterator = stream.iterator();
                boolean retry;
                do {
                    retry = false;
                    try {
                        // noinspection WhileLoopReplaceableByForEach
                        while (consumerIterator.hasNext()) {
                            if (paused) {
                                // we try not to spin here, so we wait until the lifecycle goes back to running.
                                LOG.debug("Message processing is paused, blocking until message processing is turned back on.");
                                Uninterruptibles.awaitUninterruptibly(pausedLatch);
                            }
                            // check for being stopped before actually getting the message, otherwise we could end up losing that message
                            if (stopped) {
                                break;
                            }
                            if (isThrottled()) {
                                blockUntilUnthrottled();
                            }
                            // process the message, this will immediately mark the message as having been processed. this gets tricky
                            // if we get an exception about processing it down below.
                            final MessageAndMetadata<byte[], byte[]> message = consumerIterator.next();
                            final byte[] bytes = message.message();
                            // it is possible that the message is null
                            if (bytes == null) {
                                continue;
                            }
                            totalBytesRead.addAndGet(bytes.length);
                            lastSecBytesReadTmp.addAndGet(bytes.length);
                            final RawMessage rawMessage = new RawMessage(bytes);
                            input.processRawMessage(rawMessage);
                        }
                    } catch (ConsumerTimeoutException e) {
                        // Happens when there is nothing to consume, retry to check again.
                        retry = true;
                    } catch (Exception e) {
                        LOG.error("Kafka consumer error, stopping consumer thread.", e);
                    }
                } while (retry && !stopped);
                // explicitly commit our offsets when stopping.
                // this might trigger a couple of times, but it won't hurt
                cc.commitOffsets();
                stopLatch.countDown();
            }
        });
    }
}

Example 4 with Lifecycle

use of org.graylog2.plugin.lifecycles.Lifecycle in project graylog2-server by Graylog2.

the class ClusterLoadBalancerStatusResource method override.

@PUT
@Timed
@RequiresAuthentication
@RequiresPermissions(RestPermissions.LBSTATUS_CHANGE)
@ApiOperation(value = "Override load balancer status of this graylog-server node. Next lifecycle " + "change will override it again to its default. Set to ALIVE, DEAD, or THROTTLED.")
@Path("/override/{status}")
@NoAuditEvent("this is a proxy resource, the audit event will be emitted on the target node")
public void override(@ApiParam(name = "nodeId", value = "The id of the node whose LB status will be changed", required = true) @PathParam("nodeId") String nodeId, @ApiParam(name = "status") @PathParam("status") String status) throws IOException, NodeNotFoundException {
    final Node targetNode = nodeService.byNodeId(nodeId);
    RemoteLoadBalancerStatusResource remoteLoadBalancerStatusResource = remoteInterfaceProvider.get(targetNode, this.authenticationToken, RemoteLoadBalancerStatusResource.class);
    final Response response = remoteLoadBalancerStatusResource.override(status).execute();
    if (!response.isSuccessful()) {
        LOG.warn("Unable to override load balancer status on node {}: {}", nodeId, response.message());
        throw new WebApplicationException(response.message(), BAD_GATEWAY);
    }
}

Example 5 with Lifecycle

use of org.graylog2.plugin.lifecycles.Lifecycle in project graylog2-server by Graylog2.

the class GracefulShutdown method doRun.

private void doRun(boolean exit) {
    LOG.info("Graceful shutdown initiated.");
    // Trigger a lifecycle change. Some services are listening for those and will halt operation accordingly.
    serverStatus.shutdown();
    // Give possible load balancers time to recognize state change. State is DEAD because of HALTING.
    LOG.info("Node status: [{}]. Waiting <{}sec> for possible load balancers to recognize state change.", serverStatus.getLifecycle(), configuration.getLoadBalancerRecognitionPeriodSeconds());
    Uninterruptibles.sleepUninterruptibly(configuration.getLoadBalancerRecognitionPeriodSeconds(), TimeUnit.SECONDS);
    activityWriter.write(new Activity("Graceful shutdown initiated.", GracefulShutdown.class));
    /*
         * Wait a second to give for example the calling REST call some time to respond
         * to the client. Using a latch or something here might be a bit over-engineered.
         */
    Uninterruptibles.sleepUninterruptibly(SLEEP_SECS, TimeUnit.SECONDS);
    // Stop REST API service to avoid changes from outside.
    jerseyService.stopAsync();
    // stop all inputs so no new messages can come in
    inputSetupService.stopAsync();
    jerseyService.awaitTerminated();
    inputSetupService.awaitTerminated();
    // Try to flush all remaining messages from the system
    bufferSynchronizerService.stopAsync().awaitTerminated();
    // Stop all services that registered with the shutdown service (e.g. plugins)
    // This must run after the BufferSynchronizerService shutdown to make sure the buffers are empty.
    gracefulShutdownService.stopAsync();
    // stop all maintenance tasks
    periodicalsService.stopAsync().awaitTerminated();
    // Wait until the shutdown service is done
    gracefulShutdownService.awaitTerminated();
    auditEventSender.success(AuditActor.system(serverStatus.getNodeId()), NODE_SHUTDOWN_COMPLETE);
    // Shut down hard with no shutdown hooks running.
    LOG.info("Goodbye.");
    if (exit) {
        System.exit(0);
    }
}