Examples with Messages org.graylog2.indexer.messages.Messages used on opensource projects

Example 71 with Messages

use of org.graylog2.indexer.messages.Messages in project graylog2-server by Graylog2.

the class MessagesResource method all.

@GET
@Timed
@ApiOperation(value = "Get internal Graylog system messages")
@RequiresPermissions(RestPermissions.SYSTEMMESSAGES_READ)
@Produces(MediaType.APPLICATION_JSON)
public Map<String, Object> all(@ApiParam(name = "page", value = "Page") @QueryParam("page") int page) {
    final List<Map<String, Object>> messages = Lists.newArrayList();
    for (SystemMessage sm : systemMessageService.all(page(page))) {
        Map<String, Object> message = Maps.newHashMapWithExpectedSize(4);
        message.put("caller", sm.getCaller());
        message.put("content", sm.getContent());
        message.put("timestamp", Tools.getISO8601String(sm.getTimestamp()));
        message.put("node_id", sm.getNodeId());
        messages.add(message);
    }
    return ImmutableMap.of("messages", messages, "total", systemMessageService.totalCount());
}

Example 72 with Messages

use of org.graylog2.indexer.messages.Messages in project graylog2-server by Graylog2.

the class IndexRangesResource method rebuildIndex.

@POST
@Timed
@Path("/{index: [a-z_0-9-]+}/rebuild")
@ApiOperation(value = "Rebuild/sync index range information.", notes = "This triggers a system job that scans an index and stores meta information " + "about what indices contain messages in what time ranges. It atomically overwrites " + "already existing meta information.")
@ApiResponses(value = { @ApiResponse(code = 202, message = "Rebuild/sync system job triggered.") })
@Produces(MediaType.APPLICATION_JSON)
@AuditEvent(type = AuditEventTypes.ES_INDEX_RANGE_UPDATE_JOB)
public Response rebuildIndex(@ApiParam(name = "index", value = "The name of the Graylog-managed Elasticsearch index", required = true) @PathParam("index") @NotEmpty String index) {
    if (!indexSetRegistry.isManagedIndex(index)) {
        throw new BadRequestException(index + " is not a Graylog-managed Elasticsearch index.");
    }
    checkPermission(RestPermissions.INDEXRANGES_REBUILD, index);
    final SystemJob rebuildJob = singleIndexRangeJobFactory.create(indexSetRegistry.getAll(), index);
    try {
        this.systemJobManager.submit(rebuildJob);
    } catch (SystemJobConcurrencyException e) {
        final String msg = "Concurrency level of this job reached: " + e.getMessage();
        LOG.error(msg);
        throw new ForbiddenException(msg, e);
    }
    return Response.accepted().build();
}

Example 73 with Messages

use of org.graylog2.indexer.messages.Messages in project graylog2-server by Graylog2.

the class LocalKafkaMessageQueueReader method run.

@Override
protected void run() throws Exception {
    try {
        requestedReadCount = metricRegistry.register(name(this.getClass(), "requestedReadCount"), new HdrHistogram(processBuffer.getRingBufferSize() + 1, 3));
    } catch (IllegalArgumentException e) {
        log.warn("Metric already exists", e);
        throw e;
    }
    while (isRunning()) {
        // TODO interfere with reading if we are not 100% certain we should be reading, see #listenForLifecycleChanges
        if (!shouldBeReading()) {
            Uninterruptibles.sleepUninterruptibly(100, MILLISECONDS);
            // don't read immediately, but check if we should be shutting down.
            continue;
        }
        // approximate count to read from the journal to backfill the processing chain
        final long remainingCapacity = processBuffer.getRemainingCapacity();
        requestedReadCount.update(remainingCapacity);
        final List<Journal.JournalReadEntry> encodedRawMessages = journal.read(remainingCapacity);
        if (encodedRawMessages.isEmpty()) {
            log.debug("No messages to read from Journal, waiting until the writer adds more messages.");
            // block until something is written to the journal again
            try {
                readBlocked.inc();
                journalFilled.acquire();
            } catch (InterruptedException ignored) {
                // this can happen when we are blocked but the system wants to shut down. We don't have to do anything in that case.
                continue;
            }
            log.debug("Messages have been written to Journal, continuing to read.");
            // we don't care how many messages were inserted in the meantime, we'll read all of them eventually
            journalFilled.drainPermits();
        } else {
            readMessages.mark(encodedRawMessages.size());
            readerMetrics.readMessages().mark(encodedRawMessages.size());
            log.debug("Processing {} messages from journal.", encodedRawMessages.size());
            for (final Journal.JournalReadEntry encodedRawMessage : encodedRawMessages) {
                final RawMessage rawMessage = RawMessage.decode(encodedRawMessage.getPayload(), encodedRawMessage.getOffset());
                readerMetrics.readBytes().mark(encodedRawMessage.getPayload().length);
                if (rawMessage == null) {
                    // never insert null objects into the ringbuffer, as that is useless
                    log.error("Found null raw message!");
                    journal.markJournalOffsetCommitted(encodedRawMessage.getOffset());
                    continue;
                }
                processBuffer.insertBlocking(rawMessage);
            }
        }
    }
    log.info("Stopping.");
}

Example 74 with Messages

use of org.graylog2.indexer.messages.Messages in project graylog2-server by Graylog2.

the class GracefulShutdown method doRun.

private void doRun(boolean exit) {
    LOG.info("Graceful shutdown initiated.");
    // Trigger a lifecycle change. Some services are listening for those and will halt operation accordingly.
    serverStatus.shutdown();
    // Give possible load balancers time to recognize state change. State is DEAD because of HALTING.
    LOG.info("Node status: [{}]. Waiting <{}sec> for possible load balancers to recognize state change.", serverStatus.getLifecycle(), configuration.getLoadBalancerRecognitionPeriodSeconds());
    Uninterruptibles.sleepUninterruptibly(configuration.getLoadBalancerRecognitionPeriodSeconds(), TimeUnit.SECONDS);
    activityWriter.write(new Activity("Graceful shutdown initiated.", GracefulShutdown.class));
    /*
         * Wait a second to give for example the calling REST call some time to respond
         * to the client. Using a latch or something here might be a bit over-engineered.
         */
    Uninterruptibles.sleepUninterruptibly(SLEEP_SECS, TimeUnit.SECONDS);
    // Stop REST API service to avoid changes from outside.
    jerseyService.stopAsync();
    // stop all inputs so no new messages can come in
    inputSetupService.stopAsync();
    jerseyService.awaitTerminated();
    inputSetupService.awaitTerminated();
    // Try to flush all remaining messages from the system
    bufferSynchronizerService.stopAsync().awaitTerminated();
    // Stop all services that registered with the shutdown service (e.g. plugins)
    // This must run after the BufferSynchronizerService shutdown to make sure the buffers are empty.
    gracefulShutdownService.stopAsync();
    // stop all maintenance tasks
    periodicalsService.stopAsync().awaitTerminated();
    // Wait until the shutdown service is done
    gracefulShutdownService.awaitTerminated();
    auditEventSender.success(AuditActor.system(serverStatus.getNodeId()), NODE_SHUTDOWN_COMPLETE);
    // Shut down hard with no shutdown hooks running.
    LOG.info("Goodbye.");
    if (exit) {
        System.exit(0);
    }
}

Example 75 with Messages

use of org.graylog2.indexer.messages.Messages in project graylog2-server by Graylog2.

the class AggregationEventProcessorTest method testEventsFromAggregationResultWithConditions.

@Test
public void testEventsFromAggregationResultWithConditions() {
    final DateTime now = DateTime.now(DateTimeZone.UTC);
    final AbsoluteRange timerange = AbsoluteRange.create(now.minusHours(1), now.plusHours(1));
    // We expect to get the end of the aggregation timerange as event time
    final TestEvent event1 = new TestEvent(timerange.to());
    final TestEvent event2 = new TestEvent(timerange.to());
    when(eventFactory.createEvent(any(EventDefinition.class), eq(now), anyString())).thenReturn(// first invocation return value
    event1).thenReturn(// second invocation return value
    event2);
    // There should only be one result because the second result's "abc123" value is less than 40. (it is 23)
    // See result builder below
    final AggregationConditions conditions = AggregationConditions.builder().expression(Expr.And.create(Expr.Greater.create(Expr.NumberReference.create("abc123"), Expr.NumberValue.create(40.0d)), Expr.Lesser.create(Expr.NumberReference.create("xyz789"), Expr.NumberValue.create(2.0d)))).build();
    final EventDefinitionDto eventDefinitionDto = buildEventDefinitionDto(ImmutableSet.of(), ImmutableList.of(), conditions);
    final AggregationEventProcessorParameters parameters = AggregationEventProcessorParameters.builder().timerange(timerange).build();
    final AggregationEventProcessor eventProcessor = new AggregationEventProcessor(eventDefinitionDto, searchFactory, eventProcessorDependencyCheck, stateService, moreSearch, streamService, messages);
    final AggregationResult result = AggregationResult.builder().effectiveTimerange(timerange).totalAggregatedMessages(1).sourceStreams(ImmutableSet.of("stream-1", "stream-2", "stream-3")).keyResults(ImmutableList.of(AggregationKeyResult.builder().key(ImmutableList.of("one", "two")).timestamp(now).seriesValues(ImmutableList.of(AggregationSeriesValue.builder().key(ImmutableList.of("a")).value(42.0d).series(AggregationSeries.builder().id("abc123").function(AggregationFunction.COUNT).field("source").build()).build(), AggregationSeriesValue.builder().key(ImmutableList.of("a")).value(1.0d).series(AggregationSeries.builder().id("xyz789").function(AggregationFunction.CARD).field("source").build()).build())).build(), AggregationKeyResult.builder().key(ImmutableList.of(now.toString(), "one", "two")).seriesValues(ImmutableList.of(AggregationSeriesValue.builder().key(ImmutableList.of("a")).value(// Doesn't match condition
    23.0d).series(AggregationSeries.builder().id("abc123").function(AggregationFunction.COUNT).field("source").build()).build(), AggregationSeriesValue.builder().key(ImmutableList.of("a")).value(1.0d).series(AggregationSeries.builder().id("xyz789").function(AggregationFunction.CARD).field("source").build()).build())).build())).build();
    final ImmutableList<EventWithContext> eventsWithContext = eventProcessor.eventsFromAggregationResult(eventFactory, parameters, result);
    assertThat(eventsWithContext).hasSize(1);
    assertThat(eventsWithContext.get(0)).satisfies(eventWithContext -> {
        final Event event = eventWithContext.event();
        assertThat(event.getId()).isEqualTo(event1.getId());
        assertThat(event.getMessage()).isEqualTo(event1.getMessage());
        assertThat(event.getEventTimestamp()).isEqualTo(timerange.to());
        assertThat(event.getTimerangeStart()).isEqualTo(timerange.from());
        assertThat(event.getTimerangeEnd()).isEqualTo(timerange.to());
        // Should contain all streams because when config.streams is empty, we search in all streams
        assertThat(event.getSourceStreams()).containsOnly("stream-1", "stream-2", "stream-3");
        final Message message = eventWithContext.messageContext().orElse(null);
        assertThat(message).isNotNull();
        assertThat(message.getField("group_field_one")).isEqualTo("one");
        assertThat(message.getField("group_field_two")).isEqualTo("two");
        assertThat(message.getField("aggregation_key")).isEqualTo("one|two");
        assertThat(message.getField("aggregation_value_count_source")).isEqualTo(42.0d);
        assertThat(message.getField("aggregation_value_card_source")).isEqualTo(1.0d);
        assertThat(event.getGroupByFields().get("group_field_one")).isEqualTo("one");
        assertThat(event.getGroupByFields().get("group_field_two")).isEqualTo("two");
    });
}