Examples with ChunkEntry org.graylog2.inputs.codecs.GelfChunkAggregator.ChunkEntry used on opensource projects

Example 1 with ChunkEntry

use of org.graylog2.inputs.codecs.GelfChunkAggregator.ChunkEntry in project graylog2-server by Graylog2.

the class GelfChunkAggregator method checkForCompletion.

/**
     * Checks whether the presented gelf message chunk completes the incoming raw message and returns it if it does.
     * If the message isn't complete, it adds the chunk to the internal buffer and waits for more incoming messages.
     * Outdated chunks are being purged regularly.
     *
     * @param gelfMessage the gelf message chunk
     * @return null or a {@link org.graylog2.plugin.journal.RawMessage raw message} object
     */
@Nullable
private ChannelBuffer checkForCompletion(GELFMessage gelfMessage) {
    if (!chunks.isEmpty() && log.isDebugEnabled()) {
        log.debug("Dumping GELF chunk map [chunks for {} messages]:\n{}", chunks.size(), humanReadableChunkMap());
    }
    // TODO second parameter
    final GELFMessageChunk chunk = new GELFMessageChunk(gelfMessage, null);
    final int sequenceCount = chunk.getSequenceCount();
    final String messageId = chunk.getId();
    ChunkEntry entry = new ChunkEntry(sequenceCount, chunk.getArrival(), messageId);
    final ChunkEntry existing = chunks.putIfAbsent(messageId, entry);
    if (existing == null) {
        // add this chunk entry to the eviction set
        waitingMessages.inc();
        sortedEvictionSet.add(entry);
    } else {
        // the entry is already in the eviction set and chunk map
        entry = existing;
    }
    final int sequenceNumber = chunk.getSequenceNumber();
    if (!entry.payloadArray.compareAndSet(sequenceNumber, null, chunk)) {
        log.error("Received duplicate chunk {} for message {} from {}", sequenceNumber, messageId, gelfMessage.getSourceAddress());
        duplicateChunks.inc();
        return null;
    }
    final int chunkWatermark = entry.chunkSlotsWritten.incrementAndGet();
    if (chunkWatermark > MAX_CHUNKS) {
        getAndCleanupEntry(messageId);
        throw new IllegalStateException("Maximum number of chunks reached, discarding message");
    }
    if (chunkWatermark == sequenceCount) {
        // message is complete by chunk count, assemble and return it.
        // it might still be corrupt etc, but we've seen enough chunks
        // remove before operating on it, to avoid racing too much with the clean up job, some race is inevitable, though.
        entry = getAndCleanupEntry(messageId);
        final byte[][] allChunks = new byte[sequenceCount][];
        for (int i = 0; i < entry.payloadArray.length(); i++) {
            final GELFMessageChunk messageChunk = entry.payloadArray.get(i);
            if (messageChunk == null) {
                log.debug("Couldn't read chunk {} of message {}, skipping this chunk.", i, messageId);
            } else {
                allChunks[i] = messageChunk.getData();
            }
        }
        completeMessages.inc();
        return ChannelBuffers.wrappedBuffer(allChunks);
    }
    // message isn't complete yet, check if we should remove the other parts as well
    if (isOutdated(entry)) {
        // chunks are outdated, the oldest came in over 5 seconds ago, clean them all up
        log.debug("Not all chunks of <{}> arrived within {}ms. Dropping chunks.", messageId, VALIDITY_PERIOD);
        expireEntry(messageId);
    }
    return null;
}

Example 2 with ChunkEntry

use of org.graylog2.inputs.codecs.GelfChunkAggregator.ChunkEntry in project graylog2-server by Graylog2.

the class GelfChunkAggregatorTest method testChunkEntryEquals.

@Test
public void testChunkEntryEquals() throws Exception {
    final GelfChunkAggregator.ChunkEntry entry = new ChunkEntry(1, 0L, "id");
    assertThat(entry).isEqualTo(new ChunkEntry(1, 0L, "id"));
    assertThat(entry).isEqualTo(new ChunkEntry(2, 0L, "id"));
    assertThat(entry).isNotEqualTo(new ChunkEntry(1, 1L, "id"));
    assertThat(entry).isNotEqualTo(new ChunkEntry(1, 0L, "foo"));
}

Example 3 with ChunkEntry

use of org.graylog2.inputs.codecs.GelfChunkAggregator.ChunkEntry in project graylog2-server by Graylog2.

the class GelfChunkAggregatorTest method testChunkEntryCompareTo.

@Test
public void testChunkEntryCompareTo() throws Exception {
    // Test if the ChunkEntry#compareTo() method can handle ChunkEntry objects which have the same timestamp.
    // See: https://github.com/Graylog2/graylog2-server/issues/1462
    final ConcurrentSkipListSet<GelfChunkAggregator.ChunkEntry> sortedEvictionSet = new ConcurrentSkipListSet<>();
    final long currentTime = System.currentTimeMillis();
    for (int i = 0; i < 10; i++) {
        sortedEvictionSet.add(new GelfChunkAggregator.ChunkEntry(1, currentTime, "a" + i));
    }
    final int size = sortedEvictionSet.size();
    for (int i = 0; i < size; i++) {
        sortedEvictionSet.remove(sortedEvictionSet.first());
    }
    assertTrue("eviction set should be empty", sortedEvictionSet.isEmpty());
}

Example 4 with ChunkEntry

use of org.graylog2.inputs.codecs.GelfChunkAggregator.ChunkEntry in project graylog2-server by Graylog2.

the class GelfChunkAggregatorTest method testChunkEntryHashCode.

@Test
public void testChunkEntryHashCode() throws Exception {
    final GelfChunkAggregator.ChunkEntry entry = new ChunkEntry(1, 0L, "id");
    assertThat(entry.hashCode()).isEqualTo(new ChunkEntry(1, 0L, "id").hashCode());
    assertThat(entry.hashCode()).isEqualTo(new ChunkEntry(2, 0L, "id").hashCode());
    assertThat(entry.hashCode()).isNotEqualTo(new ChunkEntry(1, 1L, "id").hashCode());
    assertThat(entry.hashCode()).isNotEqualTo(new ChunkEntry(1, 0L, "foo").hashCode());
}

Example 5 with ChunkEntry

use of org.graylog2.inputs.codecs.GelfChunkAggregator.ChunkEntry in project graylog2-server by Graylog2.

the class GelfChunkAggregator method humanReadableChunkMap.

private String humanReadableChunkMap() {
    final StringBuilder sb = new StringBuilder();
    for (final Map.Entry<String, ChunkEntry> entry : chunks.entrySet()) {
        sb.append("Message <").append(entry.getKey()).append("> ");
        sb.append("\tChunks:\n");
        for (int i = 0; i < entry.getValue().payloadArray.length(); i++) {
            final GELFMessageChunk chunk = entry.getValue().payloadArray.get(i);
            sb.append("\t\t").append(chunk == null ? "<not arrived yet>" : chunk).append("\n");
        }
    }
    return sb.toString();
}