use of org.graylog2.inputs.codecs.gelf.GELFMessageChunk in project graylog2-server by Graylog2.
the class GelfChunkAggregator method checkForCompletion.
/**
* Checks whether the presented gelf message chunk completes the incoming raw message and returns it if it does.
* If the message isn't complete, it adds the chunk to the internal buffer and waits for more incoming messages.
* Outdated chunks are being purged regularly.
*
* @param gelfMessage the gelf message chunk
* @return null or a {@link org.graylog2.plugin.journal.RawMessage raw message} object
*/
@Nullable
private ChannelBuffer checkForCompletion(GELFMessage gelfMessage) {
if (!chunks.isEmpty() && log.isDebugEnabled()) {
log.debug("Dumping GELF chunk map [chunks for {} messages]:\n{}", chunks.size(), humanReadableChunkMap());
}
// TODO second parameter
final GELFMessageChunk chunk = new GELFMessageChunk(gelfMessage, null);
final int sequenceCount = chunk.getSequenceCount();
final String messageId = chunk.getId();
ChunkEntry entry = new ChunkEntry(sequenceCount, chunk.getArrival(), messageId);
final ChunkEntry existing = chunks.putIfAbsent(messageId, entry);
if (existing == null) {
// add this chunk entry to the eviction set
waitingMessages.inc();
sortedEvictionSet.add(entry);
} else {
// the entry is already in the eviction set and chunk map
entry = existing;
}
final int sequenceNumber = chunk.getSequenceNumber();
if (!entry.payloadArray.compareAndSet(sequenceNumber, null, chunk)) {
log.error("Received duplicate chunk {} for message {} from {}", sequenceNumber, messageId, gelfMessage.getSourceAddress());
duplicateChunks.inc();
return null;
}
final int chunkWatermark = entry.chunkSlotsWritten.incrementAndGet();
if (chunkWatermark > MAX_CHUNKS) {
getAndCleanupEntry(messageId);
throw new IllegalStateException("Maximum number of chunks reached, discarding message");
}
if (chunkWatermark == sequenceCount) {
// message is complete by chunk count, assemble and return it.
// it might still be corrupt etc, but we've seen enough chunks
// remove before operating on it, to avoid racing too much with the clean up job, some race is inevitable, though.
entry = getAndCleanupEntry(messageId);
final byte[][] allChunks = new byte[sequenceCount][];
for (int i = 0; i < entry.payloadArray.length(); i++) {
final GELFMessageChunk messageChunk = entry.payloadArray.get(i);
if (messageChunk == null) {
log.debug("Couldn't read chunk {} of message {}, skipping this chunk.", i, messageId);
} else {
allChunks[i] = messageChunk.getData();
}
}
completeMessages.inc();
return ChannelBuffers.wrappedBuffer(allChunks);
}
// message isn't complete yet, check if we should remove the other parts as well
if (isOutdated(entry)) {
// chunks are outdated, the oldest came in over 5 seconds ago, clean them all up
log.debug("Not all chunks of <{}> arrived within {}ms. Dropping chunks.", messageId, VALIDITY_PERIOD);
expireEntry(messageId);
}
return null;
}
use of org.graylog2.inputs.codecs.gelf.GELFMessageChunk in project graylog2-server by Graylog2.
the class GelfChunkAggregator method humanReadableChunkMap.
private String humanReadableChunkMap() {
final StringBuilder sb = new StringBuilder();
for (final Map.Entry<String, ChunkEntry> entry : chunks.entrySet()) {
sb.append("Message <").append(entry.getKey()).append("> ");
sb.append("\tChunks:\n");
for (int i = 0; i < entry.getValue().payloadArray.length(); i++) {
final GELFMessageChunk chunk = entry.getValue().payloadArray.get(i);
sb.append("\t\t").append(chunk == null ? "<not arrived yet>" : chunk).append("\n");
}
}
return sb.toString();
}
Aggregations