Examples with SyslogEvent org.apache.nifi.processors.standard.syslog.SyslogEvent used on opensource projects

Example 1 with SyslogEvent

use of org.apache.nifi.processors.standard.syslog.SyslogEvent in project nifi by apache.

the class ListenSyslog method onTrigger.

@Override
public void onTrigger(final ProcessContext context, final ProcessSession session) throws ProcessException {
    // poll the queue with a small timeout to avoid unnecessarily yielding below
    RawSyslogEvent rawSyslogEvent = getMessage(true, true, session);
    // throttling even when no data is available
    if (rawSyslogEvent == null) {
        return;
    }
    final int maxBatchSize = context.getProperty(MAX_BATCH_SIZE).asInteger();
    final String port = context.getProperty(PORT).evaluateAttributeExpressions().getValue();
    final String protocol = context.getProperty(PROTOCOL).getValue();
    final Map<String, String> defaultAttributes = new HashMap<>(4);
    defaultAttributes.put(SyslogAttributes.PROTOCOL.key(), protocol);
    defaultAttributes.put(SyslogAttributes.PORT.key(), port);
    defaultAttributes.put(CoreAttributes.MIME_TYPE.key(), "text/plain");
    final int numAttributes = SyslogAttributes.values().length + 2;
    final boolean shouldParse = context.getProperty(PARSE_MESSAGES).asBoolean();
    final Map<String, FlowFile> flowFilePerSender = new HashMap<>();
    final SyslogParser parser = getParser();
    for (int i = 0; i < maxBatchSize; i++) {
        SyslogEvent event = null;
        // If this is our first iteration, we have already polled our queues. Otherwise, poll on each iteration.
        if (i > 0) {
            rawSyslogEvent = getMessage(true, false, session);
            if (rawSyslogEvent == null) {
                break;
            }
        }
        final String sender = rawSyslogEvent.getSender();
        FlowFile flowFile = flowFilePerSender.computeIfAbsent(sender, k -> session.create());
        if (shouldParse) {
            boolean valid = true;
            try {
                event = parser.parseEvent(rawSyslogEvent.getData(), sender);
            } catch (final ProcessException pe) {
                getLogger().warn("Failed to parse Syslog event; routing to invalid");
                valid = false;
            }
            // because the 'flowFile' object may already have data written to it.
            if (!valid || event == null || !event.isValid()) {
                FlowFile invalidFlowFile = session.create();
                invalidFlowFile = session.putAllAttributes(invalidFlowFile, defaultAttributes);
                if (sender != null) {
                    invalidFlowFile = session.putAttribute(invalidFlowFile, SyslogAttributes.SENDER.key(), sender);
                }
                try {
                    final byte[] rawBytes = rawSyslogEvent.getData();
                    invalidFlowFile = session.write(invalidFlowFile, new OutputStreamCallback() {

                        @Override
                        public void process(final OutputStream out) throws IOException {
                            out.write(rawBytes);
                        }
                    });
                } catch (final Exception e) {
                    getLogger().error("Failed to write contents of Syslog message to FlowFile due to {}; will re-queue message and try again", e);
                    errorEvents.offer(rawSyslogEvent);
                    session.remove(invalidFlowFile);
                    break;
                }
                session.transfer(invalidFlowFile, REL_INVALID);
                break;
            }
            getLogger().trace(event.getFullMessage());
            final Map<String, String> attributes = new HashMap<>(numAttributes);
            attributes.put(SyslogAttributes.PRIORITY.key(), event.getPriority());
            attributes.put(SyslogAttributes.SEVERITY.key(), event.getSeverity());
            attributes.put(SyslogAttributes.FACILITY.key(), event.getFacility());
            attributes.put(SyslogAttributes.VERSION.key(), event.getVersion());
            attributes.put(SyslogAttributes.TIMESTAMP.key(), event.getTimeStamp());
            attributes.put(SyslogAttributes.HOSTNAME.key(), event.getHostName());
            attributes.put(SyslogAttributes.BODY.key(), event.getMsgBody());
            attributes.put(SyslogAttributes.VALID.key(), String.valueOf(event.isValid()));
            flowFile = session.putAllAttributes(flowFile, attributes);
        }
        // figure out if we should write the bytes from the raw event or parsed event
        final boolean writeDemarcator = (i > 0);
        try {
            // write the raw bytes of the message as the FlowFile content
            final byte[] rawMessage = (event == null) ? rawSyslogEvent.getData() : event.getRawMessage();
            flowFile = session.append(flowFile, new OutputStreamCallback() {

                @Override
                public void process(final OutputStream out) throws IOException {
                    if (writeDemarcator) {
                        out.write(messageDemarcatorBytes);
                    }
                    out.write(rawMessage);
                }
            });
        } catch (final Exception e) {
            getLogger().error("Failed to write contents of Syslog message to FlowFile due to {}; will re-queue message and try again", e);
            errorEvents.offer(rawSyslogEvent);
            break;
        }
        flowFilePerSender.put(sender, flowFile);
    }
    for (final Map.Entry<String, FlowFile> entry : flowFilePerSender.entrySet()) {
        final String sender = entry.getKey();
        FlowFile flowFile = entry.getValue();
        if (flowFile.getSize() == 0L) {
            session.remove(flowFile);
            getLogger().debug("No data written to FlowFile from Sender {}; removing FlowFile", new Object[] { sender });
            continue;
        }
        final Map<String, String> newAttributes = new HashMap<>(defaultAttributes.size() + 1);
        newAttributes.putAll(defaultAttributes);
        newAttributes.put(SyslogAttributes.SENDER.key(), sender);
        flowFile = session.putAllAttributes(flowFile, newAttributes);
        getLogger().debug("Transferring {} to success", new Object[] { flowFile });
        session.transfer(flowFile, REL_SUCCESS);
        session.adjustCounter("FlowFiles Transferred to Success", 1L, false);
        final String senderHost = sender.startsWith("/") && sender.length() > 1 ? sender.substring(1) : sender;
        final String transitUri = new StringBuilder().append(protocol.toLowerCase()).append("://").append(senderHost).append(":").append(port).toString();
        session.getProvenanceReporter().receive(flowFile, transitUri);
    }
}

Example 2 with SyslogEvent

use of org.apache.nifi.processors.standard.syslog.SyslogEvent in project nifi by apache.

the class TestSyslogParser method testParseWithSender.

@Test
public void testParseWithSender() {
    final String sender = "127.0.0.1";
    final String message = "<31>Oct 13 15:43:23 localhost.home some message\n";
    final byte[] bytes = message.getBytes(CHARSET);
    final ByteBuffer buffer = ByteBuffer.allocate(bytes.length);
    buffer.clear();
    buffer.put(bytes);
    final SyslogEvent event = parser.parseEvent(buffer, sender);
    Assert.assertNotNull(event);
    Assert.assertTrue(event.isValid());
    Assert.assertEquals(sender, event.getSender());
}

Example 3 with SyslogEvent

use of org.apache.nifi.processors.standard.syslog.SyslogEvent in project nifi by apache.

the class TestSyslogParser method testRFC3164SingleDigitDay.

@Test
public void testRFC3164SingleDigitDay() {
    final String pri = "10";
    final String stamp = "Oct  1 13:14:04";
    final String host = "my.host.com";
    final String body = "some body message";
    final String message = "<" + pri + ">" + stamp + " " + host + " " + body;
    final byte[] bytes = message.getBytes(CHARSET);
    final ByteBuffer buffer = ByteBuffer.allocate(bytes.length);
    buffer.clear();
    buffer.put(bytes);
    final SyslogEvent event = parser.parseEvent(buffer);
    Assert.assertNotNull(event);
    Assert.assertEquals(pri, event.getPriority());
    Assert.assertEquals("2", event.getSeverity());
    Assert.assertEquals("1", event.getFacility());
    Assert.assertNull(event.getVersion());
    Assert.assertEquals(stamp, event.getTimeStamp());
    Assert.assertEquals(host, event.getHostName());
    Assert.assertEquals(body, event.getMsgBody());
    Assert.assertEquals(message, event.getFullMessage());
    Assert.assertTrue(event.isValid());
}

Example 4 with SyslogEvent

use of org.apache.nifi.processors.standard.syslog.SyslogEvent in project nifi by apache.

the class TestSyslogParser method testRFC5424WithVersion.

@Test
public void testRFC5424WithVersion() {
    final String pri = "34";
    final String version = "1";
    final String stamp = "2003-10-11T22:14:15.003Z";
    final String host = "mymachine.example.com";
    final String body = "su - ID47 - BOM'su root' failed for lonvick on /dev/pts/8";
    final String message = "<" + pri + ">" + version + " " + stamp + " " + host + " " + body;
    final byte[] bytes = message.getBytes(CHARSET);
    final ByteBuffer buffer = ByteBuffer.allocate(bytes.length);
    buffer.clear();
    buffer.put(bytes);
    final SyslogEvent event = parser.parseEvent(buffer);
    Assert.assertNotNull(event);
    Assert.assertEquals(pri, event.getPriority());
    Assert.assertEquals("2", event.getSeverity());
    Assert.assertEquals("4", event.getFacility());
    Assert.assertEquals(version, event.getVersion());
    Assert.assertEquals(stamp, event.getTimeStamp());
    Assert.assertEquals(host, event.getHostName());
    Assert.assertEquals(body, event.getMsgBody());
    Assert.assertEquals(message, event.getFullMessage());
    Assert.assertTrue(event.isValid());
}

Example 5 with SyslogEvent

use of org.apache.nifi.processors.standard.syslog.SyslogEvent in project nifi by apache.

the class TestSyslogParser method testInvalidPriority.

@Test
public void testInvalidPriority() {
    final String message = "10 Oct 13 14:14:43 localhost some body of the message";
    final byte[] bytes = message.getBytes(CHARSET);
    final ByteBuffer buffer = ByteBuffer.allocate(bytes.length);
    buffer.clear();
    buffer.put(bytes);
    final SyslogEvent event = parser.parseEvent(buffer);
    Assert.assertNotNull(event);
    Assert.assertFalse(event.isValid());
    Assert.assertEquals(message, event.getFullMessage());
}