Examples with Grok io.thekraken.grok.api.Grok used on opensource projects

Example 1 with Grok

use of io.thekraken.grok.api.Grok in project nifi by apache.

the class ExtractGrok method onScheduled.

@OnScheduled
public void onScheduled(final ProcessContext context) throws GrokException {
    for (int i = 0; i < context.getMaxConcurrentTasks(); i++) {
        final int maxBufferSize = context.getProperty(MAX_BUFFER_SIZE).asDataSize(DataUnit.B).intValue();
        final byte[] buffer = new byte[maxBufferSize];
        bufferQueue.add(buffer);
    }
    grok = new Grok();
    grok.addPatternFromFile(context.getProperty(GROK_PATTERN_FILE).getValue());
    grok.compile(context.getProperty(GROK_EXPRESSION).getValue(), context.getProperty(NAMED_CAPTURES_ONLY).asBoolean());
}

Example 2 with Grok

use of io.thekraken.grok.api.Grok in project nifi by apache.

the class TestGrokRecordReader method testParseSingleLineLogMessages.

@Test
public void testParseSingleLineLogMessages() throws GrokException, IOException, MalformedRecordException {
    try (final InputStream fis = new FileInputStream(new File("src/test/resources/grok/single-line-log-messages.txt"))) {
        final Grok grok = new Grok();
        grok.addPatternFromFile("src/main/resources/default-grok-patterns.txt");
        grok.compile("%{TIMESTAMP_ISO8601:timestamp} %{LOGLEVEL:level} %{GREEDYDATA:message}");
        final GrokRecordReader deserializer = new GrokRecordReader(fis, grok, GrokReader.createRecordSchema(grok), GrokReader.createRecordSchema(grok), true);
        final String[] logLevels = new String[] { "INFO", "WARN", "ERROR", "FATAL", "FINE" };
        final String[] messages = new String[] { "Test Message 1", "Red", "Green", "Blue", "Yellow" };
        final String[] rawMessages = new String[] { "2016-11-08 21:24:23,029 INFO Test Message 1", "2016-11-08 21:24:23,029 WARN Red", "2016-11-08 21:24:23,029 ERROR Green", "2016-11-08 21:24:23,029 FATAL Blue", "2016-11-08 21:24:23,029 FINE Yellow" };
        for (int i = 0; i < logLevels.length; i++) {
            final Object[] values = deserializer.nextRecord().getValues();
            assertNotNull(values);
            // values[] contains 4 elements: timestamp, level, message, STACK_TRACE, RAW_MESSAGE
            assertEquals(5, values.length);
            assertEquals("2016-11-08 21:24:23,029", values[0]);
            assertEquals(logLevels[i], values[1]);
            assertEquals(messages[i], values[2]);
            assertNull(values[3]);
            assertEquals(rawMessages[i], values[4]);
        }
        assertNull(deserializer.nextRecord());
        deserializer.close();
    }
}

Example 3 with Grok

use of io.thekraken.grok.api.Grok in project nifi by apache.

the class TestGrokRecordReader method testInheritNamedParameters.

@Test
public void testInheritNamedParameters() throws FileNotFoundException, IOException, GrokException, MalformedRecordException {
    final String syslogMsg = "May 22 15:58:23 my-host nifi[12345]:My Message";
    final byte[] msgBytes = syslogMsg.getBytes();
    try (final InputStream in = new ByteArrayInputStream(msgBytes)) {
        final Grok grok = new Grok();
        grok.addPatternFromFile("src/main/resources/default-grok-patterns.txt");
        grok.compile("%{SYSLOGBASE}%{GREEDYDATA:message}");
        final RecordSchema schema = GrokReader.createRecordSchema(grok);
        final List<String> fieldNames = schema.getFieldNames();
        assertEquals(9, fieldNames.size());
        assertTrue(fieldNames.contains("timestamp"));
        assertTrue(fieldNames.contains("logsource"));
        assertTrue(fieldNames.contains("facility"));
        assertTrue(fieldNames.contains("priority"));
        assertTrue(fieldNames.contains("program"));
        assertTrue(fieldNames.contains("pid"));
        assertTrue(fieldNames.contains("message"));
        // always implicitly there
        assertTrue(fieldNames.contains("stackTrace"));
        // always implicitly there
        assertTrue(fieldNames.contains("_raw"));
        final GrokRecordReader deserializer = new GrokRecordReader(in, grok, schema, schema, true);
        final Record record = deserializer.nextRecord();
        assertEquals("May 22 15:58:23", record.getValue("timestamp"));
        assertEquals("my-host", record.getValue("logsource"));
        assertNull(record.getValue("facility"));
        assertNull(record.getValue("priority"));
        assertEquals("nifi", record.getValue("program"));
        assertEquals("12345", record.getValue("pid"));
        assertEquals("My Message", record.getValue("message"));
        assertEquals("May 22 15:58:23 my-host nifi[12345]:My Message", record.getValue("_raw"));
        assertNull(deserializer.nextRecord());
        deserializer.close();
    }
}

Example 4 with Grok

use of io.thekraken.grok.api.Grok in project nifi by apache.

the class GrokReader method preCompile.

@OnEnabled
public void preCompile(final ConfigurationContext context) throws GrokException, IOException {
    grok = new Grok();
    try (final InputStream in = getClass().getResourceAsStream(DEFAULT_PATTERN_NAME);
        final Reader reader = new InputStreamReader(in)) {
        grok.addPatternFromReader(reader);
    }
    if (context.getProperty(PATTERN_FILE).isSet()) {
        grok.addPatternFromFile(context.getProperty(PATTERN_FILE).evaluateAttributeExpressions().getValue());
    }
    grok.compile(context.getProperty(GROK_EXPRESSION).getValue());
    appendUnmatchedLine = context.getProperty(NO_MATCH_BEHAVIOR).getValue().equalsIgnoreCase(APPEND_TO_PREVIOUS_MESSAGE.getValue());
    this.recordSchemaFromGrok = createRecordSchema(grok);
    final String schemaAccess = context.getProperty(getSchemaAcessStrategyDescriptor()).getValue();
    if (STRING_FIELDS_FROM_GROK_EXPRESSION.getValue().equals(schemaAccess)) {
        this.recordSchema = recordSchemaFromGrok;
    } else {
        this.recordSchema = null;
    }
}

Example 5 with Grok

use of io.thekraken.grok.api.Grok in project nifi by apache.

the class TestGrokRecordReader method testParseNiFiSampleLog.

@Test
public void testParseNiFiSampleLog() throws IOException, GrokException, MalformedRecordException {
    try (final InputStream fis = new FileInputStream(new File("src/test/resources/grok/nifi-log-sample.log"))) {
        final Grok grok = new Grok();
        grok.addPatternFromFile("src/main/resources/default-grok-patterns.txt");
        grok.compile("%{TIMESTAMP_ISO8601:timestamp} %{LOGLEVEL:level} \\[%{DATA:thread}\\] %{DATA:class} %{GREEDYDATA:message}");
        final GrokRecordReader deserializer = new GrokRecordReader(fis, grok, GrokReader.createRecordSchema(grok), GrokReader.createRecordSchema(grok), true);
        final String[] logLevels = new String[] { "INFO", "INFO", "INFO", "WARN", "WARN" };
        for (int i = 0; i < logLevels.length; i++) {
            final Object[] values = deserializer.nextRecord().getValues();
            assertNotNull(values);
            // values[] contains 6 elements: timestamp, level, thread, class, message, STACK_TRACE, RAW_MESSAGE
            assertEquals(7, values.length);
            assertEquals(logLevels[i], values[1]);
            assertNull(values[5]);
            assertNotNull(values[6]);
        }
        assertNull(deserializer.nextRecord());
        deserializer.close();
    }
}