Examples with Event com.amazon.dataprepper.model.event.Event used on opensource projects

Example 21 with Event

use of com.amazon.dataprepper.model.event.Event in project data-prepper by opensearch-project.

the class GrokPrepper method doExecute.

/**
 * execute the prepper logic which could potentially modify the incoming record. The level to which the record has
 * been modified depends on the implementation
 *
 * @param records Input records that will be modified/processed
 * @return Record  modified output records
 */
@Override
public Collection<Record<Event>> doExecute(final Collection<Record<Event>> records) {
    final List<Record<Event>> recordsOut = new LinkedList<>();
    for (final Record<Event> record : records) {
        try {
            final Event event = record.getData();
            if (grokPrepperConfig.getTimeoutMillis() == 0) {
                grokProcessingTime.record(() -> matchAndMerge(event));
            } else {
                runWithTimeout(() -> grokProcessingTime.record(() -> matchAndMerge(event)));
            }
            final Record<Event> grokkedRecord = new Record<>(event, record.getMetadata());
            recordsOut.add(grokkedRecord);
        } catch (TimeoutException e) {
            LOG.error("Matching on record [{}] took longer than [{}] and timed out", record.getData(), grokPrepperConfig.getTimeoutMillis());
            recordsOut.add(record);
            grokProcessingTimeoutsCounter.increment();
        } catch (ExecutionException e) {
            LOG.error("An exception occurred while matching on record [{}]", record.getData(), e);
            recordsOut.add(record);
            grokProcessingErrorsCounter.increment();
        } catch (InterruptedException e) {
            LOG.error("Matching on record [{}] was interrupted", record.getData(), e);
            recordsOut.add(record);
            grokProcessingErrorsCounter.increment();
        } catch (RuntimeException e) {
            LOG.error("Unknown exception occurred when matching record [{}]", record.getData(), e);
            recordsOut.add(record);
            grokProcessingErrorsCounter.increment();
        }
    }
    return recordsOut;
}

Example 22 with Event

use of com.amazon.dataprepper.model.event.Event in project data-prepper by opensearch-project.

the class DateProcessorTests method from_time_received_with_custom_destination_test.

@Test
void from_time_received_with_custom_destination_test() {
    String destination = "new_field";
    when(mockDateProcessorConfig.getFromTimeReceived()).thenReturn(true);
    when(mockDateProcessorConfig.getDestination()).thenReturn(destination);
    when(mockDateProcessorConfig.getDestinationZoneId()).thenReturn(ZoneId.systemDefault());
    expectedInstant = Instant.now();
    dateProcessor = createObjectUnderTest();
    Map<String, Object> testData = getTestData();
    final Record<Event> record = new Record<>(JacksonEvent.builder().withData(testData).withEventType("event").withTimeReceived(expectedInstant).build());
    final List<Record<Event>> processedRecords = (List<Record<Event>>) dateProcessor.doExecute(Collections.singletonList(record));
    ZonedDateTime actualZonedDateTime = processedRecords.get(0).getData().get(destination, ZonedDateTime.class);
    Assertions.assertEquals(0, actualZonedDateTime.toInstant().compareTo(expectedInstant.truncatedTo(ChronoUnit.MILLIS)));
}

Example 23 with Event

use of com.amazon.dataprepper.model.event.Event in project data-prepper by opensearch-project.

the class DateProcessorTests method match_with_different_year_formats_test.

@ParameterizedTest
@ValueSource(strings = { "MMM/dd/uuuu", "yyyy MM dd" })
void match_with_different_year_formats_test(String pattern) {
    when(mockDateMatch.getKey()).thenReturn("logDate");
    when(mockDateMatch.getPatterns()).thenReturn(Collections.singletonList(pattern));
    List<DateProcessorConfig.DateMatch> dateMatches = Collections.singletonList(mockDateMatch);
    when(mockDateProcessorConfig.getMatch()).thenReturn(dateMatches);
    when(mockDateProcessorConfig.getSourceZoneId()).thenReturn(ZoneId.systemDefault());
    when(mockDateProcessorConfig.getDestinationZoneId()).thenReturn(ZoneId.systemDefault());
    when(mockDateProcessorConfig.getSourceLocale()).thenReturn(Locale.ROOT);
    dateProcessor = createObjectUnderTest();
    Map<String, Object> testData = getTestData();
    testData.put("logDate", expectedDateTime.minus(10, ChronoUnit.YEARS).format(DateTimeFormatter.ofPattern(pattern)));
    final Record<Event> record = buildRecordWithEvent(testData);
    final List<Record<Event>> processedRecords = (List<Record<Event>>) dateProcessor.doExecute(Collections.singletonList(record));
    ZonedDateTime actualZonedDateTime = record.getData().get(TIMESTAMP_KEY, ZonedDateTime.class);
    ZonedDateTime expectedZonedDatetime = expectedDateTime.minus(10, ChronoUnit.YEARS).atZone(mockDateProcessorConfig.getSourceZoneId()).truncatedTo(ChronoUnit.SECONDS);
    Assertions.assertTrue(actualZonedDateTime.toLocalDate().isEqual(expectedZonedDatetime.toLocalDate()));
    verify(dateProcessingMatchSuccessCounter, times(1)).increment();
}

Example 24 with Event

use of com.amazon.dataprepper.model.event.Event in project data-prepper by opensearch-project.

the class DropEventsWhenConditionTest method testIsStatementFalseReturnsInvertedEvaluatorResult.

@Test
void testIsStatementFalseReturnsInvertedEvaluatorResult() {
    doReturn(HandleFailedEventsOption.SKIP).when(dropEventProcessorConfig).getHandleFailedEventsOption();
    final String whenStatement = UUID.randomUUID().toString();
    final Event event = mock(Event.class);
    final boolean evaluatorResult = true;
    doReturn(whenStatement).when(dropEventProcessorConfig).getDropWhen();
    doReturn(evaluatorResult).when(evaluator).evaluate(eq(whenStatement), eq(event));
    final DropEventsWhenCondition whenCondition = new DropEventsWhenCondition.Builder().withDropEventsProcessorConfig(dropEventProcessorConfig).withExpressionEvaluator(evaluator).build();
    final boolean result = whenCondition.isStatementFalseWith(event);
    assertThat(result, not(is(evaluatorResult)));
    verify(evaluator).evaluate(eq(whenStatement), eq(event));
}

Example 25 with Event

use of com.amazon.dataprepper.model.event.Event in project data-prepper by opensearch-project.

the class CopyValueProcessor method doExecute.

@Override
public Collection<Record<Event>> doExecute(final Collection<Record<Event>> records) {
    for (final Record<Event> record : records) {
        final Event recordEvent = record.getData();
        for (CopyValueProcessorConfig.Entry entry : entries) {
            if (entry.getFromKey().equals(entry.getToKey()) || !recordEvent.containsKey(entry.getFromKey())) {
                continue;
            }
            if (!recordEvent.containsKey(entry.getToKey()) || entry.getOverwriteIfToKeyExists()) {
                final Object source = recordEvent.get(entry.getFromKey(), Object.class);
                recordEvent.put(entry.getToKey(), source);
            }
        }
    }
    return records;
}