Examples with Record com.amazon.dataprepper.model.record.Record used on opensource projects

Example 11 with Record

use of com.amazon.dataprepper.model.record.Record in project data-prepper by opensearch-project.

the class HTTPSourceTest method testHTTPJsonResponse200.

@Test
public void testHTTPJsonResponse200() {
    // Prepare
    final String testData = "[{\"log\": \"somelog\"}]";
    final int testPayloadSize = testData.getBytes().length;
    HTTPSourceUnderTest.start(testBuffer);
    refreshMeasurements();
    // When
    WebClient.of().execute(RequestHeaders.builder().scheme(SessionProtocol.HTTP).authority("127.0.0.1:2021").method(HttpMethod.POST).path("/log/ingest").contentType(MediaType.JSON_UTF_8).build(), HttpData.ofUtf8(testData)).aggregate().whenComplete((i, ex) -> assertSecureResponseWithStatusCode(i, HttpStatus.OK)).join();
    // Then
    Assertions.assertFalse(testBuffer.isEmpty());
    final Map.Entry<Collection<Record<Log>>, CheckpointState> result = testBuffer.read(100);
    List<Record<Log>> records = new ArrayList<>(result.getKey());
    Assertions.assertEquals(1, records.size());
    final Record<Log> record = records.get(0);
    Assertions.assertEquals("somelog", record.getData().get("log", String.class));
    // Verify metrics
    final Measurement requestReceivedCount = MetricsTestUtil.getMeasurementFromList(requestsReceivedMeasurements, Statistic.COUNT);
    Assertions.assertEquals(1.0, requestReceivedCount.getValue());
    final Measurement successRequestsCount = MetricsTestUtil.getMeasurementFromList(successRequestsMeasurements, Statistic.COUNT);
    Assertions.assertEquals(1.0, successRequestsCount.getValue());
    final Measurement requestProcessDurationCount = MetricsTestUtil.getMeasurementFromList(requestProcessDurationMeasurements, Statistic.COUNT);
    Assertions.assertEquals(1.0, requestProcessDurationCount.getValue());
    final Measurement requestProcessDurationMax = MetricsTestUtil.getMeasurementFromList(requestProcessDurationMeasurements, Statistic.MAX);
    Assertions.assertTrue(requestProcessDurationMax.getValue() > 0);
    final Measurement payloadSizeMax = MetricsTestUtil.getMeasurementFromList(payloadSizeSummaryMeasurements, Statistic.MAX);
    Assertions.assertEquals(testPayloadSize, payloadSizeMax.getValue());
}

Example 12 with Record

use of com.amazon.dataprepper.model.record.Record in project data-prepper by opensearch-project.

the class GrokPrepper method doExecute.

/**
 * execute the prepper logic which could potentially modify the incoming record. The level to which the record has
 * been modified depends on the implementation
 *
 * @param records Input records that will be modified/processed
 * @return Record  modified output records
 */
@Override
public Collection<Record<Event>> doExecute(final Collection<Record<Event>> records) {
    final List<Record<Event>> recordsOut = new LinkedList<>();
    for (final Record<Event> record : records) {
        try {
            final Event event = record.getData();
            if (grokPrepperConfig.getTimeoutMillis() == 0) {
                grokProcessingTime.record(() -> matchAndMerge(event));
            } else {
                runWithTimeout(() -> grokProcessingTime.record(() -> matchAndMerge(event)));
            }
            final Record<Event> grokkedRecord = new Record<>(event, record.getMetadata());
            recordsOut.add(grokkedRecord);
        } catch (TimeoutException e) {
            LOG.error("Matching on record [{}] took longer than [{}] and timed out", record.getData(), grokPrepperConfig.getTimeoutMillis());
            recordsOut.add(record);
            grokProcessingTimeoutsCounter.increment();
        } catch (ExecutionException e) {
            LOG.error("An exception occurred while matching on record [{}]", record.getData(), e);
            recordsOut.add(record);
            grokProcessingErrorsCounter.increment();
        } catch (InterruptedException e) {
            LOG.error("Matching on record [{}] was interrupted", record.getData(), e);
            recordsOut.add(record);
            grokProcessingErrorsCounter.increment();
        } catch (RuntimeException e) {
            LOG.error("Unknown exception occurred when matching record [{}]", record.getData(), e);
            recordsOut.add(record);
            grokProcessingErrorsCounter.increment();
        }
    }
    return recordsOut;
}

Example 13 with Record

use of com.amazon.dataprepper.model.record.Record in project data-prepper by opensearch-project.

the class DateProcessorTests method from_time_received_with_custom_destination_test.

@Test
void from_time_received_with_custom_destination_test() {
    String destination = "new_field";
    when(mockDateProcessorConfig.getFromTimeReceived()).thenReturn(true);
    when(mockDateProcessorConfig.getDestination()).thenReturn(destination);
    when(mockDateProcessorConfig.getDestinationZoneId()).thenReturn(ZoneId.systemDefault());
    expectedInstant = Instant.now();
    dateProcessor = createObjectUnderTest();
    Map<String, Object> testData = getTestData();
    final Record<Event> record = new Record<>(JacksonEvent.builder().withData(testData).withEventType("event").withTimeReceived(expectedInstant).build());
    final List<Record<Event>> processedRecords = (List<Record<Event>>) dateProcessor.doExecute(Collections.singletonList(record));
    ZonedDateTime actualZonedDateTime = processedRecords.get(0).getData().get(destination, ZonedDateTime.class);
    Assertions.assertEquals(0, actualZonedDateTime.toInstant().compareTo(expectedInstant.truncatedTo(ChronoUnit.MILLIS)));
}

Example 14 with Record

use of com.amazon.dataprepper.model.record.Record in project data-prepper by opensearch-project.

the class DateProcessorTests method match_with_different_year_formats_test.

@ParameterizedTest
@ValueSource(strings = { "MMM/dd/uuuu", "yyyy MM dd" })
void match_with_different_year_formats_test(String pattern) {
    when(mockDateMatch.getKey()).thenReturn("logDate");
    when(mockDateMatch.getPatterns()).thenReturn(Collections.singletonList(pattern));
    List<DateProcessorConfig.DateMatch> dateMatches = Collections.singletonList(mockDateMatch);
    when(mockDateProcessorConfig.getMatch()).thenReturn(dateMatches);
    when(mockDateProcessorConfig.getSourceZoneId()).thenReturn(ZoneId.systemDefault());
    when(mockDateProcessorConfig.getDestinationZoneId()).thenReturn(ZoneId.systemDefault());
    when(mockDateProcessorConfig.getSourceLocale()).thenReturn(Locale.ROOT);
    dateProcessor = createObjectUnderTest();
    Map<String, Object> testData = getTestData();
    testData.put("logDate", expectedDateTime.minus(10, ChronoUnit.YEARS).format(DateTimeFormatter.ofPattern(pattern)));
    final Record<Event> record = buildRecordWithEvent(testData);
    final List<Record<Event>> processedRecords = (List<Record<Event>>) dateProcessor.doExecute(Collections.singletonList(record));
    ZonedDateTime actualZonedDateTime = record.getData().get(TIMESTAMP_KEY, ZonedDateTime.class);
    ZonedDateTime expectedZonedDatetime = expectedDateTime.minus(10, ChronoUnit.YEARS).atZone(mockDateProcessorConfig.getSourceZoneId()).truncatedTo(ChronoUnit.SECONDS);
    Assertions.assertTrue(actualZonedDateTime.toLocalDate().isEqual(expectedZonedDatetime.toLocalDate()));
    verify(dateProcessingMatchSuccessCounter, times(1)).increment();
}

Example 15 with Record

use of com.amazon.dataprepper.model.record.Record in project data-prepper by opensearch-project.

the class RandomStringSourceTests method testPutRecord.

@Test
public void testPutRecord() throws InterruptedException {
    final RandomStringSource randomStringSource = new RandomStringSource();
    final Queue<Record<Event>> bufferQueue = new ConcurrentLinkedQueue<>();
    final TestBuffer buffer = new TestBuffer(bufferQueue, 1);
    // Start source, and sleep for 1000 millis
    randomStringSource.start(buffer);
    Thread.sleep(1000);
    // Stop the source, and wait long enough that another message would be sent
    // if the source was running
    assertThat(buffer.size(), greaterThan(0));
    Thread.sleep(1000);
    randomStringSource.stop();
    assertThat(buffer.size(), greaterThan(0));
}