Examples with Record com.amazon.dataprepper.model.record.Record used on opensource projects

Example 66 with Record

use of com.amazon.dataprepper.model.record.Record in project data-prepper by opensearch-project.

the class OpenSearchSinkIT method testOutputCustomIndex.

@ParameterizedTest
@ArgumentsSource(MultipleRecordTypeArgumentProvider.class)
public void testOutputCustomIndex(final Function<String, Record> stringToRecord) throws IOException, InterruptedException {
    final String testIndexAlias = "test-alias";
    final String testTemplateFile = Objects.requireNonNull(getClass().getClassLoader().getResource(TEST_TEMPLATE_V1_FILE)).getFile();
    final String testIdField = "someId";
    final String testId = "foo";
    final List<Record<Object>> testRecords = Collections.singletonList(stringToRecord.apply(generateCustomRecordJson(testIdField, testId)));
    final PluginSetting pluginSetting = generatePluginSetting(false, false, testIndexAlias, testTemplateFile);
    pluginSetting.getSettings().put(IndexConfiguration.DOCUMENT_ID_FIELD, testIdField);
    final OpenSearchSink sink = new OpenSearchSink(pluginSetting);
    sink.output(testRecords);
    final List<Map<String, Object>> retSources = getSearchResponseDocSources(testIndexAlias);
    MatcherAssert.assertThat(retSources.size(), equalTo(1));
    MatcherAssert.assertThat(getDocumentCount(testIndexAlias, "_id", testId), equalTo(Integer.valueOf(1)));
    sink.shutdown();
    // verify metrics
    final List<Measurement> bulkRequestLatencies = MetricsTestUtil.getMeasurementList(new StringJoiner(MetricNames.DELIMITER).add(PIPELINE_NAME).add(PLUGIN_NAME).add(OpenSearchSink.BULKREQUEST_LATENCY).toString());
    MatcherAssert.assertThat(bulkRequestLatencies.size(), equalTo(3));
    // COUNT
    Assert.assertEquals(1.0, bulkRequestLatencies.get(0).getValue(), 0);
}

Example 67 with Record

use of com.amazon.dataprepper.model.record.Record in project data-prepper by opensearch-project.

the class LogHTTPService method processRequest.

private HttpResponse processRequest(final AggregatedHttpRequest aggregatedHttpRequest) {
    requestsReceivedCounter.increment();
    List<String> jsonList;
    final HttpData content = aggregatedHttpRequest.content();
    payloadSizeSummary.record(content.length());
    try {
        jsonList = jsonCodec.parse(content);
    } catch (IOException e) {
        LOG.error("Failed to write the request content [{}] due to:", content.toStringUtf8(), e);
        return requestExceptionHandler.handleException(e, "Bad request data format. Needs to be json array.");
    }
    final List<Record<Log>> records = jsonList.stream().map(this::buildRecordLog).collect(Collectors.toList());
    try {
        buffer.writeAll(records, bufferWriteTimeoutInMillis);
    } catch (Exception e) {
        LOG.error("Failed to write the request content [{}] due to:", content.toStringUtf8(), e);
        return requestExceptionHandler.handleException(e);
    }
    successRequestsCounter.increment();
    return HttpResponse.of(HttpStatus.OK);
}

Example 68 with Record

use of com.amazon.dataprepper.model.record.Record in project data-prepper by opensearch-project.

the class DateProcessorTests method from_time_received_with_default_destination_test.

@Test
void from_time_received_with_default_destination_test() {
    when(mockDateProcessorConfig.getFromTimeReceived()).thenReturn(true);
    when(mockDateProcessorConfig.getDestinationZoneId()).thenReturn(ZoneId.systemDefault());
    expectedInstant = Instant.now();
    dateProcessor = createObjectUnderTest();
    Map<String, Object> testData = getTestData();
    final Record<Event> record = new Record<>(JacksonEvent.builder().withData(testData).withEventType("event").withTimeReceived(expectedInstant).build());
    final List<Record<Event>> processedRecords = (List<Record<Event>>) dateProcessor.doExecute(Collections.singletonList(record));
    ZonedDateTime actualZonedDateTime = processedRecords.get(0).getData().get(TIMESTAMP_KEY, ZonedDateTime.class);
    Assertions.assertEquals(0, actualZonedDateTime.toInstant().compareTo(expectedInstant.truncatedTo(ChronoUnit.MILLIS)));
}

Example 69 with Record

use of com.amazon.dataprepper.model.record.Record in project data-prepper by opensearch-project.

the class DateProcessorTests method match_with_missing_hours_minutes_seconds_adds_zeros_test.

@Test
void match_with_missing_hours_minutes_seconds_adds_zeros_test() {
    when(mockDateMatch.getKey()).thenReturn("logDate");
    when(mockDateMatch.getPatterns()).thenReturn(Collections.singletonList(pattern1));
    List<DateProcessorConfig.DateMatch> dateMatches = Collections.singletonList(mockDateMatch);
    when(mockDateProcessorConfig.getMatch()).thenReturn(dateMatches);
    when(mockDateProcessorConfig.getSourceZoneId()).thenReturn(ZoneId.of("UTC"));
    when(mockDateProcessorConfig.getDestinationZoneId()).thenReturn(ZoneId.systemDefault());
    when(mockDateProcessorConfig.getSourceLocale()).thenReturn(Locale.ROOT);
    dateProcessor = createObjectUnderTest();
    LocalDate localDate = LocalDate.now(ZoneId.of("UTC"));
    testData = getTestData();
    testData.put("logDate", localDate.toString());
    final Record<Event> record = buildRecordWithEvent(testData);
    final List<Record<Event>> processedRecords = (List<Record<Event>>) dateProcessor.doExecute(Collections.singletonList(record));
    ZonedDateTime actualZonedDateTime = processedRecords.get(0).getData().get(TIMESTAMP_KEY, ZonedDateTime.class);
    ZonedDateTime expectedZonedDateTime = localDate.atStartOfDay().atZone(ZoneId.of("UTC"));
    Assertions.assertTrue(actualZonedDateTime.isEqual(expectedZonedDateTime));
    verify(dateProcessingMatchSuccessCounter, times(1)).increment();
}

Example 70 with Record

use of com.amazon.dataprepper.model.record.Record in project data-prepper by opensearch-project.

the class DateProcessorTests method match_without_year_test.

@ParameterizedTest
@ValueSource(strings = { "MMM/dd", "MM dd" })
void match_without_year_test(String pattern) {
    when(mockDateMatch.getKey()).thenReturn("logDate");
    when(mockDateMatch.getPatterns()).thenReturn(Collections.singletonList(pattern));
    List<DateProcessorConfig.DateMatch> dateMatches = Collections.singletonList(mockDateMatch);
    when(mockDateProcessorConfig.getMatch()).thenReturn(dateMatches);
    when(mockDateProcessorConfig.getSourceZoneId()).thenReturn(ZoneId.systemDefault());
    when(mockDateProcessorConfig.getDestinationZoneId()).thenReturn(ZoneId.systemDefault());
    when(mockDateProcessorConfig.getSourceLocale()).thenReturn(Locale.ROOT);
    dateProcessor = createObjectUnderTest();
    Map<String, Object> testData = getTestData();
    testData.put("logDate", expectedDateTime.format(DateTimeFormatter.ofPattern(pattern)));
    final Record<Event> record = buildRecordWithEvent(testData);
    final List<Record<Event>> processedRecords = (List<Record<Event>>) dateProcessor.doExecute(Collections.singletonList(record));
    ZonedDateTime actualZonedDateTime = record.getData().get(TIMESTAMP_KEY, ZonedDateTime.class);
    ZonedDateTime expectedZonedDatetime = expectedDateTime.atZone(mockDateProcessorConfig.getSourceZoneId()).truncatedTo(ChronoUnit.SECONDS);
    Assertions.assertTrue(actualZonedDateTime.toLocalDate().isEqual(expectedZonedDatetime.toLocalDate()));
    verify(dateProcessingMatchSuccessCounter, times(1)).increment();
}