Examples with Record com.amazon.dataprepper.model.record.Record used on opensource projects

Example 16 with Record

use of com.amazon.dataprepper.model.record.Record in project data-prepper by opensearch-project.

the class StdInSourceTests method testStdInSourceWhenStopped.

@Test
public void testStdInSourceWhenStopped() {
    final Queue<Record<Event>> bufferQueue = new LinkedList<>();
    final TestBuffer buffer = new TestBuffer(bufferQueue, 1);
    final StdInSource stdInSource = new StdInSource(TEST_WRITE_TIMEOUT, TEST_PIPELINE_NAME);
    assertThat(buffer.size(), is(equalTo(0)));
    stdInSource.stop();
    stdInSource.start(buffer);
    assertThat(buffer.size(), is(equalTo(0)));
}

Example 17 with Record

use of com.amazon.dataprepper.model.record.Record in project data-prepper by opensearch-project.

the class StdInSourceTests method testStdInSourceWhenBufferTimesout.

@Test
public void testStdInSourceWhenBufferTimesout() {
    final Queue<Record<Event>> bufferQueue = new LinkedList<>();
    final TestBuffer buffer = new TestBuffer(bufferQueue, 1, true);
    final StdInSource stdInSource = new StdInSource(TEST_WRITE_TIMEOUT, TEST_PIPELINE_NAME);
    assertThat(buffer.size(), is(equalTo(0)));
    try {
        stdInSource.start(buffer);
    } catch (RuntimeException ex) {
        assertThat(ex.getMessage(), is(equalTo(format("Pipeline [%s] - Timed out writing to buffer", TEST_PIPELINE_NAME))));
    }
    assertThat(buffer.size(), is(equalTo(0)));
}

Example 18 with Record

use of com.amazon.dataprepper.model.record.Record in project data-prepper by opensearch-project.

the class AggregateProcessorIT method aggregateWithNoConcludingGroupsReturnsExpectedResult.

@RepeatedTest(value = 10)
void aggregateWithNoConcludingGroupsReturnsExpectedResult() throws InterruptedException {
    when(aggregateProcessorConfig.getGroupDuration()).thenReturn(Duration.ofSeconds(1000));
    final AggregateProcessor objectUnderTest = createObjectUnderTest();
    final ExecutorService executorService = Executors.newFixedThreadPool(NUM_THREADS);
    final CountDownLatch countDownLatch = new CountDownLatch(NUM_THREADS);
    for (int i = 0; i < NUM_THREADS; i++) {
        executorService.execute(() -> {
            final List<Record<Event>> recordsOut = (List<Record<Event>>) objectUnderTest.doExecute(eventBatch);
            for (final Record<Event> record : recordsOut) {
                final Map<String, Object> map = record.getData().toMap();
                aggregatedResult.add(map);
            }
            countDownLatch.countDown();
        });
    }
    boolean allThreadsFinished = countDownLatch.await(5L, TimeUnit.SECONDS);
    assertThat(allThreadsFinished, equalTo(true));
    assertThat(aggregatedResult.size(), equalTo(NUM_UNIQUE_EVENTS_PER_BATCH));
    for (final Map<String, Object> uniqueEventMap : uniqueEventMaps) {
        assertThat(aggregatedResult, hasItem(uniqueEventMap));
    }
    for (final Map<String, Object> eventMap : aggregatedResult) {
        assertThat(eventMap, in(uniqueEventMaps));
    }
}

Example 19 with Record

use of com.amazon.dataprepper.model.record.Record in project data-prepper by opensearch-project.

the class AggregateProcessorIT method aggregateWithConcludingGroupsOnceReturnsExpectedResult.

@RepeatedTest(value = 2)
void aggregateWithConcludingGroupsOnceReturnsExpectedResult() throws InterruptedException {
    when(aggregateProcessorConfig.getGroupDuration()).thenReturn(Duration.ofSeconds(GROUP_DURATION_FOR_ONLY_SINGLE_CONCLUDE));
    final AggregateProcessor objectUnderTest = createObjectUnderTest();
    final ExecutorService executorService = Executors.newFixedThreadPool(NUM_THREADS);
    final CountDownLatch countDownLatch = new CountDownLatch(NUM_THREADS);
    objectUnderTest.doExecute(eventBatch);
    Thread.sleep(GROUP_DURATION_FOR_ONLY_SINGLE_CONCLUDE * 1000);
    for (int i = 0; i < NUM_THREADS; i++) {
        executorService.execute(() -> {
            final List<Record<Event>> recordsOut = (List<Record<Event>>) objectUnderTest.doExecute(eventBatch);
            for (final Record<Event> record : recordsOut) {
                final Map<String, Object> map = record.getData().toMap();
                aggregatedResult.add(map);
            }
            countDownLatch.countDown();
        });
    }
    boolean allThreadsFinished = countDownLatch.await(5L, TimeUnit.SECONDS);
    assertThat(allThreadsFinished, equalTo(true));
    assertThat(aggregatedResult.size(), equalTo(NUM_UNIQUE_EVENTS_PER_BATCH));
    for (final Map<String, Object> uniqueEventMap : uniqueEventMaps) {
        assertThat(aggregatedResult, hasItem(uniqueEventMap));
    }
    for (final Map<String, Object> eventMap : aggregatedResult) {
        assertThat(eventMap, in(uniqueEventMaps));
    }
}

Example 20 with Record

use of com.amazon.dataprepper.model.record.Record in project data-prepper by opensearch-project.

the class StringPrepperTests method testStringPrepperDefault.

@Test
public void testStringPrepperDefault() {
    final StringPrepper stringPrepper = createObjectUnderTest();
    final List<Record<Event>> modifiedRecords = (List<Record<Event>>) stringPrepper.execute(TEST_RECORDS);
    stringPrepper.shutdown();
    final List<Event> modifiedRecordEvents = modifiedRecords.stream().map(Record::getData).collect(Collectors.toList());
    assertThat(modifiedRecordEvents.size(), equalTo(2));
    final Event firstEvent = modifiedRecordEvents.get(0);
    final Event secondEvent = modifiedRecordEvents.get(1);
    assertTrue(firstEvent.containsKey(TEST_KEY));
    assertThat(firstEvent.getMetadata().getEventType(), equalTo(TEST_EVENT_TYPE));
    assertThat(firstEvent.get(TEST_KEY, String.class), equalTo(UPPERCASE_TEST_STRING.toUpperCase()));
    assertTrue(secondEvent.containsKey(TEST_KEY));
    assertThat(secondEvent.getMetadata().getEventType(), equalTo(TEST_EVENT_TYPE));
    assertThat(secondEvent.get(TEST_KEY, String.class), equalTo(LOWERCASE_TEST_STRING));
}