Examples with Record com.amazon.dataprepper.model.record.Record used on opensource projects

Example 71 with Record

use of com.amazon.dataprepper.model.record.Record in project data-prepper by opensearch-project.

the class DropEventsProcessorTests method testGivenWhenSettingThenIsStatementFalseUsed.

@Test
void testGivenWhenSettingThenIsStatementFalseUsed() {
    final Event event = mock(Event.class);
    final Record<Event> record = mock(Record.class);
    final int inputRecordCount = 10;
    final List<Record<Event>> recordsToBeProcessed = Collections.nCopies(inputRecordCount, record);
    final int numberOfMockedFalseEvaluations = 8;
    final boolean repeatedReturnValue = false;
    doReturn(whenSetting).when(dropEventProcessorConfig).getDropWhen();
    doReturn(true, true, repeatedReturnValue).when(expressionEvaluator).evaluate(eq(whenSetting), eq(event));
    doReturn(event).when(record).getData();
    dropProcessor = new DropEventsProcessor(pluginMetrics, dropEventProcessorConfig, expressionEvaluator);
    final Collection<Record<Event>> results = dropProcessor.doExecute(recordsToBeProcessed);
    assertThat(results.size(), is(numberOfMockedFalseEvaluations));
    verify(record, times(inputRecordCount)).getData();
}

Example 72 with Record

use of com.amazon.dataprepper.model.record.Record in project data-prepper by opensearch-project.

the class PipelineTests method testPipelineState.

@Test
public void testPipelineState() {
    final Source<Record<String>> testSource = new TestSource();
    final TestSink testSink = new TestSink();
    final Pipeline testPipeline = new Pipeline(TEST_PIPELINE_NAME, testSource, new BlockingBuffer(TEST_PIPELINE_NAME), Collections.emptyList(), Collections.singletonList(testSink), TEST_PROCESSOR_THREADS, TEST_READ_BATCH_TIMEOUT);
    assertThat("Pipeline isStopRequested is expected to be false", testPipeline.isStopRequested(), is(false));
    assertThat("Pipeline is expected to have a default buffer", testPipeline.getBuffer(), notNullValue());
    assertTrue("Pipeline processors should be empty", testPipeline.getProcessorSets().isEmpty());
    testPipeline.execute();
    assertThat("Pipeline isStopRequested is expected to be false", testPipeline.isStopRequested(), is(false));
    testPipeline.shutdown();
    assertThat("Pipeline isStopRequested is expected to be true", testPipeline.isStopRequested(), is(true));
    assertThat("Sink shutdown should be called", testSink.isShutdown, is(true));
}

Example 73 with Record

use of com.amazon.dataprepper.model.record.Record in project data-prepper by opensearch-project.

the class PipelineTests method testPipelineStateWithPrepper.

@Test
public void testPipelineStateWithPrepper() {
    final Source<Record<String>> testSource = new TestSource();
    final TestSink testSink = new TestSink();
    final TestProcessor testProcessor = new TestPrepper(new PluginSetting("test_processor", new HashMap<>()));
    final Pipeline testPipeline = new Pipeline(TEST_PIPELINE_NAME, testSource, new BlockingBuffer(TEST_PIPELINE_NAME), Collections.singletonList(Collections.singletonList(testProcessor)), Collections.singletonList(testSink), TEST_PROCESSOR_THREADS, TEST_READ_BATCH_TIMEOUT);
    assertThat("Pipeline isStopRequested is expected to be false", testPipeline.isStopRequested(), is(false));
    assertThat("Pipeline is expected to have a default buffer", testPipeline.getBuffer(), notNullValue());
    assertEquals("Pipeline processorSets size should be 1", 1, testPipeline.getProcessorSets().size());
    testPipeline.execute();
    assertThat("Pipeline isStopRequested is expected to be false", testPipeline.isStopRequested(), is(false));
    testPipeline.shutdown();
    assertThat("Pipeline isStopRequested is expected to be true", testPipeline.isStopRequested(), is(true));
    assertThat("Sink shutdown should be called", testSink.isShutdown, is(true));
    assertThat("Processor shutdown should be called", testProcessor.isShutdown, is(true));
}

Example 74 with Record

use of com.amazon.dataprepper.model.record.Record in project data-prepper by opensearch-project.

the class PipelineTests method testPipelineStateWithProcessor.

@Test
public void testPipelineStateWithProcessor() {
    final Source<Record<String>> testSource = new TestSource();
    final TestSink testSink = new TestSink();
    final TestProcessor testProcessor = new TestProcessor(new PluginSetting("test_processor", new HashMap<>()));
    final Pipeline testPipeline = new Pipeline(TEST_PIPELINE_NAME, testSource, new BlockingBuffer(TEST_PIPELINE_NAME), Collections.singletonList(Collections.singletonList(testProcessor)), Collections.singletonList(testSink), TEST_PROCESSOR_THREADS, TEST_READ_BATCH_TIMEOUT);
    assertThat("Pipeline isStopRequested is expected to be false", testPipeline.isStopRequested(), is(false));
    assertThat("Pipeline is expected to have a default buffer", testPipeline.getBuffer(), notNullValue());
    assertEquals("Pipeline processorSets size should be 1", 1, testPipeline.getProcessorSets().size());
    testPipeline.execute();
    assertThat("Pipeline isStopRequested is expected to be false", testPipeline.isStopRequested(), is(false));
    testPipeline.shutdown();
    assertThat("Pipeline isStopRequested is expected to be true", testPipeline.isStopRequested(), is(true));
    assertThat("Sink shutdown should be called", testSink.isShutdown, is(true));
    assertThat("Processor shutdown should be called", testProcessor.isShutdown, is(true));
}

Example 75 with Record

use of com.amazon.dataprepper.model.record.Record in project data-prepper by opensearch-project.

the class AbstractBufferTest method testCheckpointMetrics.

@Test
public void testCheckpointMetrics() throws Exception {
    // Given
    final AbstractBuffer<Record<String>> abstractBuffer = new AbstractBufferImpl(testPluginSetting);
    final Collection<Record<String>> testRecords = new ArrayList<>();
    for (int i = 0; i < 5; i++) {
        testRecords.add(new Record<>(UUID.randomUUID().toString()));
    }
    abstractBuffer.writeAll(testRecords, 1000);
    final Map.Entry<Collection<Record<String>>, CheckpointState> readResult = abstractBuffer.read(1000);
    final List<Measurement> checkpointTimeMeasurements = MetricsTestUtil.getMeasurementList(new StringJoiner(MetricNames.DELIMITER).add(PIPELINE_NAME).add(BUFFER_NAME).add(MetricNames.CHECKPOINT_TIME_ELAPSED).toString());
    Assert.assertEquals(0.0, MetricsTestUtil.getMeasurementFromList(checkpointTimeMeasurements, Statistic.COUNT).getValue(), 0);
    Assert.assertEquals(0.0, MetricsTestUtil.getMeasurementFromList(checkpointTimeMeasurements, Statistic.TOTAL_TIME).getValue(), 0);
    // When
    abstractBuffer.checkpoint(readResult.getValue());
    // Then
    Assert.assertEquals(0, abstractBuffer.getRecordsInFlight());
    final List<Measurement> recordsInFlightMeasurements = MetricsTestUtil.getMeasurementList(new StringJoiner(MetricNames.DELIMITER).add(PIPELINE_NAME).add(BUFFER_NAME).add(MetricNames.RECORDS_INFLIGHT).toString());
    final List<Measurement> recordsProcessedMeasurements = MetricsTestUtil.getMeasurementList(new StringJoiner(MetricNames.DELIMITER).add(PIPELINE_NAME).add(MetricNames.RECORDS_PROCESSED).toString());
    final Measurement recordsInFlightMeasurement = recordsInFlightMeasurements.get(0);
    final Measurement recordsProcessedMeasurement = recordsProcessedMeasurements.get(0);
    Assert.assertEquals(0.0, recordsInFlightMeasurement.getValue(), 0);
    Assert.assertEquals(5.0, recordsProcessedMeasurement.getValue(), 0);
    Assert.assertEquals(1.0, MetricsTestUtil.getMeasurementFromList(checkpointTimeMeasurements, Statistic.COUNT).getValue(), 0);
    Assert.assertTrue(MetricsTestUtil.isBetween(MetricsTestUtil.getMeasurementFromList(checkpointTimeMeasurements, Statistic.TOTAL_TIME).getValue(), 0.0, 0.001));
}