Search in sources :

Example 26 with Record

use of com.amazon.dataprepper.model.record.Record in project data-prepper by opensearch-project.

the class PipelineTests method testGetSinks.

@Test
public void testGetSinks() {
    final Source<Record<String>> testSource = new TestSource();
    final TestSink testSink = new TestSink();
    final Pipeline testPipeline = new Pipeline(TEST_PIPELINE_NAME, testSource, new BlockingBuffer(TEST_PIPELINE_NAME), Collections.emptyList(), Collections.singletonList(testSink), TEST_PROCESSOR_THREADS, TEST_READ_BATCH_TIMEOUT);
    assertEquals(1, testPipeline.getSinks().size());
    assertEquals(testSink, testPipeline.getSinks().iterator().next());
}
Also used : TestSource(com.amazon.dataprepper.plugins.TestSource) BlockingBuffer(com.amazon.dataprepper.plugins.buffer.blockingbuffer.BlockingBuffer) Record(com.amazon.dataprepper.model.record.Record) TestSink(com.amazon.dataprepper.plugins.TestSink) Test(org.junit.Test)

Example 27 with Record

use of com.amazon.dataprepper.model.record.Record in project data-prepper by opensearch-project.

the class PeerForwarderTest method testSingleRemoteIpForwardRequestEncodeError.

@Test
public void testSingleRemoteIpForwardRequestEncodeError() throws NoSuchFieldException, IllegalAccessException, DecoderException, UnsupportedEncodingException {
    final List<String> testIps = generateTestIps(2);
    final String peerIp = testIps.get(1);
    when(peerClientPool.getClient(peerIp)).thenReturn(client);
    MetricsTestUtil.initMetrics();
    final PeerForwarder testPeerForwarder = generatePeerForwarder(testIps, 3);
    when(oTelProtoEncoder.convertToResourceSpans(any(Span.class))).thenThrow(new DecoderException());
    reflectivelySetEncoder(testPeerForwarder, oTelProtoEncoder);
    final List<Record<Object>> exportedRecords = testPeerForwarder.doExecute(TEST_SPANS_B.stream().map(span -> new Record<Object>(span)).collect(Collectors.toList()));
    verifyNoInteractions(client);
    Assert.assertEquals(3, exportedRecords.size());
    final List<Span> exportedSpans = exportedRecords.stream().map(record -> (Span) record.getData()).collect(Collectors.toList());
    assertTrue(exportedSpans.containsAll(TEST_SPANS_B));
    assertTrue(TEST_SPANS_B.containsAll(exportedSpans));
}
Also used : Arrays(java.util.Arrays) ArgumentMatchers(org.mockito.ArgumentMatchers) MetricsTestUtil(com.amazon.dataprepper.metrics.MetricsTestUtil) Channel(io.grpc.Channel) Mockito.verifyNoInteractions(org.mockito.Mockito.verifyNoInteractions) ResourceSpans(io.opentelemetry.proto.trace.v1.ResourceSpans) After(org.junit.After) Map(java.util.Map) Mockito.doAnswer(org.mockito.Mockito.doAnswer) Resource(io.opentelemetry.proto.resource.v1.Resource) Record(com.amazon.dataprepper.model.record.Record) DefaultTraceGroupFields(com.amazon.dataprepper.model.trace.DefaultTraceGroupFields) Span(com.amazon.dataprepper.model.trace.Span) DecoderException(org.apache.commons.codec.DecoderException) UUID(java.util.UUID) TraceServiceGrpc(io.opentelemetry.proto.collector.trace.v1.TraceServiceGrpc) Collectors(java.util.stream.Collectors) ByteString(com.google.protobuf.ByteString) MockedStatic(org.mockito.MockedStatic) List(java.util.List) JacksonSpan(com.amazon.dataprepper.model.trace.JacksonSpan) ExportTraceServiceRequest(io.opentelemetry.proto.collector.trace.v1.ExportTraceServiceRequest) OTelProtoCodec(com.amazon.dataprepper.plugins.otel.codec.OTelProtoCodec) MockitoJUnitRunner(org.mockito.junit.MockitoJUnitRunner) UnsupportedEncodingException(java.io.UnsupportedEncodingException) Mockito.mock(org.mockito.Mockito.mock) ArgumentMatchers.any(org.mockito.ArgumentMatchers.any) Mock(org.mockito.Mock) Mockito.mockStatic(org.mockito.Mockito.mockStatic) RunWith(org.junit.runner.RunWith) InstrumentationLibrarySpans(io.opentelemetry.proto.trace.v1.InstrumentationLibrarySpans) HashMap(java.util.HashMap) CompletableFuture(java.util.concurrent.CompletableFuture) MetricNames(com.amazon.dataprepper.metrics.MetricNames) Hex(org.apache.commons.codec.binary.Hex) Supplier(java.util.function.Supplier) ArrayList(java.util.ArrayList) ExecutorService(java.util.concurrent.ExecutorService) Before(org.junit.Before) PluginSetting(com.amazon.dataprepper.model.configuration.PluginSetting) OTelProtoCodec.convertUnixNanosToISO8601(com.amazon.dataprepper.plugins.otel.codec.OTelProtoCodec.convertUnixNanosToISO8601) Assert.assertTrue(org.junit.Assert.assertTrue) Test(org.junit.Test) Mockito.times(org.mockito.Mockito.times) Mockito.when(org.mockito.Mockito.when) Field(java.lang.reflect.Field) Measurement(io.micrometer.core.instrument.Measurement) Mockito.verify(org.mockito.Mockito.verify) ExecutionException(java.util.concurrent.ExecutionException) InstrumentationLibrary(io.opentelemetry.proto.common.v1.InstrumentationLibrary) StringJoiner(java.util.StringJoiner) Assert(org.junit.Assert) Collections(java.util.Collections) Assert.assertEquals(org.junit.Assert.assertEquals) DecoderException(org.apache.commons.codec.DecoderException) Record(com.amazon.dataprepper.model.record.Record) ByteString(com.google.protobuf.ByteString) Span(com.amazon.dataprepper.model.trace.Span) JacksonSpan(com.amazon.dataprepper.model.trace.JacksonSpan) Test(org.junit.Test)

Example 28 with Record

use of com.amazon.dataprepper.model.record.Record in project data-prepper by opensearch-project.

the class PeerForwarderTest method testSingleRemoteIpForwardRequestFutureError.

@Test
public void testSingleRemoteIpForwardRequestFutureError() throws ExecutionException, InterruptedException {
    try (final MockedStatic<CompletableFuture> completableFutureMockedStatic = mockStatic(CompletableFuture.class)) {
        completableFutureMockedStatic.when(() -> CompletableFuture.supplyAsync(ArgumentMatchers.<Supplier<ExportTraceServiceRequest>>any(), any(ExecutorService.class))).thenReturn(completableFuture);
        when(completableFuture.get()).thenThrow(new InterruptedException());
        final List<String> testIps = generateTestIps(2);
        final Channel channel = mock(Channel.class);
        final String peerIp = testIps.get(1);
        final String fullPeerIp = String.format("%s:21890", peerIp);
        when(channel.authority()).thenReturn(fullPeerIp);
        when(peerClientPool.getClient(peerIp)).thenReturn(client);
        when(client.getChannel()).thenReturn(channel);
        MetricsTestUtil.initMetrics();
        final PeerForwarder testPeerForwarder = generatePeerForwarder(testIps, 3);
        final List<Record<Object>> exportedRecords = testPeerForwarder.doExecute(TEST_SPANS_B.stream().map(span -> new Record<Object>(span)).collect(Collectors.toList()));
        verify(completableFuture, times(1)).get();
        Assert.assertEquals(3, exportedRecords.size());
        final List<Span> exportedSpans = exportedRecords.stream().map(record -> (Span) record.getData()).collect(Collectors.toList());
        assertTrue(exportedSpans.containsAll(TEST_SPANS_B));
        assertTrue(TEST_SPANS_B.containsAll(exportedSpans));
    }
}
Also used : ExportTraceServiceRequest(io.opentelemetry.proto.collector.trace.v1.ExportTraceServiceRequest) Arrays(java.util.Arrays) ArgumentMatchers(org.mockito.ArgumentMatchers) MetricsTestUtil(com.amazon.dataprepper.metrics.MetricsTestUtil) Channel(io.grpc.Channel) Mockito.verifyNoInteractions(org.mockito.Mockito.verifyNoInteractions) ResourceSpans(io.opentelemetry.proto.trace.v1.ResourceSpans) After(org.junit.After) Map(java.util.Map) Mockito.doAnswer(org.mockito.Mockito.doAnswer) Resource(io.opentelemetry.proto.resource.v1.Resource) Record(com.amazon.dataprepper.model.record.Record) DefaultTraceGroupFields(com.amazon.dataprepper.model.trace.DefaultTraceGroupFields) Span(com.amazon.dataprepper.model.trace.Span) DecoderException(org.apache.commons.codec.DecoderException) UUID(java.util.UUID) TraceServiceGrpc(io.opentelemetry.proto.collector.trace.v1.TraceServiceGrpc) Collectors(java.util.stream.Collectors) ByteString(com.google.protobuf.ByteString) MockedStatic(org.mockito.MockedStatic) List(java.util.List) JacksonSpan(com.amazon.dataprepper.model.trace.JacksonSpan) ExportTraceServiceRequest(io.opentelemetry.proto.collector.trace.v1.ExportTraceServiceRequest) OTelProtoCodec(com.amazon.dataprepper.plugins.otel.codec.OTelProtoCodec) MockitoJUnitRunner(org.mockito.junit.MockitoJUnitRunner) UnsupportedEncodingException(java.io.UnsupportedEncodingException) Mockito.mock(org.mockito.Mockito.mock) ArgumentMatchers.any(org.mockito.ArgumentMatchers.any) Mock(org.mockito.Mock) Mockito.mockStatic(org.mockito.Mockito.mockStatic) RunWith(org.junit.runner.RunWith) InstrumentationLibrarySpans(io.opentelemetry.proto.trace.v1.InstrumentationLibrarySpans) HashMap(java.util.HashMap) CompletableFuture(java.util.concurrent.CompletableFuture) MetricNames(com.amazon.dataprepper.metrics.MetricNames) Hex(org.apache.commons.codec.binary.Hex) Supplier(java.util.function.Supplier) ArrayList(java.util.ArrayList) ExecutorService(java.util.concurrent.ExecutorService) Before(org.junit.Before) PluginSetting(com.amazon.dataprepper.model.configuration.PluginSetting) OTelProtoCodec.convertUnixNanosToISO8601(com.amazon.dataprepper.plugins.otel.codec.OTelProtoCodec.convertUnixNanosToISO8601) Assert.assertTrue(org.junit.Assert.assertTrue) Test(org.junit.Test) Mockito.times(org.mockito.Mockito.times) Mockito.when(org.mockito.Mockito.when) Field(java.lang.reflect.Field) Measurement(io.micrometer.core.instrument.Measurement) Mockito.verify(org.mockito.Mockito.verify) ExecutionException(java.util.concurrent.ExecutionException) InstrumentationLibrary(io.opentelemetry.proto.common.v1.InstrumentationLibrary) StringJoiner(java.util.StringJoiner) Assert(org.junit.Assert) Collections(java.util.Collections) Assert.assertEquals(org.junit.Assert.assertEquals) Channel(io.grpc.Channel) ByteString(com.google.protobuf.ByteString) Span(com.amazon.dataprepper.model.trace.Span) JacksonSpan(com.amazon.dataprepper.model.trace.JacksonSpan) CompletableFuture(java.util.concurrent.CompletableFuture) ExecutorService(java.util.concurrent.ExecutorService) Record(com.amazon.dataprepper.model.record.Record) Test(org.junit.Test)

Example 29 with Record

use of com.amazon.dataprepper.model.record.Record in project data-prepper by opensearch-project.

the class PeerForwarderTest method testSingleRemoteIpBothLocalAndForwardedRequestWithEventRecordData.

@Test
public void testSingleRemoteIpBothLocalAndForwardedRequestWithEventRecordData() throws DecoderException {
    final List<String> testIps = generateTestIps(2);
    final Channel channel = mock(Channel.class);
    final String peerIp = testIps.get(1);
    when(channel.authority()).thenReturn(String.format("%s:21890", peerIp));
    when(peerClientPool.getClient(peerIp)).thenReturn(client);
    when(client.getChannel()).thenReturn(channel);
    final Map<String, List<ExportTraceServiceRequest>> requestsByIp = testIps.stream().collect(Collectors.toMap(ip -> ip, ip -> new ArrayList<>()));
    doAnswer(invocation -> {
        final ExportTraceServiceRequest exportTraceServiceRequest = invocation.getArgument(0);
        requestsByIp.get(peerIp).add(exportTraceServiceRequest);
        return null;
    }).when(client).export(any(ExportTraceServiceRequest.class));
    MetricsTestUtil.initMetrics();
    final PeerForwarder testPeerForwarder = generatePeerForwarder(testIps, 3);
    final List<Record<Object>> exportedRecords = testPeerForwarder.doExecute(TEST_SPANS_ALL.stream().map(span -> new Record<Object>(span)).collect(Collectors.toList()));
    final List<Span> expectedLocalSpans = Arrays.asList(SPAN_1, SPAN_2, SPAN_3);
    Assert.assertEquals(3, exportedRecords.size());
    final List<Span> localSpans = exportedRecords.stream().map(record -> (Span) record.getData()).collect(Collectors.toList());
    assertTrue(localSpans.containsAll(expectedLocalSpans));
    assertTrue(expectedLocalSpans.containsAll(localSpans));
    Assert.assertEquals(1, requestsByIp.get(peerIp).size());
    final ExportTraceServiceRequest forwardedRequest = requestsByIp.get(peerIp).get(0);
    final List<ResourceSpans> forwardedResourceSpans = forwardedRequest.getResourceSpansList();
    assertEquals(3, forwardedResourceSpans.size());
    forwardedResourceSpans.forEach(rs -> {
        assertEquals(TEST_SERVICE_B, extractServiceName(rs));
        assertEquals(1, rs.getInstrumentationLibrarySpansCount());
        final InstrumentationLibrarySpans ils = rs.getInstrumentationLibrarySpans(0);
        assertEquals(1, ils.getSpansCount());
        final io.opentelemetry.proto.trace.v1.Span sp = ils.getSpans(0);
        assertEquals(TEST_TRACE_ID_2, Hex.encodeHexString(sp.getTraceId().toByteArray()));
    });
}
Also used : Arrays(java.util.Arrays) ArgumentMatchers(org.mockito.ArgumentMatchers) MetricsTestUtil(com.amazon.dataprepper.metrics.MetricsTestUtil) Channel(io.grpc.Channel) Mockito.verifyNoInteractions(org.mockito.Mockito.verifyNoInteractions) ResourceSpans(io.opentelemetry.proto.trace.v1.ResourceSpans) After(org.junit.After) Map(java.util.Map) Mockito.doAnswer(org.mockito.Mockito.doAnswer) Resource(io.opentelemetry.proto.resource.v1.Resource) Record(com.amazon.dataprepper.model.record.Record) DefaultTraceGroupFields(com.amazon.dataprepper.model.trace.DefaultTraceGroupFields) Span(com.amazon.dataprepper.model.trace.Span) DecoderException(org.apache.commons.codec.DecoderException) UUID(java.util.UUID) TraceServiceGrpc(io.opentelemetry.proto.collector.trace.v1.TraceServiceGrpc) Collectors(java.util.stream.Collectors) ByteString(com.google.protobuf.ByteString) MockedStatic(org.mockito.MockedStatic) List(java.util.List) JacksonSpan(com.amazon.dataprepper.model.trace.JacksonSpan) ExportTraceServiceRequest(io.opentelemetry.proto.collector.trace.v1.ExportTraceServiceRequest) OTelProtoCodec(com.amazon.dataprepper.plugins.otel.codec.OTelProtoCodec) MockitoJUnitRunner(org.mockito.junit.MockitoJUnitRunner) UnsupportedEncodingException(java.io.UnsupportedEncodingException) Mockito.mock(org.mockito.Mockito.mock) ArgumentMatchers.any(org.mockito.ArgumentMatchers.any) Mock(org.mockito.Mock) Mockito.mockStatic(org.mockito.Mockito.mockStatic) RunWith(org.junit.runner.RunWith) InstrumentationLibrarySpans(io.opentelemetry.proto.trace.v1.InstrumentationLibrarySpans) HashMap(java.util.HashMap) CompletableFuture(java.util.concurrent.CompletableFuture) MetricNames(com.amazon.dataprepper.metrics.MetricNames) Hex(org.apache.commons.codec.binary.Hex) Supplier(java.util.function.Supplier) ArrayList(java.util.ArrayList) ExecutorService(java.util.concurrent.ExecutorService) Before(org.junit.Before) PluginSetting(com.amazon.dataprepper.model.configuration.PluginSetting) OTelProtoCodec.convertUnixNanosToISO8601(com.amazon.dataprepper.plugins.otel.codec.OTelProtoCodec.convertUnixNanosToISO8601) Assert.assertTrue(org.junit.Assert.assertTrue) Test(org.junit.Test) Mockito.times(org.mockito.Mockito.times) Mockito.when(org.mockito.Mockito.when) Field(java.lang.reflect.Field) Measurement(io.micrometer.core.instrument.Measurement) Mockito.verify(org.mockito.Mockito.verify) ExecutionException(java.util.concurrent.ExecutionException) InstrumentationLibrary(io.opentelemetry.proto.common.v1.InstrumentationLibrary) StringJoiner(java.util.StringJoiner) Assert(org.junit.Assert) Collections(java.util.Collections) Assert.assertEquals(org.junit.Assert.assertEquals) ExportTraceServiceRequest(io.opentelemetry.proto.collector.trace.v1.ExportTraceServiceRequest) Channel(io.grpc.Channel) ArrayList(java.util.ArrayList) ByteString(com.google.protobuf.ByteString) ResourceSpans(io.opentelemetry.proto.trace.v1.ResourceSpans) Span(com.amazon.dataprepper.model.trace.Span) JacksonSpan(com.amazon.dataprepper.model.trace.JacksonSpan) InstrumentationLibrarySpans(io.opentelemetry.proto.trace.v1.InstrumentationLibrarySpans) List(java.util.List) ArrayList(java.util.ArrayList) Record(com.amazon.dataprepper.model.record.Record) Test(org.junit.Test)

Example 30 with Record

use of com.amazon.dataprepper.model.record.Record in project data-prepper by opensearch-project.

the class BufferAccumulatorTest method adding_to_accumulation_count_should_write_to_Buffer.

@ParameterizedTest
@ValueSource(ints = { 2, 10, 20 })
void adding_to_accumulation_count_should_write_to_Buffer(final int accumulationCount) throws Exception {
    recordsToAccumulate = accumulationCount;
    final BufferAccumulator objectUnderTest = createObjectUnderTest();
    final List<Record<?>> knownRecords = new ArrayList<>();
    for (int i = 0; i < accumulationCount - 1; i++) {
        final Record record = createRecord();
        knownRecords.add(record);
        objectUnderTest.add(record);
    }
    verifyNoInteractions(buffer);
    final Collection<Record<?>> actualRecordsWritten = new ArrayList<>();
    doAnswer(a -> actualRecordsWritten.addAll(a.getArgument(0, Collection.class))).when(buffer).writeAll(anyCollection(), anyInt());
    final Record record = createRecord();
    knownRecords.add(record);
    objectUnderTest.add(record);
    verify(buffer).writeAll(anyCollection(), eq(timeoutMillis));
    assertThat(actualRecordsWritten.size(), equalTo(accumulationCount));
    assertThat(actualRecordsWritten, equalTo(knownRecords));
}
Also used : ArrayList(java.util.ArrayList) Record(com.amazon.dataprepper.model.record.Record) ValueSource(org.junit.jupiter.params.provider.ValueSource) ParameterizedTest(org.junit.jupiter.params.ParameterizedTest)

Aggregations

Record (com.amazon.dataprepper.model.record.Record)103 Test (org.junit.Test)43 Measurement (io.micrometer.core.instrument.Measurement)35 StringJoiner (java.util.StringJoiner)35 PluginSetting (com.amazon.dataprepper.model.configuration.PluginSetting)33 ArrayList (java.util.ArrayList)31 Map (java.util.Map)30 Test (org.junit.jupiter.api.Test)30 HashMap (java.util.HashMap)29 List (java.util.List)28 Event (com.amazon.dataprepper.model.event.Event)23 ExportTraceServiceRequest (io.opentelemetry.proto.collector.trace.v1.ExportTraceServiceRequest)20 ResourceSpans (io.opentelemetry.proto.trace.v1.ResourceSpans)19 ParameterizedTest (org.junit.jupiter.params.ParameterizedTest)16 JacksonEvent (com.amazon.dataprepper.model.event.JacksonEvent)14 ByteString (com.google.protobuf.ByteString)13 ExecutorService (java.util.concurrent.ExecutorService)13 Resource (io.opentelemetry.proto.resource.v1.Resource)12 Channel (io.grpc.Channel)11 MetricNames (com.amazon.dataprepper.metrics.MetricNames)10