Examples with Log com.amazon.dataprepper.model.log.Log used on opensource projects

Example 1 with Log

use of com.amazon.dataprepper.model.log.Log in project data-prepper by opensearch-project.

the class HTTPSourceTest method testHTTPJsonResponse400.

@Test
public void testHTTPJsonResponse400() {
    // Prepare
    final String testBadData = "}";
    final int testPayloadSize = testBadData.getBytes().length;
    HTTPSourceUnderTest.start(testBuffer);
    refreshMeasurements();
    // When
    WebClient.of().execute(RequestHeaders.builder().scheme(SessionProtocol.HTTP).authority("127.0.0.1:2021").method(HttpMethod.POST).path("/log/ingest").contentType(MediaType.JSON_UTF_8).build(), HttpData.ofUtf8(testBadData)).aggregate().whenComplete((i, ex) -> assertSecureResponseWithStatusCode(i, HttpStatus.BAD_REQUEST)).join();
    // Then
    Assertions.assertTrue(testBuffer.isEmpty());
    // Verify metrics
    final Measurement requestReceivedCount = MetricsTestUtil.getMeasurementFromList(requestsReceivedMeasurements, Statistic.COUNT);
    Assertions.assertEquals(1.0, requestReceivedCount.getValue());
    final Measurement badRequestsCount = MetricsTestUtil.getMeasurementFromList(badRequestsMeasurements, Statistic.COUNT);
    Assertions.assertEquals(1.0, badRequestsCount.getValue());
}

Example 2 with Log

use of com.amazon.dataprepper.model.log.Log in project data-prepper by opensearch-project.

the class HTTPSourceTest method testHTTPJsonResponse200.

@Test
public void testHTTPJsonResponse200() {
    // Prepare
    final String testData = "[{\"log\": \"somelog\"}]";
    final int testPayloadSize = testData.getBytes().length;
    HTTPSourceUnderTest.start(testBuffer);
    refreshMeasurements();
    // When
    WebClient.of().execute(RequestHeaders.builder().scheme(SessionProtocol.HTTP).authority("127.0.0.1:2021").method(HttpMethod.POST).path("/log/ingest").contentType(MediaType.JSON_UTF_8).build(), HttpData.ofUtf8(testData)).aggregate().whenComplete((i, ex) -> assertSecureResponseWithStatusCode(i, HttpStatus.OK)).join();
    // Then
    Assertions.assertFalse(testBuffer.isEmpty());
    final Map.Entry<Collection<Record<Log>>, CheckpointState> result = testBuffer.read(100);
    List<Record<Log>> records = new ArrayList<>(result.getKey());
    Assertions.assertEquals(1, records.size());
    final Record<Log> record = records.get(0);
    Assertions.assertEquals("somelog", record.getData().get("log", String.class));
    // Verify metrics
    final Measurement requestReceivedCount = MetricsTestUtil.getMeasurementFromList(requestsReceivedMeasurements, Statistic.COUNT);
    Assertions.assertEquals(1.0, requestReceivedCount.getValue());
    final Measurement successRequestsCount = MetricsTestUtil.getMeasurementFromList(successRequestsMeasurements, Statistic.COUNT);
    Assertions.assertEquals(1.0, successRequestsCount.getValue());
    final Measurement requestProcessDurationCount = MetricsTestUtil.getMeasurementFromList(requestProcessDurationMeasurements, Statistic.COUNT);
    Assertions.assertEquals(1.0, requestProcessDurationCount.getValue());
    final Measurement requestProcessDurationMax = MetricsTestUtil.getMeasurementFromList(requestProcessDurationMeasurements, Statistic.MAX);
    Assertions.assertTrue(requestProcessDurationMax.getValue() > 0);
    final Measurement payloadSizeMax = MetricsTestUtil.getMeasurementFromList(payloadSizeSummaryMeasurements, Statistic.MAX);
    Assertions.assertEquals(testPayloadSize, payloadSizeMax.getValue());
}

Example 3 with Log

use of com.amazon.dataprepper.model.log.Log in project data-prepper by opensearch-project.

the class HTTPSourceTest method testHTTPJsonResponse415.

@Test
public void testHTTPJsonResponse415() {
    // Prepare
    final int testMaxPendingRequests = 1;
    final int testThreadCount = 1;
    final int serverTimeoutInMillis = 500;
    when(sourceConfig.getRequestTimeoutInMillis()).thenReturn(serverTimeoutInMillis);
    when(sourceConfig.getMaxPendingRequests()).thenReturn(testMaxPendingRequests);
    when(sourceConfig.getThreadCount()).thenReturn(testThreadCount);
    HTTPSourceUnderTest = new HTTPSource(sourceConfig, pluginMetrics, pluginFactory);
    // Start the source
    HTTPSourceUnderTest.start(testBuffer);
    refreshMeasurements();
    final RequestHeaders testRequestHeaders = RequestHeaders.builder().scheme(SessionProtocol.HTTP).authority("127.0.0.1:2021").method(HttpMethod.POST).path("/log/ingest").contentType(MediaType.JSON_UTF_8).build();
    final HttpData testHttpData = HttpData.ofUtf8("[{\"log\": \"somelog\"}]");
    // Fill in the buffer
    WebClient.of().execute(testRequestHeaders, testHttpData).aggregate().whenComplete((i, ex) -> assertSecureResponseWithStatusCode(i, HttpStatus.OK)).join();
    // Disable client timeout
    WebClient testWebClient = WebClient.builder().responseTimeoutMillis(0).build();
    // When/Then
    testWebClient.execute(testRequestHeaders, testHttpData).aggregate().whenComplete((i, ex) -> assertSecureResponseWithStatusCode(i, HttpStatus.REQUEST_TIMEOUT)).join();
    // verify metrics
    final Measurement requestReceivedCount = MetricsTestUtil.getMeasurementFromList(requestsReceivedMeasurements, Statistic.COUNT);
    Assertions.assertEquals(2.0, requestReceivedCount.getValue());
    final Measurement successRequestsCount = MetricsTestUtil.getMeasurementFromList(successRequestsMeasurements, Statistic.COUNT);
    Assertions.assertEquals(1.0, successRequestsCount.getValue());
    final Measurement requestTimeoutsCount = MetricsTestUtil.getMeasurementFromList(requestTimeoutsMeasurements, Statistic.COUNT);
    Assertions.assertEquals(1.0, requestTimeoutsCount.getValue());
    final Measurement requestProcessDurationMax = MetricsTestUtil.getMeasurementFromList(requestProcessDurationMeasurements, Statistic.MAX);
    final double maxDurationInMillis = 1000 * requestProcessDurationMax.getValue();
    Assertions.assertTrue(maxDurationInMillis > serverTimeoutInMillis);
}

Example 4 with Log

use of com.amazon.dataprepper.model.log.Log in project data-prepper by opensearch-project.

the class HTTPSourceTest method testHTTPJsonResponse413.

@Test
public void testHTTPJsonResponse413() throws InterruptedException {
    // Prepare
    final String testData = "[{\"log\": \"test log 1\"}, {\"log\": \"test log 2\"}]";
    final int testPayloadSize = testData.getBytes().length;
    HTTPSourceUnderTest.start(testBuffer);
    refreshMeasurements();
    // When
    WebClient.of().execute(RequestHeaders.builder().scheme(SessionProtocol.HTTP).authority("127.0.0.1:2021").method(HttpMethod.POST).path("/log/ingest").contentType(MediaType.JSON_UTF_8).build(), HttpData.ofUtf8(testData)).aggregate().whenComplete((i, ex) -> assertSecureResponseWithStatusCode(i, HttpStatus.REQUEST_ENTITY_TOO_LARGE)).join();
    // Then
    Assertions.assertTrue(testBuffer.isEmpty());
    // Verify metrics
    final Measurement requestReceivedCount = MetricsTestUtil.getMeasurementFromList(requestsReceivedMeasurements, Statistic.COUNT);
    Assertions.assertEquals(1.0, requestReceivedCount.getValue());
    final Measurement successRequestsCount = MetricsTestUtil.getMeasurementFromList(successRequestsMeasurements, Statistic.COUNT);
    Assertions.assertEquals(0.0, successRequestsCount.getValue());
    final Measurement requestsTooLargeCount = MetricsTestUtil.getMeasurementFromList(requestsTooLargeMeasurements, Statistic.COUNT);
    Assertions.assertEquals(1.0, requestsTooLargeCount.getValue());
    final Measurement requestProcessDurationCount = MetricsTestUtil.getMeasurementFromList(requestProcessDurationMeasurements, Statistic.COUNT);
    Assertions.assertEquals(1.0, requestProcessDurationCount.getValue());
    final Measurement requestProcessDurationMax = MetricsTestUtil.getMeasurementFromList(requestProcessDurationMeasurements, Statistic.MAX);
    Assertions.assertTrue(requestProcessDurationMax.getValue() > 0);
    final Measurement payloadSizeMax = MetricsTestUtil.getMeasurementFromList(payloadSizeSummaryMeasurements, Statistic.MAX);
    Assertions.assertEquals(testPayloadSize, payloadSizeMax.getValue());
}

Example 5 with Log

use of com.amazon.dataprepper.model.log.Log in project data-prepper by opensearch-project.

the class HTTPSourceTest method testHTTPJsonResponse429.

@Test
public void testHTTPJsonResponse429() throws InterruptedException {
    // Prepare
    final Map<String, Object> settings = new HashMap<>();
    final int testMaxPendingRequests = 1;
    final int testThreadCount = 1;
    final int clientTimeoutInMillis = 100;
    final int serverTimeoutInMillis = (testMaxPendingRequests + testThreadCount + 1) * clientTimeoutInMillis;
    when(sourceConfig.getRequestTimeoutInMillis()).thenReturn(serverTimeoutInMillis);
    when(sourceConfig.getMaxPendingRequests()).thenReturn(testMaxPendingRequests);
    when(sourceConfig.getThreadCount()).thenReturn(testThreadCount);
    HTTPSourceUnderTest = new HTTPSource(sourceConfig, pluginMetrics, pluginFactory);
    // Start the source
    HTTPSourceUnderTest.start(testBuffer);
    refreshMeasurements();
    final RequestHeaders testRequestHeaders = RequestHeaders.builder().scheme(SessionProtocol.HTTP).authority("127.0.0.1:2021").method(HttpMethod.POST).path("/log/ingest").contentType(MediaType.JSON_UTF_8).build();
    final HttpData testHttpData = HttpData.ofUtf8("[{\"log\": \"somelog\"}]");
    // Fill in the buffer
    WebClient.of().execute(testRequestHeaders, testHttpData).aggregate().whenComplete((i, ex) -> assertSecureResponseWithStatusCode(i, HttpStatus.OK)).join();
    // Send requests to throttle the server when buffer is full
    // Set the client timeout to be less than source serverTimeoutInMillis / (testMaxPendingRequests + testThreadCount)
    WebClient testWebClient = WebClient.builder().responseTimeoutMillis(clientTimeoutInMillis).build();
    for (int i = 0; i < testMaxPendingRequests + testThreadCount; i++) {
        CompletionException actualException = Assertions.assertThrows(CompletionException.class, () -> testWebClient.execute(testRequestHeaders, testHttpData).aggregate().join());
        assertThat(actualException.getCause(), instanceOf(ResponseTimeoutException.class));
    }
    // When/Then
    testWebClient.execute(testRequestHeaders, testHttpData).aggregate().whenComplete((i, ex) -> assertSecureResponseWithStatusCode(i, HttpStatus.TOO_MANY_REQUESTS)).join();
    // Wait until source server timeout a request processing thread
    Thread.sleep(serverTimeoutInMillis);
    // New request should timeout instead of being rejected
    CompletionException actualException = Assertions.assertThrows(CompletionException.class, () -> testWebClient.execute(testRequestHeaders, testHttpData).aggregate().join());
    assertThat(actualException.getCause(), instanceOf(ResponseTimeoutException.class));
    // verify metrics
    final Measurement requestReceivedCount = MetricsTestUtil.getMeasurementFromList(requestsReceivedMeasurements, Statistic.COUNT);
    Assertions.assertEquals(testMaxPendingRequests + testThreadCount + 2, requestReceivedCount.getValue());
    final Measurement successRequestsCount = MetricsTestUtil.getMeasurementFromList(successRequestsMeasurements, Statistic.COUNT);
    Assertions.assertEquals(1.0, successRequestsCount.getValue());
    final Measurement rejectedRequestsCount = MetricsTestUtil.getMeasurementFromList(rejectedRequestsMeasurements, Statistic.COUNT);
    Assertions.assertEquals(1.0, rejectedRequestsCount.getValue());
}