Examples with Status org.apache.druid.indexing.seekablestream.SeekableStreamIndexTaskRunner.Status used on opensource projects

Example 1 with Status

use of org.apache.druid.indexing.seekablestream.SeekableStreamIndexTaskRunner.Status in project druid by druid-io.

the class KafkaIndexTaskTest method testMultipleParseExceptionsFailure.

@Test(timeout = 60_000L)
public void testMultipleParseExceptionsFailure() throws Exception {
    reportParseExceptions = false;
    maxParseExceptions = 2;
    maxSavedParseExceptions = 2;
    // Insert data
    insertData();
    final KafkaIndexTask task = createTask(null, new KafkaIndexTaskIOConfig(0, "sequence0", new SeekableStreamStartSequenceNumbers<>(topic, ImmutableMap.of(0, 2L), ImmutableSet.of()), new SeekableStreamEndSequenceNumbers<>(topic, ImmutableMap.of(0, 10L)), kafkaServer.consumerProperties(), KafkaSupervisorIOConfig.DEFAULT_POLL_TIMEOUT_MILLIS, true, null, null, INPUT_FORMAT));
    final ListenableFuture<TaskStatus> future = runTask(task);
    TaskStatus status = future.get();
    // Wait for task to exit
    Assert.assertEquals(TaskState.FAILED, status.getStatusCode());
    IndexTaskTest.checkTaskStatusErrorMsgForParseExceptionsExceeded(status);
    // Check metrics
    Assert.assertEquals(3, task.getRunner().getRowIngestionMeters().getProcessed());
    Assert.assertEquals(0, task.getRunner().getRowIngestionMeters().getProcessedWithError());
    Assert.assertEquals(3, task.getRunner().getRowIngestionMeters().getUnparseable());
    Assert.assertEquals(0, task.getRunner().getRowIngestionMeters().getThrownAway());
    // Check published metadata
    Assert.assertEquals(ImmutableList.of(), publishedDescriptors());
    Assert.assertNull(newDataSchemaMetadata());
    IngestionStatsAndErrorsTaskReportData reportData = getTaskReportData();
    Map<String, Object> expectedMetrics = ImmutableMap.of(RowIngestionMeters.BUILD_SEGMENTS, ImmutableMap.of(RowIngestionMeters.PROCESSED, 3, RowIngestionMeters.PROCESSED_WITH_ERROR, 0, RowIngestionMeters.UNPARSEABLE, 3, RowIngestionMeters.THROWN_AWAY, 0));
    Assert.assertEquals(expectedMetrics, reportData.getRowStats());
    List<LinkedHashMap> parseExceptionReports = (List<LinkedHashMap>) reportData.getUnparseableEvents().get(RowIngestionMeters.BUILD_SEGMENTS);
    List<String> expectedMessages = Arrays.asList("Unable to parse [] as the intermediateRow resulted in empty input row (Record: 1)", "Unable to parse row [unparseable] (Record: 1)");
    List<String> actualMessages = parseExceptionReports.stream().map((r) -> {
        return ((List<String>) r.get("details")).get(0);
    }).collect(Collectors.toList());
    Assert.assertEquals(expectedMessages, actualMessages);
    List<String> expectedInputs = Arrays.asList("", "unparseable");
    List<String> actualInputs = parseExceptionReports.stream().map((r) -> {
        return (String) r.get("input");
    }).collect(Collectors.toList());
    Assert.assertEquals(expectedInputs, actualInputs);
}

Example 2 with Status

use of org.apache.druid.indexing.seekablestream.SeekableStreamIndexTaskRunner.Status in project druid by druid-io.

the class KafkaIndexTaskClientTest method testGetStatus.

@Test
public void testGetStatus() throws Exception {
    Status status = Status.READING;
    Capture<Request> captured = Capture.newInstance();
    EasyMock.expect(responseHolder.getContent()).andReturn(StringUtils.format("\"%s\"", status.toString())).anyTimes();
    EasyMock.expect(httpClient.go(EasyMock.capture(captured), EasyMock.anyObject(ObjectOrErrorResponseHandler.class), EasyMock.eq(TEST_HTTP_TIMEOUT))).andReturn(okResponseHolder());
    replayAll();
    Status results = client.getStatus(TEST_ID);
    verifyAll();
    Request request = captured.getValue();
    Assert.assertEquals(HttpMethod.GET, request.getMethod());
    Assert.assertEquals(new URL("http://test-host:1234/druid/worker/v1/chat/test-id/status"), request.getUrl());
    Assert.assertTrue(null, request.getHeaders().get("X-Druid-Task-Id").contains("test-id"));
    Assert.assertEquals(status, results);
}

Example 3 with Status

use of org.apache.druid.indexing.seekablestream.SeekableStreamIndexTaskRunner.Status in project druid by druid-io.

the class KinesisIndexTaskClientTest method testGetStatusAsync.

@Test
public void testGetStatusAsync() throws Exception {
    final int numRequests = TEST_IDS.size();
    Capture<Request> captured = Capture.newInstance(CaptureType.ALL);
    EasyMock.expect(responseHolder.getStatus()).andReturn(HttpResponseStatus.OK).anyTimes();
    EasyMock.expect(responseHolder.getContent()).andReturn("\"READING\"").anyTimes();
    EasyMock.expect(httpClient.go(EasyMock.capture(captured), EasyMock.anyObject(ObjectOrErrorResponseHandler.class), EasyMock.eq(TEST_HTTP_TIMEOUT))).andReturn(okResponseHolder()).times(numRequests);
    replayAll();
    List<URL> expectedUrls = new ArrayList<>();
    List<ListenableFuture<Status>> futures = new ArrayList<>();
    for (String testId : TEST_IDS) {
        expectedUrls.add(new URL(StringUtils.format(URL_FORMATTER, TEST_HOST, TEST_PORT, testId, "status")));
        futures.add(client.getStatusAsync(testId));
    }
    List<Status> responses = Futures.allAsList(futures).get();
    verifyAll();
    List<Request> requests = captured.getValues();
    Assert.assertEquals(numRequests, requests.size());
    Assert.assertEquals(numRequests, responses.size());
    for (int i = 0; i < numRequests; i++) {
        Assert.assertEquals(HttpMethod.GET, requests.get(i).getMethod());
        Assert.assertTrue("unexpectedURL", expectedUrls.contains(requests.get(i).getUrl()));
        Assert.assertEquals(Status.READING, responses.get(i));
    }
}

Example 4 with Status

use of org.apache.druid.indexing.seekablestream.SeekableStreamIndexTaskRunner.Status in project druid by druid-io.

the class KinesisIndexTaskClientTest method testGetStatus.

@Test
public void testGetStatus() throws Exception {
    Status status = Status.READING;
    Capture<Request> captured = Capture.newInstance();
    EasyMock.expect(responseHolder.getContent()).andReturn(StringUtils.format("\"%s\"", status.toString())).anyTimes();
    EasyMock.expect(httpClient.go(EasyMock.capture(captured), EasyMock.anyObject(ObjectOrErrorResponseHandler.class), EasyMock.eq(TEST_HTTP_TIMEOUT))).andReturn(okResponseHolder());
    replayAll();
    Status results = client.getStatus(TEST_ID);
    verifyAll();
    Request request = captured.getValue();
    Assert.assertEquals(HttpMethod.GET, request.getMethod());
    Assert.assertEquals(new URL("http://test-host:1234/druid/worker/v1/chat/test-id/status"), request.getUrl());
    Assert.assertTrue(null, request.getHeaders().get("X-Druid-Task-Id").contains("test-id"));
    Assert.assertEquals(status, results);
}

Example 5 with Status

use of org.apache.druid.indexing.seekablestream.SeekableStreamIndexTaskRunner.Status in project druid by druid-io.

the class KafkaIndexTaskTest method testMultipleParseExceptionsSuccess.

@Test(timeout = 60_000L)
public void testMultipleParseExceptionsSuccess() throws Exception {
    reportParseExceptions = false;
    maxParseExceptions = 6;
    maxSavedParseExceptions = 6;
    // Insert data
    insertData();
    final KafkaIndexTask task = createTask(null, new KafkaIndexTaskIOConfig(0, "sequence0", new SeekableStreamStartSequenceNumbers<>(topic, ImmutableMap.of(0, 2L), ImmutableSet.of()), new SeekableStreamEndSequenceNumbers<>(topic, ImmutableMap.of(0, 13L)), kafkaServer.consumerProperties(), KafkaSupervisorIOConfig.DEFAULT_POLL_TIMEOUT_MILLIS, true, null, null, INPUT_FORMAT));
    final ListenableFuture<TaskStatus> future = runTask(task);
    TaskStatus status = future.get();
    // Wait for task to exit
    Assert.assertEquals(TaskState.SUCCESS, status.getStatusCode());
    Assert.assertNull(status.getErrorMsg());
    // Check metrics
    Assert.assertEquals(4, task.getRunner().getRowIngestionMeters().getProcessed());
    Assert.assertEquals(3, task.getRunner().getRowIngestionMeters().getProcessedWithError());
    Assert.assertEquals(3, task.getRunner().getRowIngestionMeters().getUnparseable());
    Assert.assertEquals(1, task.getRunner().getRowIngestionMeters().getThrownAway());
    // Check published metadata
    assertEqualsExceptVersion(ImmutableList.of(sdd("2010/P1D", 0), sdd("2011/P1D", 0), sdd("2013/P1D", 0), sdd("2049/P1D", 0)), publishedDescriptors());
    Assert.assertEquals(new KafkaDataSourceMetadata(new SeekableStreamEndSequenceNumbers<>(topic, ImmutableMap.of(0, 13L))), newDataSchemaMetadata());
    IngestionStatsAndErrorsTaskReportData reportData = getTaskReportData();
    Map<String, Object> expectedMetrics = ImmutableMap.of(RowIngestionMeters.BUILD_SEGMENTS, ImmutableMap.of(RowIngestionMeters.PROCESSED, 4, RowIngestionMeters.PROCESSED_WITH_ERROR, 3, RowIngestionMeters.UNPARSEABLE, 3, RowIngestionMeters.THROWN_AWAY, 1));
    Assert.assertEquals(expectedMetrics, reportData.getRowStats());
    List<LinkedHashMap> parseExceptionReports = (List<LinkedHashMap>) reportData.getUnparseableEvents().get(RowIngestionMeters.BUILD_SEGMENTS);
    List<String> expectedMessages = Arrays.asList("Unable to parse value[notanumber] for field[met1]", "could not convert value [notanumber] to float", "could not convert value [notanumber] to long", "Unable to parse [] as the intermediateRow resulted in empty input row (Record: 1)", "Unable to parse row [unparseable] (Record: 1)", "Encountered row with timestamp[246140482-04-24T15:36:27.903Z] that cannot be represented as a long: [{timestamp=246140482-04-24T15:36:27.903Z, dim1=x, dim2=z, dimLong=10, dimFloat=20.0, met1=1.0}] (Record: 1)");
    List<String> actualMessages = parseExceptionReports.stream().map((r) -> {
        return ((List<String>) r.get("details")).get(0);
    }).collect(Collectors.toList());
    Assert.assertEquals(expectedMessages, actualMessages);
    List<String> expectedInputs = Arrays.asList("{timestamp=2049, dim1=f, dim2=y, dimLong=10, dimFloat=20.0, met1=notanumber}", "{timestamp=2049, dim1=f, dim2=y, dimLong=10, dimFloat=notanumber, met1=1.0}", "{timestamp=2049, dim1=f, dim2=y, dimLong=notanumber, dimFloat=20.0, met1=1.0}", "", "unparseable", "{timestamp=246140482-04-24T15:36:27.903Z, dim1=x, dim2=z, dimLong=10, dimFloat=20.0, met1=1.0}");
    List<String> actualInputs = parseExceptionReports.stream().map((r) -> {
        return (String) r.get("input");
    }).collect(Collectors.toList());
    Assert.assertEquals(expectedInputs, actualInputs);
}