Examples with StringFullResponseHolder org.apache.druid.java.util.http.client.response.StringFullResponseHolder used on opensource projects

Example 36 with StringFullResponseHolder

use of org.apache.druid.java.util.http.client.response.StringFullResponseHolder in project hive by apache.

the class DruidStorageHandler method fetchKafkaSupervisorReport.

/**
 * Fetches kafka supervisor status report from druid overlord. This method will return null if can not fetch report
 *
 * @param table object.
 * @return kafka supervisor report or null when druid overlord is unreachable.
 */
@Nullable
private KafkaSupervisorReport fetchKafkaSupervisorReport(Table table) {
    final String overlordAddress = Preconditions.checkNotNull(HiveConf.getVar(getConf(), HiveConf.ConfVars.HIVE_DRUID_OVERLORD_DEFAULT_ADDRESS), "Druid Overlord Address is null");
    final String dataSourceName = Preconditions.checkNotNull(DruidStorageHandlerUtils.getTableProperty(table, Constants.DRUID_DATA_SOURCE), "Druid Datasource name is null");
    try {
        StringFullResponseHolder response = RetryUtils.retry(() -> DruidStorageHandlerUtils.getResponseFromCurrentLeader(getHttpClient(), new Request(HttpMethod.GET, new URL(String.format("http://%s/druid/indexer/v1/supervisor/%s/status", overlordAddress, dataSourceName))), new StringFullResponseHandler(Charset.forName("UTF-8"))), input -> input instanceof IOException, getMaxRetryCount());
        if (response.getStatus().equals(HttpResponseStatus.OK)) {
            return DruidStorageHandlerUtils.JSON_MAPPER.readValue(response.getContent(), KafkaSupervisorReport.class);
        // Druid Returns 400 Bad Request when not found.
        } else if (response.getStatus().equals(HttpResponseStatus.NOT_FOUND) || response.getStatus().equals(HttpResponseStatus.BAD_REQUEST)) {
            LOG.info("No Kafka Supervisor found for datasource[%s]", dataSourceName);
            return null;
        } else {
            LOG.error("Unable to fetch Kafka Supervisor status [%d] full response [%s]", response.getStatus().getCode(), response.getContent());
            return null;
        }
    } catch (Exception e) {
        LOG.error("Exception while fetching kafka ingestion spec from druid", e);
        return null;
    }
}

Example 37 with StringFullResponseHolder

use of org.apache.druid.java.util.http.client.response.StringFullResponseHolder in project druid by druid-io.

the class RemoteTaskActionClientTest method testSubmitSimple.

@Test
public void testSubmitSimple() throws Exception {
    Request request = new Request(HttpMethod.POST, new URL("http://localhost:1234/xx"));
    EasyMock.expect(druidLeaderClient.makeRequest(HttpMethod.POST, "/druid/indexer/v1/action")).andReturn(request);
    // return status code 200 and a list with size equals 1
    Map<String, Object> responseBody = new HashMap<>();
    final List<TaskLock> expectedLocks = Collections.singletonList(new TimeChunkLock(TaskLockType.SHARED, "groupId", "dataSource", Intervals.of("2019/2020"), "version", 0));
    responseBody.put("result", expectedLocks);
    String strResult = objectMapper.writeValueAsString(responseBody);
    final HttpResponse response = EasyMock.createNiceMock(HttpResponse.class);
    EasyMock.expect(response.getStatus()).andReturn(HttpResponseStatus.OK).anyTimes();
    EasyMock.expect(response.getContent()).andReturn(new BigEndianHeapChannelBuffer(0));
    EasyMock.replay(response);
    StringFullResponseHolder responseHolder = new StringFullResponseHolder(response, StandardCharsets.UTF_8).addChunk(strResult);
    // set up mocks
    EasyMock.expect(druidLeaderClient.go(request)).andReturn(responseHolder);
    EasyMock.replay(druidLeaderClient);
    Task task = NoopTask.create("id", 0);
    RemoteTaskActionClient client = new RemoteTaskActionClient(task, druidLeaderClient, new RetryPolicyFactory(new RetryPolicyConfig()), objectMapper);
    final List<TaskLock> locks = client.submit(new LockListAction());
    Assert.assertEquals(expectedLocks, locks);
    EasyMock.verify(druidLeaderClient);
}

Example 38 with StringFullResponseHolder

use of org.apache.druid.java.util.http.client.response.StringFullResponseHolder in project druid by druid-io.

the class RemoteTaskActionClientTest method testSubmitWithIllegalStatusCode.

@Test
public void testSubmitWithIllegalStatusCode() throws Exception {
    // return status code 400 and a list with size equals 1
    Request request = new Request(HttpMethod.POST, new URL("http://localhost:1234/xx"));
    EasyMock.expect(druidLeaderClient.makeRequest(HttpMethod.POST, "/druid/indexer/v1/action")).andReturn(request);
    // return status code 200 and a list with size equals 1
    final HttpResponse response = EasyMock.createNiceMock(HttpResponse.class);
    EasyMock.expect(response.getStatus()).andReturn(HttpResponseStatus.BAD_REQUEST).anyTimes();
    EasyMock.expect(response.getContent()).andReturn(new BigEndianHeapChannelBuffer(0));
    EasyMock.replay(response);
    StringFullResponseHolder responseHolder = new StringFullResponseHolder(response, StandardCharsets.UTF_8).addChunk("testSubmitWithIllegalStatusCode");
    // set up mocks
    EasyMock.expect(druidLeaderClient.go(request)).andReturn(responseHolder);
    EasyMock.replay(druidLeaderClient);
    Task task = NoopTask.create("id", 0);
    RemoteTaskActionClient client = new RemoteTaskActionClient(task, druidLeaderClient, new RetryPolicyFactory(objectMapper.readValue("{\"maxRetryCount\":0}", RetryPolicyConfig.class)), objectMapper);
    expectedException.expect(IOException.class);
    expectedException.expectMessage("Error with status[400 Bad Request] and message[testSubmitWithIllegalStatusCode]. " + "Check overlord logs for details.");
    client.submit(new LockListAction());
}

Example 39 with StringFullResponseHolder

use of org.apache.druid.java.util.http.client.response.StringFullResponseHolder in project druid by druid-io.

the class WorkerTaskManager method scheduleCompletedTasksCleanup.

private void scheduleCompletedTasksCleanup() {
    completedTasksCleanupExecutor.scheduleAtFixedRate(() -> {
        try {
            if (completedTasks.isEmpty()) {
                log.debug("Skipping completed tasks cleanup. Its empty.");
                return;
            }
            ImmutableSet<String> taskIds = ImmutableSet.copyOf(completedTasks.keySet());
            Map<String, TaskStatus> taskStatusesFromOverlord = null;
            try {
                StringFullResponseHolder fullResponseHolder = overlordClient.go(overlordClient.makeRequest(HttpMethod.POST, "/druid/indexer/v1/taskStatus").setContent(jsonMapper.writeValueAsBytes(taskIds)).addHeader(HttpHeaders.Names.ACCEPT, MediaType.APPLICATION_JSON).addHeader(HttpHeaders.Names.CONTENT_TYPE, MediaType.APPLICATION_JSON));
                if (fullResponseHolder.getStatus().getCode() == 200) {
                    String responseContent = fullResponseHolder.getContent();
                    taskStatusesFromOverlord = jsonMapper.readValue(responseContent, new TypeReference<Map<String, TaskStatus>>() {
                    });
                    log.debug("Received completed task status response [%s].", responseContent);
                } else if (fullResponseHolder.getStatus().getCode() == 404) {
                    // NOTE: this is to support backward compatibility, when overlord doesn't have "activeTasks" endpoint.
                    // this if clause should be removed in a future release.
                    log.debug("Deleting all completed tasks. Overlord appears to be running on older version.");
                    taskStatusesFromOverlord = ImmutableMap.of();
                } else {
                    log.info("Got non-success code[%s] from overlord while getting active tasks. will retry on next scheduled run.", fullResponseHolder.getStatus().getCode());
                }
            } catch (Exception ex) {
                log.warn(ex, "Exception while getting active tasks from overlord. will retry on next scheduled run.");
                if (ex instanceof InterruptedException) {
                    Thread.currentThread().interrupt();
                }
            }
            if (taskStatusesFromOverlord == null) {
                return;
            }
            for (String taskId : taskIds) {
                TaskStatus status = taskStatusesFromOverlord.get(taskId);
                if (status == null || status.isComplete()) {
                    log.debug("Deleting completed task[%s] information, overlord task status[%s].", taskId, status == null ? "unknown" : status.getStatusCode());
                    completedTasks.remove(taskId);
                    File taskFile = new File(getCompletedTaskDir(), taskId);
                    try {
                        Files.deleteIfExists(taskFile.toPath());
                        changeHistory.addChangeRequest(new WorkerHistoryItem.TaskRemoval(taskId));
                    } catch (IOException ex) {
                        log.error(ex, "Failed to delete completed task from disk [%s].", taskFile);
                    }
                }
            }
        } catch (Throwable th) {
            log.error(th, "Got unknown exception while running the scheduled cleanup.");
        }
    }, 1, 5, TimeUnit.MINUTES);
}

Example 40 with StringFullResponseHolder

use of org.apache.druid.java.util.http.client.response.StringFullResponseHolder in project druid by druid-io.

the class IndexTaskClientTest method dontRetryIfNotFoundWithCorrectTaskId.

@Test
public void dontRetryIfNotFoundWithCorrectTaskId() {
    final String taskId = "taskId";
    final HttpClient httpClient = EasyMock.createMock(HttpClient.class);
    final DefaultHttpResponse response = new DefaultHttpResponse(HttpVersion.HTTP_1_1, HttpResponseStatus.NOT_FOUND);
    response.headers().add(ChatHandlerResource.TASK_ID_HEADER, taskId);
    EasyMock.expect(httpClient.go(EasyMock.anyObject(), EasyMock.anyObject(), EasyMock.anyObject())).andReturn(Futures.immediateFuture(Either.error(new StringFullResponseHolder(response, StandardCharsets.UTF_8).addChunk("Error")))).times(1);
    EasyMock.replay(httpClient);
    try (IndexTaskClient indexTaskClient = buildIndexTaskClient(httpClient, id -> TaskLocation.create(id, 8000, -1))) {
        final IOException e = Assert.assertThrows(IOException.class, () -> indexTaskClient.submitRequestWithEmptyContent(taskId, HttpMethod.GET, "test", null, false));
        Assert.assertEquals("Received server error with status [404 Not Found]; first 1KB of body: Error", e.getMessage());
    }
    EasyMock.verify(httpClient);
}