Examples with StringFullResponseHolder org.apache.druid.java.util.http.client.response.StringFullResponseHolder used on opensource projects

Example 66 with StringFullResponseHolder

use of org.apache.druid.java.util.http.client.response.StringFullResponseHolder in project druid by apache.

the class SeekableStreamIndexTaskClient method pause.

public Map<PartitionIdType, SequenceOffsetType> pause(final String id) {
    log.debug("Pause task[%s]", id);
    try {
        final StringFullResponseHolder response = submitRequestWithEmptyContent(id, HttpMethod.POST, "pause", null, true);
        final HttpResponseStatus responseStatus = response.getStatus();
        final String responseContent = response.getContent();
        if (responseStatus.equals(HttpResponseStatus.OK)) {
            log.info("Task [%s] paused successfully", id);
            return deserializeMap(responseContent, Map.class, getPartitionType(), getSequenceType());
        } else if (responseStatus.equals(HttpResponseStatus.ACCEPTED)) {
            // The task received the pause request, but its status hasn't been changed yet.
            final RetryPolicy retryPolicy = newRetryPolicy();
            while (true) {
                final SeekableStreamIndexTaskRunner.Status status = getStatus(id);
                if (status == SeekableStreamIndexTaskRunner.Status.PAUSED) {
                    return getCurrentOffsets(id, true);
                }
                final Duration delay = retryPolicy.getAndIncrementRetryDelay();
                if (delay == null) {
                    throw new ISE("Task [%s] failed to change its status from [%s] to [%s], aborting", id, status, SeekableStreamIndexTaskRunner.Status.PAUSED);
                } else {
                    final long sleepTime = delay.getMillis();
                    log.info("Still waiting for task [%s] to change its status to [%s]; will try again in [%s]", id, SeekableStreamIndexTaskRunner.Status.PAUSED, new Duration(sleepTime).toString());
                    Thread.sleep(sleepTime);
                }
            }
        } else {
            throw new ISE("Pause request for task [%s] failed with response [%s] : [%s]", id, responseStatus, responseContent);
        }
    } catch (NoTaskLocationException e) {
        log.error("Exception [%s] while pausing Task [%s]", e.getMessage(), id);
        return ImmutableMap.of();
    } catch (IOException | InterruptedException e) {
        throw new RE(e, "Exception [%s] while pausing Task [%s]", e.getMessage(), id);
    }
}

Example 67 with StringFullResponseHolder

use of org.apache.druid.java.util.http.client.response.StringFullResponseHolder in project druid by apache.

the class WorkerTaskManager method scheduleCompletedTasksCleanup.

private void scheduleCompletedTasksCleanup() {
    completedTasksCleanupExecutor.scheduleAtFixedRate(() -> {
        try {
            if (completedTasks.isEmpty()) {
                log.debug("Skipping completed tasks cleanup. Its empty.");
                return;
            }
            ImmutableSet<String> taskIds = ImmutableSet.copyOf(completedTasks.keySet());
            Map<String, TaskStatus> taskStatusesFromOverlord = null;
            try {
                StringFullResponseHolder fullResponseHolder = overlordClient.go(overlordClient.makeRequest(HttpMethod.POST, "/druid/indexer/v1/taskStatus").setContent(jsonMapper.writeValueAsBytes(taskIds)).addHeader(HttpHeaders.Names.ACCEPT, MediaType.APPLICATION_JSON).addHeader(HttpHeaders.Names.CONTENT_TYPE, MediaType.APPLICATION_JSON));
                if (fullResponseHolder.getStatus().getCode() == 200) {
                    String responseContent = fullResponseHolder.getContent();
                    taskStatusesFromOverlord = jsonMapper.readValue(responseContent, new TypeReference<Map<String, TaskStatus>>() {
                    });
                    log.debug("Received completed task status response [%s].", responseContent);
                } else if (fullResponseHolder.getStatus().getCode() == 404) {
                    // NOTE: this is to support backward compatibility, when overlord doesn't have "activeTasks" endpoint.
                    // this if clause should be removed in a future release.
                    log.debug("Deleting all completed tasks. Overlord appears to be running on older version.");
                    taskStatusesFromOverlord = ImmutableMap.of();
                } else {
                    log.info("Got non-success code[%s] from overlord while getting active tasks. will retry on next scheduled run.", fullResponseHolder.getStatus().getCode());
                }
            } catch (Exception ex) {
                log.warn(ex, "Exception while getting active tasks from overlord. will retry on next scheduled run.");
                if (ex instanceof InterruptedException) {
                    Thread.currentThread().interrupt();
                }
            }
            if (taskStatusesFromOverlord == null) {
                return;
            }
            for (String taskId : taskIds) {
                TaskStatus status = taskStatusesFromOverlord.get(taskId);
                if (status == null || status.isComplete()) {
                    log.debug("Deleting completed task[%s] information, overlord task status[%s].", taskId, status == null ? "unknown" : status.getStatusCode());
                    completedTasks.remove(taskId);
                    File taskFile = new File(getCompletedTaskDir(), taskId);
                    try {
                        Files.deleteIfExists(taskFile.toPath());
                        changeHistory.addChangeRequest(new WorkerHistoryItem.TaskRemoval(taskId));
                    } catch (IOException ex) {
                        log.error(ex, "Failed to delete completed task from disk [%s].", taskFile);
                    }
                }
            }
        } catch (Throwable th) {
            log.error(th, "Got unknown exception while running the scheduled cleanup.");
        }
    }, 1, 5, TimeUnit.MINUTES);
}

Example 68 with StringFullResponseHolder

use of org.apache.druid.java.util.http.client.response.StringFullResponseHolder in project druid by apache.

the class LookupReferencesManagerTest method testDestroyIsCalledAfterRemove.

@Test
public void testDestroyIsCalledAfterRemove() throws Exception {
    LookupExtractorFactory lookupExtractorFactory = EasyMock.createStrictMock(LookupExtractorFactory.class);
    EasyMock.expect(lookupExtractorFactory.start()).andReturn(true).once();
    EasyMock.expect(lookupExtractorFactory.destroy()).andReturn(true).once();
    EasyMock.replay(lookupExtractorFactory);
    Map<String, Object> lookupMap = new HashMap<>();
    lookupMap.put("testMockForDestroyIsCalledAfterRemove", container);
    String strResult = mapper.writeValueAsString(lookupMap);
    Request request = new Request(HttpMethod.GET, new URL("http://localhost:1234/xx"));
    EasyMock.expect(config.getLookupTier()).andReturn(LOOKUP_TIER).anyTimes();
    EasyMock.replay(config);
    EasyMock.expect(druidLeaderClient.makeRequest(HttpMethod.GET, "/druid/coordinator/v1/lookups/config/lookupTier?detailed=true")).andReturn(request);
    StringFullResponseHolder responseHolder = new StringFullResponseHolder(newEmptyResponse(HttpResponseStatus.OK), StandardCharsets.UTF_8).addChunk(strResult);
    EasyMock.expect(druidLeaderClient.go(request)).andReturn(responseHolder);
    EasyMock.replay(druidLeaderClient);
    lookupReferencesManager.start();
    lookupReferencesManager.add("testMock", new LookupExtractorFactoryContainer("0", lookupExtractorFactory));
    lookupReferencesManager.handlePendingNotices();
    lookupReferencesManager.remove("testMock");
    lookupReferencesManager.handlePendingNotices();
    EasyMock.verify(lookupExtractorFactory);
}

Example 69 with StringFullResponseHolder

use of org.apache.druid.java.util.http.client.response.StringFullResponseHolder in project druid by apache.

the class LookupReferencesManagerTest method testStartStop.

@Test
public void testStartStop() throws InterruptedException, IOException {
    lookupReferencesManager = new LookupReferencesManager(new LookupConfig(null), mapper, druidLeaderClient, config);
    Map<String, Object> lookupMap = new HashMap<>();
    lookupMap.put("testMockForStartStop", container);
    String strResult = mapper.writeValueAsString(lookupMap);
    Request request = new Request(HttpMethod.GET, new URL("http://localhost:1234/xx"));
    EasyMock.expect(config.getLookupTier()).andReturn(LOOKUP_TIER).anyTimes();
    EasyMock.replay(config);
    EasyMock.expect(druidLeaderClient.makeRequest(HttpMethod.GET, "/druid/coordinator/v1/lookups/config/lookupTier?detailed=true")).andReturn(request);
    StringFullResponseHolder responseHolder = new StringFullResponseHolder(newEmptyResponse(HttpResponseStatus.OK), StandardCharsets.UTF_8).addChunk(strResult);
    EasyMock.expect(druidLeaderClient.go(request)).andReturn(responseHolder);
    EasyMock.replay(druidLeaderClient);
    Assert.assertFalse(lookupReferencesManager.lifecycleLock.awaitStarted(1, TimeUnit.MICROSECONDS));
    Assert.assertNull(lookupReferencesManager.mainThread);
    Assert.assertNull(lookupReferencesManager.stateRef.get());
    lookupReferencesManager.start();
    Assert.assertTrue(lookupReferencesManager.lifecycleLock.awaitStarted(1, TimeUnit.MICROSECONDS));
    Assert.assertTrue(lookupReferencesManager.mainThread.isAlive());
    Assert.assertNotNull(lookupReferencesManager.stateRef.get());
    lookupReferencesManager.stop();
    Assert.assertFalse(lookupReferencesManager.lifecycleLock.awaitStarted(1, TimeUnit.MICROSECONDS));
    Assert.assertFalse(lookupReferencesManager.mainThread.isAlive());
}

Example 70 with StringFullResponseHolder

use of org.apache.druid.java.util.http.client.response.StringFullResponseHolder in project druid by apache.

the class LookupReferencesManagerTest method testGetNotThere.

@Test
public void testGetNotThere() throws Exception {
    Map<String, Object> lookupMap = new HashMap<>();
    lookupMap.put("testMockForGetNotThere", container);
    String strResult = mapper.writeValueAsString(lookupMap);
    Request request = new Request(HttpMethod.GET, new URL("http://localhost:1234/xx"));
    EasyMock.expect(config.getLookupTier()).andReturn(LOOKUP_TIER).anyTimes();
    EasyMock.replay(config);
    EasyMock.expect(druidLeaderClient.makeRequest(HttpMethod.GET, "/druid/coordinator/v1/lookups/config/lookupTier?detailed=true")).andReturn(request);
    StringFullResponseHolder responseHolder = new StringFullResponseHolder(newEmptyResponse(HttpResponseStatus.OK), StandardCharsets.UTF_8).addChunk(strResult);
    EasyMock.expect(druidLeaderClient.go(request)).andReturn(responseHolder);
    EasyMock.replay(druidLeaderClient);
    lookupReferencesManager.start();
    Assert.assertEquals(Optional.empty(), lookupReferencesManager.get("notThere"));
}