Examples with RetryPolicyFactory org.apache.druid.indexing.common.RetryPolicyFactory used on opensource projects

Example 6 with RetryPolicyFactory

use of org.apache.druid.indexing.common.RetryPolicyFactory in project druid by druid-io.

the class CompactionTask method runTask.

@Override
public TaskStatus runTask(TaskToolbox toolbox) throws Exception {
    final List<ParallelIndexIngestionSpec> ingestionSpecs = createIngestionSchema(toolbox, getTaskLockHelper().getLockGranularityToUse(), segmentProvider, partitionConfigurationManager, dimensionsSpec, transformSpec, metricsSpec, granularitySpec, toolbox.getCoordinatorClient(), segmentCacheManagerFactory, retryPolicyFactory, ioConfig.isDropExisting());
    final List<ParallelIndexSupervisorTask> indexTaskSpecs = IntStream.range(0, ingestionSpecs.size()).mapToObj(i -> {
        // The ID of SubtaskSpecs is used as the base sequenceName in segment allocation protocol.
        // The indexing tasks generated by the compaction task should use different sequenceNames
        // so that they can allocate valid segment IDs with no duplication.
        ParallelIndexIngestionSpec ingestionSpec = ingestionSpecs.get(i);
        final String baseSequenceName = createIndexTaskSpecId(i);
        return newTask(baseSequenceName, ingestionSpec);
    }).collect(Collectors.toList());
    if (indexTaskSpecs.isEmpty()) {
        String msg = StringUtils.format("Can't find segments from inputSpec[%s], nothing to do.", ioConfig.getInputSpec());
        log.warn(msg);
        return TaskStatus.failure(getId(), msg);
    } else {
        registerResourceCloserOnAbnormalExit(currentSubTaskHolder);
        final int totalNumSpecs = indexTaskSpecs.size();
        log.info("Generated [%d] compaction task specs", totalNumSpecs);
        int failCnt = 0;
        for (ParallelIndexSupervisorTask eachSpec : indexTaskSpecs) {
            final String json = toolbox.getJsonMapper().writerWithDefaultPrettyPrinter().writeValueAsString(eachSpec);
            if (!currentSubTaskHolder.setTask(eachSpec)) {
                String errMsg = "Task was asked to stop. Finish as failed.";
                log.info(errMsg);
                return TaskStatus.failure(getId(), errMsg);
            }
            try {
                if (eachSpec.isReady(toolbox.getTaskActionClient())) {
                    log.info("Running indexSpec: " + json);
                    final TaskStatus eachResult = eachSpec.run(toolbox);
                    if (!eachResult.isSuccess()) {
                        failCnt++;
                        log.warn("Failed to run indexSpec: [%s].\nTrying the next indexSpec.", json);
                    }
                } else {
                    failCnt++;
                    log.warn("indexSpec is not ready: [%s].\nTrying the next indexSpec.", json);
                }
            } catch (Exception e) {
                failCnt++;
                log.warn(e, "Failed to run indexSpec: [%s].\nTrying the next indexSpec.", json);
            }
        }
        String msg = StringUtils.format("Ran [%d] specs, [%d] succeeded, [%d] failed", totalNumSpecs, totalNumSpecs - failCnt, failCnt);
        log.info(msg);
        return failCnt == 0 ? TaskStatus.success(getId()) : TaskStatus.failure(getId(), msg);
    }
}

Example 7 with RetryPolicyFactory

use of org.apache.druid.indexing.common.RetryPolicyFactory in project druid by druid-io.

the class RemoteTaskActionClientTest method testSubmitSimple.

@Test
public void testSubmitSimple() throws Exception {
    Request request = new Request(HttpMethod.POST, new URL("http://localhost:1234/xx"));
    EasyMock.expect(druidLeaderClient.makeRequest(HttpMethod.POST, "/druid/indexer/v1/action")).andReturn(request);
    // return status code 200 and a list with size equals 1
    Map<String, Object> responseBody = new HashMap<>();
    final List<TaskLock> expectedLocks = Collections.singletonList(new TimeChunkLock(TaskLockType.SHARED, "groupId", "dataSource", Intervals.of("2019/2020"), "version", 0));
    responseBody.put("result", expectedLocks);
    String strResult = objectMapper.writeValueAsString(responseBody);
    final HttpResponse response = EasyMock.createNiceMock(HttpResponse.class);
    EasyMock.expect(response.getStatus()).andReturn(HttpResponseStatus.OK).anyTimes();
    EasyMock.expect(response.getContent()).andReturn(new BigEndianHeapChannelBuffer(0));
    EasyMock.replay(response);
    StringFullResponseHolder responseHolder = new StringFullResponseHolder(response, StandardCharsets.UTF_8).addChunk(strResult);
    // set up mocks
    EasyMock.expect(druidLeaderClient.go(request)).andReturn(responseHolder);
    EasyMock.replay(druidLeaderClient);
    Task task = NoopTask.create("id", 0);
    RemoteTaskActionClient client = new RemoteTaskActionClient(task, druidLeaderClient, new RetryPolicyFactory(new RetryPolicyConfig()), objectMapper);
    final List<TaskLock> locks = client.submit(new LockListAction());
    Assert.assertEquals(expectedLocks, locks);
    EasyMock.verify(druidLeaderClient);
}

Example 8 with RetryPolicyFactory

use of org.apache.druid.indexing.common.RetryPolicyFactory in project druid by druid-io.

the class RemoteTaskActionClientTest method testSubmitWithIllegalStatusCode.

@Test
public void testSubmitWithIllegalStatusCode() throws Exception {
    // return status code 400 and a list with size equals 1
    Request request = new Request(HttpMethod.POST, new URL("http://localhost:1234/xx"));
    EasyMock.expect(druidLeaderClient.makeRequest(HttpMethod.POST, "/druid/indexer/v1/action")).andReturn(request);
    // return status code 200 and a list with size equals 1
    final HttpResponse response = EasyMock.createNiceMock(HttpResponse.class);
    EasyMock.expect(response.getStatus()).andReturn(HttpResponseStatus.BAD_REQUEST).anyTimes();
    EasyMock.expect(response.getContent()).andReturn(new BigEndianHeapChannelBuffer(0));
    EasyMock.replay(response);
    StringFullResponseHolder responseHolder = new StringFullResponseHolder(response, StandardCharsets.UTF_8).addChunk("testSubmitWithIllegalStatusCode");
    // set up mocks
    EasyMock.expect(druidLeaderClient.go(request)).andReturn(responseHolder);
    EasyMock.replay(druidLeaderClient);
    Task task = NoopTask.create("id", 0);
    RemoteTaskActionClient client = new RemoteTaskActionClient(task, druidLeaderClient, new RetryPolicyFactory(objectMapper.readValue("{\"maxRetryCount\":0}", RetryPolicyConfig.class)), objectMapper);
    expectedException.expect(IOException.class);
    expectedException.expectMessage("Error with status[400 Bad Request] and message[testSubmitWithIllegalStatusCode]. " + "Check overlord logs for details.");
    client.submit(new LockListAction());
}