Examples with TaskResource io.druid.indexing.common.task.TaskResource used on opensource projects

Example 6 with TaskResource

use of io.druid.indexing.common.task.TaskResource in project druid by druid-io.

the class KafkaSupervisor method createKafkaTasksForGroup.

private void createKafkaTasksForGroup(int groupId, int replicas) {
    Map<Integer, Long> startPartitions = taskGroups.get(groupId).partitionOffsets;
    Map<Integer, Long> endPartitions = new HashMap<>();
    for (Integer partition : startPartitions.keySet()) {
        endPartitions.put(partition, Long.MAX_VALUE);
    }
    String sequenceName = generateSequenceName(groupId);
    Map<String, String> consumerProperties = Maps.newHashMap(ioConfig.getConsumerProperties());
    DateTime minimumMessageTime = taskGroups.get(groupId).minimumMessageTime.orNull();
    KafkaIOConfig kafkaIOConfig = new KafkaIOConfig(sequenceName, new KafkaPartitions(ioConfig.getTopic(), startPartitions), new KafkaPartitions(ioConfig.getTopic(), endPartitions), consumerProperties, true, false, minimumMessageTime);
    for (int i = 0; i < replicas; i++) {
        String taskId = Joiner.on("_").join(sequenceName, getRandomId());
        KafkaIndexTask indexTask = new KafkaIndexTask(taskId, new TaskResource(sequenceName, 1), spec.getDataSchema(), taskTuningConfig, kafkaIOConfig, spec.getContext(), null);
        Optional<TaskQueue> taskQueue = taskMaster.getTaskQueue();
        if (taskQueue.isPresent()) {
            try {
                taskQueue.get().add(indexTask);
            } catch (EntryExistsException e) {
                log.error("Tried to add task [%s] but it already exists", indexTask.getId());
            }
        } else {
            log.error("Failed to get task queue because I'm not the leader!");
        }
    }
}

Example 7 with TaskResource

use of io.druid.indexing.common.task.TaskResource in project druid by druid-io.

the class RemoteTaskRunnerTest method testRunWithCapacity.

@Test
public void testRunWithCapacity() throws Exception {
    doSetup();
    TestRealtimeTask task1 = new TestRealtimeTask("rt1", new TaskResource("rt1", 1), "foo", TaskStatus.running("rt1"), jsonMapper);
    remoteTaskRunner.run(task1);
    Assert.assertTrue(taskAnnounced(task1.getId()));
    mockWorkerRunningTask(task1);
    TestRealtimeTask task2 = new TestRealtimeTask("rt2", new TaskResource("rt2", 3), "foo", TaskStatus.running("rt2"), jsonMapper);
    remoteTaskRunner.run(task2);
    TestRealtimeTask task3 = new TestRealtimeTask("rt3", new TaskResource("rt3", 2), "foo", TaskStatus.running("rt3"), jsonMapper);
    remoteTaskRunner.run(task3);
    Assert.assertTrue(taskAnnounced(task3.getId()));
    mockWorkerRunningTask(task3);
    Assert.assertTrue(TestUtils.conditionValid(new IndexingServiceCondition() {

        @Override
        public boolean isValid() {
            return remoteTaskRunner.getRunningTasks().size() == 2;
        }
    }));
    Assert.assertTrue(TestUtils.conditionValid(new IndexingServiceCondition() {

        @Override
        public boolean isValid() {
            return remoteTaskRunner.getPendingTasks().size() == 1;
        }
    }));
    Assert.assertTrue(remoteTaskRunner.getPendingTasks().iterator().next().getTaskId().equals("rt2"));
}

Example 8 with TaskResource

use of io.druid.indexing.common.task.TaskResource in project druid by druid-io.

the class TaskLifecycleTest method testSimple.

@Test
public void testSimple() throws Exception {
    final Task task = new AbstractFixedIntervalTask("id1", "id1", new TaskResource("id1", 1), "ds", new Interval("2012-01-01/P1D"), null) {

        @Override
        public String getType() {
            return "test";
        }

        @Override
        public TaskStatus run(TaskToolbox toolbox) throws Exception {
            final TaskLock myLock = Iterables.getOnlyElement(toolbox.getTaskActionClient().submit(new LockListAction()));
            final DataSegment segment = DataSegment.builder().dataSource("ds").interval(new Interval("2012-01-01/P1D")).version(myLock.getVersion()).build();
            toolbox.getTaskActionClient().submit(new SegmentInsertAction(ImmutableSet.of(segment)));
            return TaskStatus.success(getId());
        }
    };
    final TaskStatus status = runTask(task);
    Assert.assertEquals("statusCode", TaskStatus.Status.SUCCESS, status.getStatusCode());
    Assert.assertEquals("segments published", 1, mdc.getPublished().size());
    Assert.assertEquals("segments nuked", 0, mdc.getNuked().size());
}