Examples with TaskQueue io.druid.indexing.overlord.TaskQueue used on opensource projects

Example 1 with TaskQueue

use of io.druid.indexing.overlord.TaskQueue in project druid by druid-io.

the class OverlordResource method taskPost.

@POST
@Path("/task")
@Consumes(MediaType.APPLICATION_JSON)
@Produces(MediaType.APPLICATION_JSON)
public Response taskPost(final Task task, @Context final HttpServletRequest req) {
    if (authConfig.isEnabled()) {
        // This is an experimental feature, see - https://github.com/druid-io/druid/pull/2424
        final String dataSource = task.getDataSource();
        final AuthorizationInfo authorizationInfo = (AuthorizationInfo) req.getAttribute(AuthConfig.DRUID_AUTH_TOKEN);
        Preconditions.checkNotNull(authorizationInfo, "Security is enabled but no authorization info found in the request");
        Access authResult = authorizationInfo.isAuthorized(new Resource(dataSource, ResourceType.DATASOURCE), Action.WRITE);
        if (!authResult.isAllowed()) {
            return Response.status(Response.Status.FORBIDDEN).header("Access-Check-Result", authResult).build();
        }
    }
    return asLeaderWith(taskMaster.getTaskQueue(), new Function<TaskQueue, Response>() {

        @Override
        public Response apply(TaskQueue taskQueue) {
            try {
                taskQueue.add(task);
                return Response.ok(ImmutableMap.of("task", task.getId())).build();
            } catch (EntryExistsException e) {
                return Response.status(Response.Status.BAD_REQUEST).entity(ImmutableMap.of("error", String.format("Task[%s] already exists!", task.getId()))).build();
            }
        }
    });
}

Example 2 with TaskQueue

use of io.druid.indexing.overlord.TaskQueue in project druid by druid-io.

the class KafkaSupervisorTest method setUp.

@Before
public void setUp() throws Exception {
    taskStorage = createMock(TaskStorage.class);
    taskMaster = createMock(TaskMaster.class);
    taskRunner = createMock(TaskRunner.class);
    indexerMetadataStorageCoordinator = createMock(IndexerMetadataStorageCoordinator.class);
    taskClient = createMock(KafkaIndexTaskClient.class);
    taskQueue = createMock(TaskQueue.class);
    zkServer = new TestingCluster(1);
    zkServer.start();
    kafkaServer = new TestBroker(zkServer.getConnectString(), tempFolder.newFolder(), 1, ImmutableMap.of("num.partitions", String.valueOf(NUM_PARTITIONS)));
    kafkaServer.start();
    kafkaHost = String.format("localhost:%d", kafkaServer.getPort());
    dataSchema = getDataSchema(DATASOURCE);
    tuningConfig = new KafkaSupervisorTuningConfig(1000, 50000, new Period("P1Y"), new File("/test"), null, null, true, false, null, null, numThreads, TEST_CHAT_THREADS, TEST_CHAT_RETRIES, TEST_HTTP_TIMEOUT, TEST_SHUTDOWN_TIMEOUT);
}

Example 3 with TaskQueue

use of io.druid.indexing.overlord.TaskQueue in project druid by druid-io.

the class KafkaSupervisor method createKafkaTasksForGroup.

private void createKafkaTasksForGroup(int groupId, int replicas) {
    Map<Integer, Long> startPartitions = taskGroups.get(groupId).partitionOffsets;
    Map<Integer, Long> endPartitions = new HashMap<>();
    for (Integer partition : startPartitions.keySet()) {
        endPartitions.put(partition, Long.MAX_VALUE);
    }
    String sequenceName = generateSequenceName(groupId);
    Map<String, String> consumerProperties = Maps.newHashMap(ioConfig.getConsumerProperties());
    DateTime minimumMessageTime = taskGroups.get(groupId).minimumMessageTime.orNull();
    KafkaIOConfig kafkaIOConfig = new KafkaIOConfig(sequenceName, new KafkaPartitions(ioConfig.getTopic(), startPartitions), new KafkaPartitions(ioConfig.getTopic(), endPartitions), consumerProperties, true, false, minimumMessageTime);
    for (int i = 0; i < replicas; i++) {
        String taskId = Joiner.on("_").join(sequenceName, getRandomId());
        KafkaIndexTask indexTask = new KafkaIndexTask(taskId, new TaskResource(sequenceName, 1), spec.getDataSchema(), taskTuningConfig, kafkaIOConfig, spec.getContext(), null);
        Optional<TaskQueue> taskQueue = taskMaster.getTaskQueue();
        if (taskQueue.isPresent()) {
            try {
                taskQueue.get().add(indexTask);
            } catch (EntryExistsException e) {
                log.error("Tried to add task [%s] but it already exists", indexTask.getId());
            }
        } else {
            log.error("Failed to get task queue because I'm not the leader!");
        }
    }
}