Examples with TaskStatus io.druid.indexing.common.TaskStatus used on opensource projects

Example 6 with TaskStatus

use of io.druid.indexing.common.TaskStatus in project druid by druid-io.

the class KafkaIndexTaskTest method testHandoffConditionTimeoutWhenHandoffOccurs.

@Test(timeout = 60_000L)
public void testHandoffConditionTimeoutWhenHandoffOccurs() throws Exception {
    handoffConditionTimeout = 5_000;
    // Insert data
    try (final KafkaProducer<byte[], byte[]> kafkaProducer = kafkaServer.newProducer()) {
        for (ProducerRecord<byte[], byte[]> record : RECORDS) {
            kafkaProducer.send(record).get();
        }
    }
    final KafkaIndexTask task = createTask(null, new KafkaIOConfig("sequence0", new KafkaPartitions("topic0", ImmutableMap.of(0, 2L)), new KafkaPartitions("topic0", ImmutableMap.of(0, 5L)), kafkaServer.consumerProperties(), true, false, null), null, null);
    final ListenableFuture<TaskStatus> future = runTask(task);
    // Wait for task to exit
    Assert.assertEquals(TaskStatus.Status.SUCCESS, future.get().getStatusCode());
    // Check metrics
    Assert.assertEquals(3, task.getFireDepartmentMetrics().processed());
    Assert.assertEquals(0, task.getFireDepartmentMetrics().unparseable());
    Assert.assertEquals(0, task.getFireDepartmentMetrics().thrownAway());
    // Check published metadata
    SegmentDescriptor desc1 = SD(task, "2010/P1D", 0);
    SegmentDescriptor desc2 = SD(task, "2011/P1D", 0);
    Assert.assertEquals(ImmutableSet.of(desc1, desc2), publishedDescriptors());
    Assert.assertEquals(new KafkaDataSourceMetadata(new KafkaPartitions("topic0", ImmutableMap.of(0, 5L))), metadataStorageCoordinator.getDataSourceMetadata(DATA_SCHEMA.getDataSource()));
    // Check segments in deep storage
    Assert.assertEquals(ImmutableList.of("c"), readSegmentDim1(desc1));
    Assert.assertEquals(ImmutableList.of("d", "e"), readSegmentDim1(desc2));
}

Example 7 with TaskStatus

use of io.druid.indexing.common.TaskStatus in project druid by druid-io.

the class QuotableWhiteSpaceSplitter method restore.

@Override
public List<Pair<Task, ListenableFuture<TaskStatus>>> restore() {
    final File restoreFile = getRestoreFile();
    final TaskRestoreInfo taskRestoreInfo;
    if (restoreFile.exists()) {
        try {
            taskRestoreInfo = jsonMapper.readValue(restoreFile, TaskRestoreInfo.class);
        } catch (Exception e) {
            log.error(e, "Failed to read restorable tasks from file[%s]. Skipping restore.", restoreFile);
            return ImmutableList.of();
        }
    } else {
        return ImmutableList.of();
    }
    final List<Pair<Task, ListenableFuture<TaskStatus>>> retVal = Lists.newArrayList();
    for (final String taskId : taskRestoreInfo.getRunningTasks()) {
        try {
            final File taskFile = new File(taskConfig.getTaskDir(taskId), "task.json");
            final Task task = jsonMapper.readValue(taskFile, Task.class);
            if (!task.getId().equals(taskId)) {
                throw new ISE("WTF?! Task[%s] restore file had wrong id[%s].", taskId, task.getId());
            }
            if (taskConfig.isRestoreTasksOnRestart() && task.canRestore()) {
                log.info("Restoring task[%s].", task.getId());
                retVal.add(Pair.of(task, run(task)));
            }
        } catch (Exception e) {
            log.warn(e, "Failed to restore task[%s]. Trying to restore other tasks.", taskId);
        }
    }
    log.info("Restored %,d tasks.", retVal.size());
    return retVal;
}

Example 8 with TaskStatus

use of io.druid.indexing.common.TaskStatus in project druid by druid-io.

the class RealtimeIndexTaskTest method testStopBeforeStarting.

@Test(timeout = 60_000L)
public void testStopBeforeStarting() throws Exception {
    final File directory = tempFolder.newFolder();
    final RealtimeIndexTask task1 = makeRealtimeTask(null);
    task1.stopGracefully();
    final TestIndexerMetadataStorageCoordinator mdc = new TestIndexerMetadataStorageCoordinator();
    final TaskToolbox taskToolbox = makeToolbox(task1, mdc, directory);
    final ListenableFuture<TaskStatus> statusFuture = runTask(task1, taskToolbox);
    // Wait for the task to finish.
    final TaskStatus taskStatus = statusFuture.get();
    Assert.assertEquals(TaskStatus.Status.SUCCESS, taskStatus.getStatusCode());
}

Example 9 with TaskStatus

use of io.druid.indexing.common.TaskStatus in project druid by druid-io.

the class RealtimeIndexTaskTest method testRestore.

@Test(timeout = 60_000L)
public void testRestore() throws Exception {
    final File directory = tempFolder.newFolder();
    final RealtimeIndexTask task1 = makeRealtimeTask(null);
    final DataSegment publishedSegment;
    // First run:
    {
        final TestIndexerMetadataStorageCoordinator mdc = new TestIndexerMetadataStorageCoordinator();
        final TaskToolbox taskToolbox = makeToolbox(task1, mdc, directory);
        final ListenableFuture<TaskStatus> statusFuture = runTask(task1, taskToolbox);
        // Wait for firehose to show up, it starts off null.
        while (task1.getFirehose() == null) {
            Thread.sleep(50);
        }
        final TestFirehose firehose = (TestFirehose) task1.getFirehose();
        firehose.addRows(ImmutableList.<InputRow>of(new MapBasedInputRow(now, ImmutableList.of("dim1"), ImmutableMap.<String, Object>of("dim1", "foo"))));
        // Trigger graceful shutdown.
        task1.stopGracefully();
        // Wait for the task to finish. The status doesn't really matter, but we'll check it anyway.
        final TaskStatus taskStatus = statusFuture.get();
        Assert.assertEquals(TaskStatus.Status.SUCCESS, taskStatus.getStatusCode());
        // Nothing should be published.
        Assert.assertEquals(Sets.newHashSet(), mdc.getPublished());
    }
    // Second run:
    {
        final TestIndexerMetadataStorageCoordinator mdc = new TestIndexerMetadataStorageCoordinator();
        final RealtimeIndexTask task2 = makeRealtimeTask(task1.getId());
        final TaskToolbox taskToolbox = makeToolbox(task2, mdc, directory);
        final ListenableFuture<TaskStatus> statusFuture = runTask(task2, taskToolbox);
        // Wait for firehose to show up, it starts off null.
        while (task2.getFirehose() == null) {
            Thread.sleep(50);
        }
        // Do a query, at this point the previous data should be loaded.
        Assert.assertEquals(1, sumMetric(task2, "rows"));
        final TestFirehose firehose = (TestFirehose) task2.getFirehose();
        firehose.addRows(ImmutableList.<InputRow>of(new MapBasedInputRow(now, ImmutableList.of("dim2"), ImmutableMap.<String, Object>of("dim2", "bar"))));
        // Stop the firehose, this will drain out existing events.
        firehose.close();
        // Wait for publish.
        while (mdc.getPublished().isEmpty()) {
            Thread.sleep(50);
        }
        publishedSegment = Iterables.getOnlyElement(mdc.getPublished());
        // Do a query.
        Assert.assertEquals(2, sumMetric(task2, "rows"));
        // Simulate handoff.
        for (Map.Entry<SegmentDescriptor, Pair<Executor, Runnable>> entry : handOffCallbacks.entrySet()) {
            final Pair<Executor, Runnable> executorRunnablePair = entry.getValue();
            Assert.assertEquals(new SegmentDescriptor(publishedSegment.getInterval(), publishedSegment.getVersion(), publishedSegment.getShardSpec().getPartitionNum()), entry.getKey());
            executorRunnablePair.lhs.execute(executorRunnablePair.rhs);
        }
        handOffCallbacks.clear();
        // Wait for the task to finish.
        final TaskStatus taskStatus = statusFuture.get();
        Assert.assertEquals(TaskStatus.Status.SUCCESS, taskStatus.getStatusCode());
    }
}

Example 10 with TaskStatus

use of io.druid.indexing.common.TaskStatus in project druid by druid-io.

the class TaskLifecycleTest method runTask.

private TaskStatus runTask(final Task task) throws Exception {
    final Task dummyTask = new DefaultObjectMapper().readValue("{\"type\":\"noop\", \"isReadyResult\":\"exception\"}\"", Task.class);
    final long startTime = System.currentTimeMillis();
    Preconditions.checkArgument(!task.getId().equals(dummyTask.getId()));
    // Since multiple tasks can be run in a single unit test using runTask(), hence this check and synchronization
    synchronized (this) {
        if (!taskQueue.isActive()) {
            taskQueue.start();
        }
    }
    taskQueue.add(dummyTask);
    taskQueue.add(task);
    TaskStatus retVal = null;
    for (final String taskId : ImmutableList.of(dummyTask.getId(), task.getId())) {
        try {
            TaskStatus status;
            while ((status = tsqa.getStatus(taskId).get()).isRunnable()) {
                if (System.currentTimeMillis() > startTime + 10 * 1000) {
                    throw new ISE("Where did the task go?!: %s", task.getId());
                }
                Thread.sleep(100);
            }
            if (taskId.equals(task.getId())) {
                retVal = status;
            }
        } catch (Exception e) {
            throw Throwables.propagate(e);
        }
    }
    return retVal;
}