Search in sources :

Example 6 with TaskToolbox

use of io.druid.indexing.common.TaskToolbox in project druid by druid-io.

the class IndexTaskTest method runTask.

private final List<DataSegment> runTask(final IndexTask indexTask) throws Exception {
    final List<DataSegment> segments = Lists.newArrayList();
    indexTask.run(new TaskToolbox(null, null, new TaskActionClient() {

        @Override
        public <RetType> RetType submit(TaskAction<RetType> taskAction) throws IOException {
            if (taskAction instanceof LockListAction) {
                return (RetType) Arrays.asList(new TaskLock("", "", null, new DateTime().toString()));
            }
            if (taskAction instanceof LockAcquireAction) {
                return (RetType) new TaskLock("groupId", "test", ((LockAcquireAction) taskAction).getInterval(), new DateTime().toString());
            }
            if (taskAction instanceof SegmentTransactionalInsertAction) {
                return (RetType) new SegmentPublishResult(((SegmentTransactionalInsertAction) taskAction).getSegments(), true);
            }
            if (taskAction instanceof SegmentAllocateAction) {
                SegmentAllocateAction action = (SegmentAllocateAction) taskAction;
                Interval interval = action.getPreferredSegmentGranularity().bucket(action.getTimestamp());
                ShardSpec shardSpec = new NumberedShardSpec(segmentAllocatePartitionCounter++, 0);
                return (RetType) new SegmentIdentifier(action.getDataSource(), interval, "latestVersion", shardSpec);
            }
            return null;
        }
    }, null, new DataSegmentPusher() {

        @Deprecated
        @Override
        public String getPathForHadoop(String dataSource) {
            return getPathForHadoop();
        }

        @Override
        public String getPathForHadoop() {
            return null;
        }

        @Override
        public DataSegment push(File file, DataSegment segment) throws IOException {
            segments.add(segment);
            return segment;
        }
    }, null, null, null, null, null, null, null, null, null, jsonMapper, temporaryFolder.newFolder(), indexMerger, indexIO, null, null, indexMergerV9));
    Collections.sort(segments);
    return segments;
}
Also used : LockListAction(io.druid.indexing.common.actions.LockListAction) DataSegmentPusher(io.druid.segment.loading.DataSegmentPusher) SegmentIdentifier(io.druid.segment.realtime.appenderator.SegmentIdentifier) TaskAction(io.druid.indexing.common.actions.TaskAction) SegmentTransactionalInsertAction(io.druid.indexing.common.actions.SegmentTransactionalInsertAction) DataSegment(io.druid.timeline.DataSegment) DateTime(org.joda.time.DateTime) NoneShardSpec(io.druid.timeline.partition.NoneShardSpec) ShardSpec(io.druid.timeline.partition.ShardSpec) NumberedShardSpec(io.druid.timeline.partition.NumberedShardSpec) HashBasedNumberedShardSpec(io.druid.timeline.partition.HashBasedNumberedShardSpec) TaskToolbox(io.druid.indexing.common.TaskToolbox) SegmentPublishResult(io.druid.indexing.overlord.SegmentPublishResult) TaskActionClient(io.druid.indexing.common.actions.TaskActionClient) TaskLock(io.druid.indexing.common.TaskLock) SegmentAllocateAction(io.druid.indexing.common.actions.SegmentAllocateAction) LockAcquireAction(io.druid.indexing.common.actions.LockAcquireAction) File(java.io.File) NumberedShardSpec(io.druid.timeline.partition.NumberedShardSpec) HashBasedNumberedShardSpec(io.druid.timeline.partition.HashBasedNumberedShardSpec) Interval(org.joda.time.Interval)

Example 7 with TaskToolbox

use of io.druid.indexing.common.TaskToolbox in project druid by druid-io.

the class TaskLifecycleTest method testBadInterval.

@Test
public void testBadInterval() throws Exception {
    final Task task = new AbstractFixedIntervalTask("id1", "id1", "ds", new Interval("2012-01-01/P1D"), null) {

        @Override
        public String getType() {
            return "test";
        }

        @Override
        public TaskStatus run(TaskToolbox toolbox) throws Exception {
            final TaskLock myLock = Iterables.getOnlyElement(toolbox.getTaskActionClient().submit(new LockListAction()));
            final DataSegment segment = DataSegment.builder().dataSource("ds").interval(new Interval("2012-01-01/P2D")).version(myLock.getVersion()).build();
            toolbox.getTaskActionClient().submit(new SegmentInsertAction(ImmutableSet.of(segment)));
            return TaskStatus.success(getId());
        }
    };
    final TaskStatus status = runTask(task);
    Assert.assertEquals("statusCode", TaskStatus.Status.FAILED, status.getStatusCode());
    Assert.assertEquals("segments published", 0, mdc.getPublished().size());
    Assert.assertEquals("segments nuked", 0, mdc.getNuked().size());
}
Also used : TaskToolbox(io.druid.indexing.common.TaskToolbox) LockListAction(io.druid.indexing.common.actions.LockListAction) IndexTask(io.druid.indexing.common.task.IndexTask) RealtimeIndexTask(io.druid.indexing.common.task.RealtimeIndexTask) Task(io.druid.indexing.common.task.Task) AbstractFixedIntervalTask(io.druid.indexing.common.task.AbstractFixedIntervalTask) KillTask(io.druid.indexing.common.task.KillTask) TaskLock(io.druid.indexing.common.TaskLock) SegmentInsertAction(io.druid.indexing.common.actions.SegmentInsertAction) TaskStatus(io.druid.indexing.common.TaskStatus) DataSegment(io.druid.timeline.DataSegment) AbstractFixedIntervalTask(io.druid.indexing.common.task.AbstractFixedIntervalTask) Interval(org.joda.time.Interval) FireDepartmentTest(io.druid.segment.realtime.FireDepartmentTest) Test(org.junit.Test)

Example 8 with TaskToolbox

use of io.druid.indexing.common.TaskToolbox in project druid by druid-io.

the class TaskLifecycleTest method testBadVersion.

@Test
public void testBadVersion() throws Exception {
    final Task task = new AbstractFixedIntervalTask("id1", "id1", "ds", new Interval("2012-01-01/P1D"), null) {

        @Override
        public String getType() {
            return "test";
        }

        @Override
        public TaskStatus run(TaskToolbox toolbox) throws Exception {
            final TaskLock myLock = Iterables.getOnlyElement(toolbox.getTaskActionClient().submit(new LockListAction()));
            final DataSegment segment = DataSegment.builder().dataSource("ds").interval(new Interval("2012-01-01/P1D")).version(myLock.getVersion() + "1!!!1!!").build();
            toolbox.getTaskActionClient().submit(new SegmentInsertAction(ImmutableSet.of(segment)));
            return TaskStatus.success(getId());
        }
    };
    final TaskStatus status = runTask(task);
    Assert.assertEquals("statusCode", TaskStatus.Status.FAILED, status.getStatusCode());
    Assert.assertEquals("segments published", 0, mdc.getPublished().size());
    Assert.assertEquals("segments nuked", 0, mdc.getNuked().size());
}
Also used : TaskToolbox(io.druid.indexing.common.TaskToolbox) LockListAction(io.druid.indexing.common.actions.LockListAction) IndexTask(io.druid.indexing.common.task.IndexTask) RealtimeIndexTask(io.druid.indexing.common.task.RealtimeIndexTask) Task(io.druid.indexing.common.task.Task) AbstractFixedIntervalTask(io.druid.indexing.common.task.AbstractFixedIntervalTask) KillTask(io.druid.indexing.common.task.KillTask) TaskLock(io.druid.indexing.common.TaskLock) SegmentInsertAction(io.druid.indexing.common.actions.SegmentInsertAction) TaskStatus(io.druid.indexing.common.TaskStatus) DataSegment(io.druid.timeline.DataSegment) AbstractFixedIntervalTask(io.druid.indexing.common.task.AbstractFixedIntervalTask) Interval(org.joda.time.Interval) FireDepartmentTest(io.druid.segment.realtime.FireDepartmentTest) Test(org.junit.Test)

Example 9 with TaskToolbox

use of io.druid.indexing.common.TaskToolbox in project druid by druid-io.

the class RealtimeIndexTaskTest method testReportParseExceptionsOnBadMetric.

@Test(timeout = 60_000L)
public void testReportParseExceptionsOnBadMetric() throws Exception {
    final TestIndexerMetadataStorageCoordinator mdc = new TestIndexerMetadataStorageCoordinator();
    final RealtimeIndexTask task = makeRealtimeTask(null, true);
    final TaskToolbox taskToolbox = makeToolbox(task, mdc, tempFolder.newFolder());
    final ListenableFuture<TaskStatus> statusFuture = runTask(task, taskToolbox);
    // Wait for firehose to show up, it starts off null.
    while (task.getFirehose() == null) {
        Thread.sleep(50);
    }
    final TestFirehose firehose = (TestFirehose) task.getFirehose();
    firehose.addRows(ImmutableList.<InputRow>of(new MapBasedInputRow(now, ImmutableList.of("dim1"), ImmutableMap.<String, Object>of("dim1", "foo", "met1", "1")), new MapBasedInputRow(now, ImmutableList.of("dim1"), ImmutableMap.<String, Object>of("dim1", "foo", "met1", "foo")), new MapBasedInputRow(now.minus(new Period("P1D")), ImmutableList.of("dim1"), ImmutableMap.<String, Object>of("dim1", "foo", "met1", "foo")), new MapBasedInputRow(now, ImmutableList.of("dim2"), ImmutableMap.<String, Object>of("dim2", "bar", "met1", 2.0))));
    // Stop the firehose, this will drain out existing events.
    firehose.close();
    // Wait for the task to finish.
    expectedException.expect(ExecutionException.class);
    expectedException.expectCause(CoreMatchers.<Throwable>instanceOf(ParseException.class));
    expectedException.expectCause(ThrowableMessageMatcher.hasMessage(CoreMatchers.containsString("Encountered parse error for aggregator[met1]")));
    expectedException.expect(ThrowableCauseMatcher.hasCause(ThrowableCauseMatcher.hasCause(CoreMatchers.allOf(CoreMatchers.<Throwable>instanceOf(ParseException.class), ThrowableMessageMatcher.hasMessage(CoreMatchers.containsString("Unable to parse metrics[met1], value[foo]"))))));
    statusFuture.get();
}
Also used : TaskToolbox(io.druid.indexing.common.TaskToolbox) TestIndexerMetadataStorageCoordinator(io.druid.indexing.test.TestIndexerMetadataStorageCoordinator) Period(org.joda.time.Period) MapBasedInputRow(io.druid.data.input.MapBasedInputRow) ParseException(io.druid.java.util.common.parsers.ParseException) TaskStatus(io.druid.indexing.common.TaskStatus) Test(org.junit.Test)

Example 10 with TaskToolbox

use of io.druid.indexing.common.TaskToolbox in project druid by druid-io.

the class RealtimeIndexTaskTest method testRestoreAfterHandoffAttemptDuringShutdown.

@Test(timeout = 60_000L)
public void testRestoreAfterHandoffAttemptDuringShutdown() throws Exception {
    final TaskStorage taskStorage = new HeapMemoryTaskStorage(new TaskStorageConfig(null));
    final TestIndexerMetadataStorageCoordinator mdc = new TestIndexerMetadataStorageCoordinator();
    final File directory = tempFolder.newFolder();
    final RealtimeIndexTask task1 = makeRealtimeTask(null);
    final DataSegment publishedSegment;
    // First run:
    {
        final TaskToolbox taskToolbox = makeToolbox(task1, taskStorage, mdc, directory);
        final ListenableFuture<TaskStatus> statusFuture = runTask(task1, taskToolbox);
        // Wait for firehose to show up, it starts off null.
        while (task1.getFirehose() == null) {
            Thread.sleep(50);
        }
        final TestFirehose firehose = (TestFirehose) task1.getFirehose();
        firehose.addRows(ImmutableList.<InputRow>of(new MapBasedInputRow(now, ImmutableList.of("dim1"), ImmutableMap.<String, Object>of("dim1", "foo"))));
        // Stop the firehose, this will trigger a finishJob.
        firehose.close();
        // Wait for publish.
        while (mdc.getPublished().isEmpty()) {
            Thread.sleep(50);
        }
        publishedSegment = Iterables.getOnlyElement(mdc.getPublished());
        // Do a query.
        Assert.assertEquals(1, sumMetric(task1, "rows"));
        // Trigger graceful shutdown.
        task1.stopGracefully();
        // Wait for the task to finish. The status doesn't really matter.
        while (!statusFuture.isDone()) {
            Thread.sleep(50);
        }
    }
    // Second run:
    {
        final RealtimeIndexTask task2 = makeRealtimeTask(task1.getId());
        final TaskToolbox taskToolbox = makeToolbox(task2, taskStorage, mdc, directory);
        final ListenableFuture<TaskStatus> statusFuture = runTask(task2, taskToolbox);
        // Wait for firehose to show up, it starts off null.
        while (task2.getFirehose() == null) {
            Thread.sleep(50);
        }
        // Stop the firehose again, this will start another handoff.
        final TestFirehose firehose = (TestFirehose) task2.getFirehose();
        // Stop the firehose, this will trigger a finishJob.
        firehose.close();
        // publishedSegment is still published. No reason it shouldn't be.
        Assert.assertEquals(ImmutableSet.of(publishedSegment), mdc.getPublished());
        // Wait for a handoffCallback to show up.
        while (handOffCallbacks.isEmpty()) {
            Thread.sleep(50);
        }
        // Simulate handoff.
        for (Map.Entry<SegmentDescriptor, Pair<Executor, Runnable>> entry : handOffCallbacks.entrySet()) {
            final Pair<Executor, Runnable> executorRunnablePair = entry.getValue();
            Assert.assertEquals(new SegmentDescriptor(publishedSegment.getInterval(), publishedSegment.getVersion(), publishedSegment.getShardSpec().getPartitionNum()), entry.getKey());
            executorRunnablePair.lhs.execute(executorRunnablePair.rhs);
        }
        handOffCallbacks.clear();
        // Wait for the task to finish.
        final TaskStatus taskStatus = statusFuture.get();
        Assert.assertEquals(TaskStatus.Status.SUCCESS, taskStatus.getStatusCode());
    }
}
Also used : TaskStorageConfig(io.druid.indexing.common.config.TaskStorageConfig) HeapMemoryTaskStorage(io.druid.indexing.overlord.HeapMemoryTaskStorage) TaskStatus(io.druid.indexing.common.TaskStatus) DataSegment(io.druid.timeline.DataSegment) TaskToolbox(io.druid.indexing.common.TaskToolbox) Executor(java.util.concurrent.Executor) TaskStorage(io.druid.indexing.overlord.TaskStorage) HeapMemoryTaskStorage(io.druid.indexing.overlord.HeapMemoryTaskStorage) TestIndexerMetadataStorageCoordinator(io.druid.indexing.test.TestIndexerMetadataStorageCoordinator) SegmentDescriptor(io.druid.query.SegmentDescriptor) MapBasedInputRow(io.druid.data.input.MapBasedInputRow) InputRow(io.druid.data.input.InputRow) ListenableFuture(com.google.common.util.concurrent.ListenableFuture) MapBasedInputRow(io.druid.data.input.MapBasedInputRow) File(java.io.File) Pair(io.druid.java.util.common.Pair) Test(org.junit.Test)

Aggregations

TaskToolbox (io.druid.indexing.common.TaskToolbox)16 TaskStatus (io.druid.indexing.common.TaskStatus)13 Test (org.junit.Test)12 DataSegment (io.druid.timeline.DataSegment)9 TestIndexerMetadataStorageCoordinator (io.druid.indexing.test.TestIndexerMetadataStorageCoordinator)8 MapBasedInputRow (io.druid.data.input.MapBasedInputRow)7 File (java.io.File)6 TaskLock (io.druid.indexing.common.TaskLock)5 LockListAction (io.druid.indexing.common.actions.LockListAction)5 Interval (org.joda.time.Interval)5 SegmentInsertAction (io.druid.indexing.common.actions.SegmentInsertAction)4 Pair (io.druid.java.util.common.Pair)4 SegmentDescriptor (io.druid.query.SegmentDescriptor)4 ListenableFuture (com.google.common.util.concurrent.ListenableFuture)3 InputRow (io.druid.data.input.InputRow)3 TaskActionClient (io.druid.indexing.common.actions.TaskActionClient)3 AbstractFixedIntervalTask (io.druid.indexing.common.task.AbstractFixedIntervalTask)3 IndexTask (io.druid.indexing.common.task.IndexTask)3 KillTask (io.druid.indexing.common.task.KillTask)3 RealtimeIndexTask (io.druid.indexing.common.task.RealtimeIndexTask)3