Search in sources :

Example 1 with LockListAction

use of io.druid.indexing.common.actions.LockListAction in project druid by druid-io.

the class SameIntervalMergeTaskTest method runTask.

private List<DataSegment> runTask(final SameIntervalMergeTask mergeTask, final String version) throws Exception {
    boolean isReady = mergeTask.isReady(new TaskActionClient() {

        @Override
        public <RetType> RetType submit(TaskAction<RetType> taskAction) throws IOException {
            if (taskAction instanceof LockTryAcquireAction) {
                // the lock of this interval is required
                Assert.assertEquals(mergeTask.getInterval(), ((LockTryAcquireAction) taskAction).getInterval());
                isRedayCountDown.countDown();
                taskLock = new TaskLock(mergeTask.getGroupId(), mergeTask.getDataSource(), mergeTask.getInterval(), version);
                return (RetType) taskLock;
            }
            return null;
        }
    });
    // ensure LockTryAcquireAction is submitted
    Assert.assertTrue(isReady);
    final List<DataSegment> segments = Lists.newArrayList();
    mergeTask.run(new TaskToolbox(null, null, new TaskActionClient() {

        @Override
        public <RetType> RetType submit(TaskAction<RetType> taskAction) throws IOException {
            if (taskAction instanceof LockListAction) {
                Assert.assertNotNull("taskLock should be acquired before list", taskLock);
                return (RetType) Arrays.asList(taskLock);
            }
            if (taskAction instanceof SegmentListUsedAction) {
                List<DataSegment> segments = ImmutableList.of(DataSegment.builder().dataSource(mergeTask.getDataSource()).interval(new Interval("2010-01-01/PT1H")).version("oldVersion").shardSpec(new LinearShardSpec(0)).build(), DataSegment.builder().dataSource(mergeTask.getDataSource()).interval(new Interval("2010-01-01/PT1H")).version("oldVersion").shardSpec(new LinearShardSpec(0)).build(), DataSegment.builder().dataSource(mergeTask.getDataSource()).interval(new Interval("2010-01-01/PT2H")).version("oldVersion").shardSpec(new LinearShardSpec(0)).build());
                return (RetType) segments;
            }
            if (taskAction instanceof SegmentInsertAction) {
                publishCountDown.countDown();
                return null;
            }
            return null;
        }
    }, new NoopServiceEmitter(), new DataSegmentPusher() {

        @Deprecated
        @Override
        public String getPathForHadoop(String dataSource) {
            return getPathForHadoop();
        }

        @Override
        public String getPathForHadoop() {
            return null;
        }

        @Override
        public DataSegment push(File file, DataSegment segment) throws IOException {
            // the merged segment is pushed to storage
            segments.add(segment);
            return segment;
        }
    }, null, null, null, null, null, null, null, null, new SegmentLoader() {

        @Override
        public boolean isSegmentLoaded(DataSegment segment) throws SegmentLoadingException {
            return false;
        }

        @Override
        public Segment getSegment(DataSegment segment) throws SegmentLoadingException {
            return null;
        }

        @Override
        public File getSegmentFiles(DataSegment segment) throws SegmentLoadingException {
            // dummy file to represent the downloaded segment's dir
            return new File("" + segment.getShardSpec().getPartitionNum());
        }

        @Override
        public void cleanup(DataSegment segment) throws SegmentLoadingException {
        }
    }, jsonMapper, temporaryFolder.newFolder(), EasyMock.createMock(IndexMerger.class), indexIO, null, null, EasyMock.createMock(IndexMergerV9.class)));
    return segments;
}
Also used : LockListAction(io.druid.indexing.common.actions.LockListAction) DataSegmentPusher(io.druid.segment.loading.DataSegmentPusher) LockTryAcquireAction(io.druid.indexing.common.actions.LockTryAcquireAction) TaskAction(io.druid.indexing.common.actions.TaskAction) LinearShardSpec(io.druid.timeline.partition.LinearShardSpec) NoopServiceEmitter(io.druid.server.metrics.NoopServiceEmitter) IOException(java.io.IOException) DataSegment(io.druid.timeline.DataSegment) SegmentLoader(io.druid.segment.loading.SegmentLoader) TaskToolbox(io.druid.indexing.common.TaskToolbox) TaskActionClient(io.druid.indexing.common.actions.TaskActionClient) TaskLock(io.druid.indexing.common.TaskLock) SegmentInsertAction(io.druid.indexing.common.actions.SegmentInsertAction) SegmentListUsedAction(io.druid.indexing.common.actions.SegmentListUsedAction) File(java.io.File) Interval(org.joda.time.Interval)

Example 2 with LockListAction

use of io.druid.indexing.common.actions.LockListAction in project druid by druid-io.

the class IndexTaskTest method runTask.

private final List<DataSegment> runTask(final IndexTask indexTask) throws Exception {
    final List<DataSegment> segments = Lists.newArrayList();
    indexTask.run(new TaskToolbox(null, null, new TaskActionClient() {

        @Override
        public <RetType> RetType submit(TaskAction<RetType> taskAction) throws IOException {
            if (taskAction instanceof LockListAction) {
                return (RetType) Arrays.asList(new TaskLock("", "", null, new DateTime().toString()));
            }
            if (taskAction instanceof LockAcquireAction) {
                return (RetType) new TaskLock("groupId", "test", ((LockAcquireAction) taskAction).getInterval(), new DateTime().toString());
            }
            if (taskAction instanceof SegmentTransactionalInsertAction) {
                return (RetType) new SegmentPublishResult(((SegmentTransactionalInsertAction) taskAction).getSegments(), true);
            }
            if (taskAction instanceof SegmentAllocateAction) {
                SegmentAllocateAction action = (SegmentAllocateAction) taskAction;
                Interval interval = action.getPreferredSegmentGranularity().bucket(action.getTimestamp());
                ShardSpec shardSpec = new NumberedShardSpec(segmentAllocatePartitionCounter++, 0);
                return (RetType) new SegmentIdentifier(action.getDataSource(), interval, "latestVersion", shardSpec);
            }
            return null;
        }
    }, null, new DataSegmentPusher() {

        @Deprecated
        @Override
        public String getPathForHadoop(String dataSource) {
            return getPathForHadoop();
        }

        @Override
        public String getPathForHadoop() {
            return null;
        }

        @Override
        public DataSegment push(File file, DataSegment segment) throws IOException {
            segments.add(segment);
            return segment;
        }
    }, null, null, null, null, null, null, null, null, null, jsonMapper, temporaryFolder.newFolder(), indexMerger, indexIO, null, null, indexMergerV9));
    Collections.sort(segments);
    return segments;
}
Also used : LockListAction(io.druid.indexing.common.actions.LockListAction) DataSegmentPusher(io.druid.segment.loading.DataSegmentPusher) SegmentIdentifier(io.druid.segment.realtime.appenderator.SegmentIdentifier) TaskAction(io.druid.indexing.common.actions.TaskAction) SegmentTransactionalInsertAction(io.druid.indexing.common.actions.SegmentTransactionalInsertAction) DataSegment(io.druid.timeline.DataSegment) DateTime(org.joda.time.DateTime) NoneShardSpec(io.druid.timeline.partition.NoneShardSpec) ShardSpec(io.druid.timeline.partition.ShardSpec) NumberedShardSpec(io.druid.timeline.partition.NumberedShardSpec) HashBasedNumberedShardSpec(io.druid.timeline.partition.HashBasedNumberedShardSpec) TaskToolbox(io.druid.indexing.common.TaskToolbox) SegmentPublishResult(io.druid.indexing.overlord.SegmentPublishResult) TaskActionClient(io.druid.indexing.common.actions.TaskActionClient) TaskLock(io.druid.indexing.common.TaskLock) SegmentAllocateAction(io.druid.indexing.common.actions.SegmentAllocateAction) LockAcquireAction(io.druid.indexing.common.actions.LockAcquireAction) File(java.io.File) NumberedShardSpec(io.druid.timeline.partition.NumberedShardSpec) HashBasedNumberedShardSpec(io.druid.timeline.partition.HashBasedNumberedShardSpec) Interval(org.joda.time.Interval)

Example 3 with LockListAction

use of io.druid.indexing.common.actions.LockListAction in project druid by druid-io.

the class TaskLifecycleTest method testBadInterval.

@Test
public void testBadInterval() throws Exception {
    final Task task = new AbstractFixedIntervalTask("id1", "id1", "ds", new Interval("2012-01-01/P1D"), null) {

        @Override
        public String getType() {
            return "test";
        }

        @Override
        public TaskStatus run(TaskToolbox toolbox) throws Exception {
            final TaskLock myLock = Iterables.getOnlyElement(toolbox.getTaskActionClient().submit(new LockListAction()));
            final DataSegment segment = DataSegment.builder().dataSource("ds").interval(new Interval("2012-01-01/P2D")).version(myLock.getVersion()).build();
            toolbox.getTaskActionClient().submit(new SegmentInsertAction(ImmutableSet.of(segment)));
            return TaskStatus.success(getId());
        }
    };
    final TaskStatus status = runTask(task);
    Assert.assertEquals("statusCode", TaskStatus.Status.FAILED, status.getStatusCode());
    Assert.assertEquals("segments published", 0, mdc.getPublished().size());
    Assert.assertEquals("segments nuked", 0, mdc.getNuked().size());
}
Also used : TaskToolbox(io.druid.indexing.common.TaskToolbox) LockListAction(io.druid.indexing.common.actions.LockListAction) IndexTask(io.druid.indexing.common.task.IndexTask) RealtimeIndexTask(io.druid.indexing.common.task.RealtimeIndexTask) Task(io.druid.indexing.common.task.Task) AbstractFixedIntervalTask(io.druid.indexing.common.task.AbstractFixedIntervalTask) KillTask(io.druid.indexing.common.task.KillTask) TaskLock(io.druid.indexing.common.TaskLock) SegmentInsertAction(io.druid.indexing.common.actions.SegmentInsertAction) TaskStatus(io.druid.indexing.common.TaskStatus) DataSegment(io.druid.timeline.DataSegment) AbstractFixedIntervalTask(io.druid.indexing.common.task.AbstractFixedIntervalTask) Interval(org.joda.time.Interval) FireDepartmentTest(io.druid.segment.realtime.FireDepartmentTest) Test(org.junit.Test)

Example 4 with LockListAction

use of io.druid.indexing.common.actions.LockListAction in project druid by druid-io.

the class TaskLifecycleTest method testBadVersion.

@Test
public void testBadVersion() throws Exception {
    final Task task = new AbstractFixedIntervalTask("id1", "id1", "ds", new Interval("2012-01-01/P1D"), null) {

        @Override
        public String getType() {
            return "test";
        }

        @Override
        public TaskStatus run(TaskToolbox toolbox) throws Exception {
            final TaskLock myLock = Iterables.getOnlyElement(toolbox.getTaskActionClient().submit(new LockListAction()));
            final DataSegment segment = DataSegment.builder().dataSource("ds").interval(new Interval("2012-01-01/P1D")).version(myLock.getVersion() + "1!!!1!!").build();
            toolbox.getTaskActionClient().submit(new SegmentInsertAction(ImmutableSet.of(segment)));
            return TaskStatus.success(getId());
        }
    };
    final TaskStatus status = runTask(task);
    Assert.assertEquals("statusCode", TaskStatus.Status.FAILED, status.getStatusCode());
    Assert.assertEquals("segments published", 0, mdc.getPublished().size());
    Assert.assertEquals("segments nuked", 0, mdc.getNuked().size());
}
Also used : TaskToolbox(io.druid.indexing.common.TaskToolbox) LockListAction(io.druid.indexing.common.actions.LockListAction) IndexTask(io.druid.indexing.common.task.IndexTask) RealtimeIndexTask(io.druid.indexing.common.task.RealtimeIndexTask) Task(io.druid.indexing.common.task.Task) AbstractFixedIntervalTask(io.druid.indexing.common.task.AbstractFixedIntervalTask) KillTask(io.druid.indexing.common.task.KillTask) TaskLock(io.druid.indexing.common.TaskLock) SegmentInsertAction(io.druid.indexing.common.actions.SegmentInsertAction) TaskStatus(io.druid.indexing.common.TaskStatus) DataSegment(io.druid.timeline.DataSegment) AbstractFixedIntervalTask(io.druid.indexing.common.task.AbstractFixedIntervalTask) Interval(org.joda.time.Interval) FireDepartmentTest(io.druid.segment.realtime.FireDepartmentTest) Test(org.junit.Test)

Example 5 with LockListAction

use of io.druid.indexing.common.actions.LockListAction in project druid by druid-io.

the class TaskLifecycleTest method testSimple.

@Test
public void testSimple() throws Exception {
    final Task task = new AbstractFixedIntervalTask("id1", "id1", new TaskResource("id1", 1), "ds", new Interval("2012-01-01/P1D"), null) {

        @Override
        public String getType() {
            return "test";
        }

        @Override
        public TaskStatus run(TaskToolbox toolbox) throws Exception {
            final TaskLock myLock = Iterables.getOnlyElement(toolbox.getTaskActionClient().submit(new LockListAction()));
            final DataSegment segment = DataSegment.builder().dataSource("ds").interval(new Interval("2012-01-01/P1D")).version(myLock.getVersion()).build();
            toolbox.getTaskActionClient().submit(new SegmentInsertAction(ImmutableSet.of(segment)));
            return TaskStatus.success(getId());
        }
    };
    final TaskStatus status = runTask(task);
    Assert.assertEquals("statusCode", TaskStatus.Status.SUCCESS, status.getStatusCode());
    Assert.assertEquals("segments published", 1, mdc.getPublished().size());
    Assert.assertEquals("segments nuked", 0, mdc.getNuked().size());
}
Also used : TaskToolbox(io.druid.indexing.common.TaskToolbox) LockListAction(io.druid.indexing.common.actions.LockListAction) IndexTask(io.druid.indexing.common.task.IndexTask) RealtimeIndexTask(io.druid.indexing.common.task.RealtimeIndexTask) Task(io.druid.indexing.common.task.Task) AbstractFixedIntervalTask(io.druid.indexing.common.task.AbstractFixedIntervalTask) KillTask(io.druid.indexing.common.task.KillTask) TaskResource(io.druid.indexing.common.task.TaskResource) TaskLock(io.druid.indexing.common.TaskLock) SegmentInsertAction(io.druid.indexing.common.actions.SegmentInsertAction) TaskStatus(io.druid.indexing.common.TaskStatus) DataSegment(io.druid.timeline.DataSegment) AbstractFixedIntervalTask(io.druid.indexing.common.task.AbstractFixedIntervalTask) Interval(org.joda.time.Interval) FireDepartmentTest(io.druid.segment.realtime.FireDepartmentTest) Test(org.junit.Test)

Aggregations

TaskLock (io.druid.indexing.common.TaskLock)6 LockListAction (io.druid.indexing.common.actions.LockListAction)6 Interval (org.joda.time.Interval)6 TaskToolbox (io.druid.indexing.common.TaskToolbox)5 SegmentInsertAction (io.druid.indexing.common.actions.SegmentInsertAction)5 DataSegment (io.druid.timeline.DataSegment)5 TaskStatus (io.druid.indexing.common.TaskStatus)3 AbstractFixedIntervalTask (io.druid.indexing.common.task.AbstractFixedIntervalTask)3 IndexTask (io.druid.indexing.common.task.IndexTask)3 KillTask (io.druid.indexing.common.task.KillTask)3 RealtimeIndexTask (io.druid.indexing.common.task.RealtimeIndexTask)3 Task (io.druid.indexing.common.task.Task)3 FireDepartmentTest (io.druid.segment.realtime.FireDepartmentTest)3 Test (org.junit.Test)3 LockAcquireAction (io.druid.indexing.common.actions.LockAcquireAction)2 TaskAction (io.druid.indexing.common.actions.TaskAction)2 TaskActionClient (io.druid.indexing.common.actions.TaskActionClient)2 DataSegmentPusher (io.druid.segment.loading.DataSegmentPusher)2 File (java.io.File)2 LockReleaseAction (io.druid.indexing.common.actions.LockReleaseAction)1