Examples with TaskLock io.druid.indexing.common.TaskLock used on opensource projects

Example 6 with TaskLock

use of io.druid.indexing.common.TaskLock in project druid by druid-io.

the class TaskLockbox method unlock.

/**
   * Release lock held for a task on a particular interval. Does nothing if the task does not currently
   * hold the mentioned lock.
   *
   * @param task task to unlock
   * @param interval interval to unlock
   */
public void unlock(final Task task, final Interval interval) {
    giant.lock();
    try {
        final String dataSource = task.getDataSource();
        final NavigableMap<Interval, TaskLockPosse> dsRunning = running.get(dataSource);
        // So we can alert if activeTasks try to release stuff they don't have
        boolean removed = false;
        if (dsRunning != null) {
            final TaskLockPosse taskLockPosse = dsRunning.get(interval);
            if (taskLockPosse != null) {
                final TaskLock taskLock = taskLockPosse.getTaskLock();
                // Remove task from live list
                log.info("Removing task[%s] from TaskLock[%s]", task.getId(), taskLock.getGroupId());
                removed = taskLockPosse.getTaskIds().remove(task.getId());
                if (taskLockPosse.getTaskIds().isEmpty()) {
                    log.info("TaskLock is now empty: %s", taskLock);
                    running.get(dataSource).remove(taskLock.getInterval());
                }
                if (running.get(dataSource).size() == 0) {
                    running.remove(dataSource);
                }
                // Wake up blocking-lock waiters
                lockReleaseCondition.signalAll();
                // Remove lock from storage. If it cannot be removed, just ignore the failure.
                try {
                    taskStorage.removeLock(task.getId(), taskLock);
                } catch (Exception e) {
                    log.makeAlert(e, "Failed to clean up lock from storage").addData("task", task.getId()).addData("dataSource", taskLock.getDataSource()).addData("interval", taskLock.getInterval()).addData("version", taskLock.getVersion()).emit();
                }
            }
        }
        if (!removed) {
            log.makeAlert("Lock release without acquire").addData("task", task.getId()).addData("interval", interval).emit();
        }
    } finally {
        giant.unlock();
    }
}

Example 7 with TaskLock

use of io.druid.indexing.common.TaskLock in project druid by druid-io.

the class IndexTaskTest method runTask.

private final List<DataSegment> runTask(final IndexTask indexTask) throws Exception {
    final List<DataSegment> segments = Lists.newArrayList();
    indexTask.run(new TaskToolbox(null, null, new TaskActionClient() {

        @Override
        public <RetType> RetType submit(TaskAction<RetType> taskAction) throws IOException {
            if (taskAction instanceof LockListAction) {
                return (RetType) Arrays.asList(new TaskLock("", "", null, new DateTime().toString()));
            }
            if (taskAction instanceof LockAcquireAction) {
                return (RetType) new TaskLock("groupId", "test", ((LockAcquireAction) taskAction).getInterval(), new DateTime().toString());
            }
            if (taskAction instanceof SegmentTransactionalInsertAction) {
                return (RetType) new SegmentPublishResult(((SegmentTransactionalInsertAction) taskAction).getSegments(), true);
            }
            if (taskAction instanceof SegmentAllocateAction) {
                SegmentAllocateAction action = (SegmentAllocateAction) taskAction;
                Interval interval = action.getPreferredSegmentGranularity().bucket(action.getTimestamp());
                ShardSpec shardSpec = new NumberedShardSpec(segmentAllocatePartitionCounter++, 0);
                return (RetType) new SegmentIdentifier(action.getDataSource(), interval, "latestVersion", shardSpec);
            }
            return null;
        }
    }, null, new DataSegmentPusher() {

        @Deprecated
        @Override
        public String getPathForHadoop(String dataSource) {
            return getPathForHadoop();
        }

        @Override
        public String getPathForHadoop() {
            return null;
        }

        @Override
        public DataSegment push(File file, DataSegment segment) throws IOException {
            segments.add(segment);
            return segment;
        }
    }, null, null, null, null, null, null, null, null, null, jsonMapper, temporaryFolder.newFolder(), indexMerger, indexIO, null, null, indexMergerV9));
    Collections.sort(segments);
    return segments;
}

Example 8 with TaskLock

use of io.druid.indexing.common.TaskLock in project druid by druid-io.

the class SegmentAllocateActionTest method testManySegmentsSameInterval.

@Test
public void testManySegmentsSameInterval() throws Exception {
    final Task task = new NoopTask(null, 0, 0, null, null, null);
    taskActionTestKit.getTaskLockbox().add(task);
    final SegmentIdentifier id1 = allocate(task, PARTY_TIME, Granularities.NONE, Granularities.HOUR, "s1", null);
    final SegmentIdentifier id2 = allocate(task, PARTY_TIME, Granularities.NONE, Granularities.HOUR, "s1", id1.getIdentifierAsString());
    final SegmentIdentifier id3 = allocate(task, PARTY_TIME, Granularities.NONE, Granularities.HOUR, "s1", id2.getIdentifierAsString());
    final TaskLock partyLock = Iterables.getOnlyElement(FluentIterable.from(taskActionTestKit.getTaskLockbox().findLocksForTask(task)).filter(new Predicate<TaskLock>() {

        @Override
        public boolean apply(TaskLock input) {
            return input.getInterval().contains(PARTY_TIME);
        }
    }));
    assertSameIdentifier(id1, new SegmentIdentifier(DATA_SOURCE, Granularities.HOUR.bucket(PARTY_TIME), partyLock.getVersion(), new NumberedShardSpec(0, 0)));
    assertSameIdentifier(id2, new SegmentIdentifier(DATA_SOURCE, Granularities.HOUR.bucket(PARTY_TIME), partyLock.getVersion(), new NumberedShardSpec(1, 0)));
    assertSameIdentifier(id3, new SegmentIdentifier(DATA_SOURCE, Granularities.HOUR.bucket(PARTY_TIME), partyLock.getVersion(), new NumberedShardSpec(2, 0)));
}

Example 9 with TaskLock

use of io.druid.indexing.common.TaskLock in project druid by druid-io.

the class TaskLifecycleTest method testBadInterval.

@Test
public void testBadInterval() throws Exception {
    final Task task = new AbstractFixedIntervalTask("id1", "id1", "ds", new Interval("2012-01-01/P1D"), null) {

        @Override
        public String getType() {
            return "test";
        }

        @Override
        public TaskStatus run(TaskToolbox toolbox) throws Exception {
            final TaskLock myLock = Iterables.getOnlyElement(toolbox.getTaskActionClient().submit(new LockListAction()));
            final DataSegment segment = DataSegment.builder().dataSource("ds").interval(new Interval("2012-01-01/P2D")).version(myLock.getVersion()).build();
            toolbox.getTaskActionClient().submit(new SegmentInsertAction(ImmutableSet.of(segment)));
            return TaskStatus.success(getId());
        }
    };
    final TaskStatus status = runTask(task);
    Assert.assertEquals("statusCode", TaskStatus.Status.FAILED, status.getStatusCode());
    Assert.assertEquals("segments published", 0, mdc.getPublished().size());
    Assert.assertEquals("segments nuked", 0, mdc.getNuked().size());
}

Example 10 with TaskLock

use of io.druid.indexing.common.TaskLock in project druid by druid-io.

the class TaskLifecycleTest method testBadVersion.

@Test
public void testBadVersion() throws Exception {
    final Task task = new AbstractFixedIntervalTask("id1", "id1", "ds", new Interval("2012-01-01/P1D"), null) {

        @Override
        public String getType() {
            return "test";
        }

        @Override
        public TaskStatus run(TaskToolbox toolbox) throws Exception {
            final TaskLock myLock = Iterables.getOnlyElement(toolbox.getTaskActionClient().submit(new LockListAction()));
            final DataSegment segment = DataSegment.builder().dataSource("ds").interval(new Interval("2012-01-01/P1D")).version(myLock.getVersion() + "1!!!1!!").build();
            toolbox.getTaskActionClient().submit(new SegmentInsertAction(ImmutableSet.of(segment)));
            return TaskStatus.success(getId());
        }
    };
    final TaskStatus status = runTask(task);
    Assert.assertEquals("statusCode", TaskStatus.Status.FAILED, status.getStatusCode());
    Assert.assertEquals("segments published", 0, mdc.getPublished().size());
    Assert.assertEquals("segments nuked", 0, mdc.getNuked().size());
}