Examples with TaskLock io.druid.indexing.common.TaskLock used on opensource projects

Example 1 with TaskLock

use of io.druid.indexing.common.TaskLock in project druid by druid-io.

the class MetadataTaskStorage method removeLock.

@Override
public void removeLock(String taskid, TaskLock taskLockToRemove) {
    Preconditions.checkNotNull(taskid, "taskid");
    Preconditions.checkNotNull(taskLockToRemove, "taskLockToRemove");
    final Map<Long, TaskLock> taskLocks = getLocksWithIds(taskid);
    for (final Map.Entry<Long, TaskLock> taskLockWithId : taskLocks.entrySet()) {
        final long id = taskLockWithId.getKey();
        final TaskLock taskLock = taskLockWithId.getValue();
        if (taskLock.equals(taskLockToRemove)) {
            log.info("Deleting TaskLock with id[%d]: %s", id, taskLock);
            handler.removeLock(id);
        }
    }
}

Example 2 with TaskLock

use of io.druid.indexing.common.TaskLock in project druid by druid-io.

the class SameIntervalMergeTaskTest method runTask.

private List<DataSegment> runTask(final SameIntervalMergeTask mergeTask, final String version) throws Exception {
    boolean isReady = mergeTask.isReady(new TaskActionClient() {

        @Override
        public <RetType> RetType submit(TaskAction<RetType> taskAction) throws IOException {
            if (taskAction instanceof LockTryAcquireAction) {
                // the lock of this interval is required
                Assert.assertEquals(mergeTask.getInterval(), ((LockTryAcquireAction) taskAction).getInterval());
                isRedayCountDown.countDown();
                taskLock = new TaskLock(mergeTask.getGroupId(), mergeTask.getDataSource(), mergeTask.getInterval(), version);
                return (RetType) taskLock;
            }
            return null;
        }
    });
    // ensure LockTryAcquireAction is submitted
    Assert.assertTrue(isReady);
    final List<DataSegment> segments = Lists.newArrayList();
    mergeTask.run(new TaskToolbox(null, null, new TaskActionClient() {

        @Override
        public <RetType> RetType submit(TaskAction<RetType> taskAction) throws IOException {
            if (taskAction instanceof LockListAction) {
                Assert.assertNotNull("taskLock should be acquired before list", taskLock);
                return (RetType) Arrays.asList(taskLock);
            }
            if (taskAction instanceof SegmentListUsedAction) {
                List<DataSegment> segments = ImmutableList.of(DataSegment.builder().dataSource(mergeTask.getDataSource()).interval(new Interval("2010-01-01/PT1H")).version("oldVersion").shardSpec(new LinearShardSpec(0)).build(), DataSegment.builder().dataSource(mergeTask.getDataSource()).interval(new Interval("2010-01-01/PT1H")).version("oldVersion").shardSpec(new LinearShardSpec(0)).build(), DataSegment.builder().dataSource(mergeTask.getDataSource()).interval(new Interval("2010-01-01/PT2H")).version("oldVersion").shardSpec(new LinearShardSpec(0)).build());
                return (RetType) segments;
            }
            if (taskAction instanceof SegmentInsertAction) {
                publishCountDown.countDown();
                return null;
            }
            return null;
        }
    }, new NoopServiceEmitter(), new DataSegmentPusher() {

        @Deprecated
        @Override
        public String getPathForHadoop(String dataSource) {
            return getPathForHadoop();
        }

        @Override
        public String getPathForHadoop() {
            return null;
        }

        @Override
        public DataSegment push(File file, DataSegment segment) throws IOException {
            // the merged segment is pushed to storage
            segments.add(segment);
            return segment;
        }
    }, null, null, null, null, null, null, null, null, new SegmentLoader() {

        @Override
        public boolean isSegmentLoaded(DataSegment segment) throws SegmentLoadingException {
            return false;
        }

        @Override
        public Segment getSegment(DataSegment segment) throws SegmentLoadingException {
            return null;
        }

        @Override
        public File getSegmentFiles(DataSegment segment) throws SegmentLoadingException {
            // dummy file to represent the downloaded segment's dir
            return new File("" + segment.getShardSpec().getPartitionNum());
        }

        @Override
        public void cleanup(DataSegment segment) throws SegmentLoadingException {
        }
    }, jsonMapper, temporaryFolder.newFolder(), EasyMock.createMock(IndexMerger.class), indexIO, null, null, EasyMock.createMock(IndexMergerV9.class)));
    return segments;
}

Example 3 with TaskLock

use of io.druid.indexing.common.TaskLock in project druid by druid-io.

the class RemoteTaskActionClientTest method setUp.

@Before
public void setUp() {
    httpClient = createMock(HttpClient.class);
    selector = createMock(ServerDiscoverySelector.class);
    server = new Server() {

        @Override
        public String getScheme() {
            return "http";
        }

        @Override
        public int getPort() {
            return 8080;
        }

        @Override
        public String getHost() {
            return "localhost";
        }

        @Override
        public String getAddress() {
            return "localhost";
        }
    };
    long now = System.currentTimeMillis();
    result = Arrays.asList(new TaskLock("groupId", "dataSource", new Interval(now - 30 * 1000, now), "version"));
}

Example 4 with TaskLock

use of io.druid.indexing.common.TaskLock in project druid by druid-io.

the class SegmentAllocateActionTest method testResumeSequence.

@Test
public void testResumeSequence() throws Exception {
    final Task task = new NoopTask(null, 0, 0, null, null, null);
    taskActionTestKit.getTaskLockbox().add(task);
    final SegmentIdentifier id1 = allocate(task, PARTY_TIME, Granularities.NONE, Granularities.HOUR, "s1", null);
    final SegmentIdentifier id2 = allocate(task, THE_DISTANT_FUTURE, Granularities.NONE, Granularities.HOUR, "s1", id1.getIdentifierAsString());
    final SegmentIdentifier id3 = allocate(task, PARTY_TIME, Granularities.NONE, Granularities.HOUR, "s1", id2.getIdentifierAsString());
    final SegmentIdentifier id4 = allocate(task, PARTY_TIME, Granularities.NONE, Granularities.HOUR, "s1", id1.getIdentifierAsString());
    final SegmentIdentifier id5 = allocate(task, THE_DISTANT_FUTURE, Granularities.NONE, Granularities.HOUR, "s1", id1.getIdentifierAsString());
    final SegmentIdentifier id6 = allocate(task, THE_DISTANT_FUTURE, Granularities.NONE, Granularities.MINUTE, "s1", id1.getIdentifierAsString());
    final SegmentIdentifier id7 = allocate(task, THE_DISTANT_FUTURE, Granularities.NONE, Granularities.DAY, "s1", id1.getIdentifierAsString());
    final TaskLock partyLock = Iterables.getOnlyElement(FluentIterable.from(taskActionTestKit.getTaskLockbox().findLocksForTask(task)).filter(new Predicate<TaskLock>() {

        @Override
        public boolean apply(TaskLock input) {
            return input.getInterval().contains(PARTY_TIME);
        }
    }));
    final TaskLock futureLock = Iterables.getOnlyElement(FluentIterable.from(taskActionTestKit.getTaskLockbox().findLocksForTask(task)).filter(new Predicate<TaskLock>() {

        @Override
        public boolean apply(TaskLock input) {
            return input.getInterval().contains(THE_DISTANT_FUTURE);
        }
    }));
    assertSameIdentifier(id1, new SegmentIdentifier(DATA_SOURCE, Granularities.HOUR.bucket(PARTY_TIME), partyLock.getVersion(), new NumberedShardSpec(0, 0)));
    assertSameIdentifier(id2, new SegmentIdentifier(DATA_SOURCE, Granularities.HOUR.bucket(THE_DISTANT_FUTURE), futureLock.getVersion(), new NumberedShardSpec(0, 0)));
    assertSameIdentifier(id3, new SegmentIdentifier(DATA_SOURCE, Granularities.HOUR.bucket(PARTY_TIME), partyLock.getVersion(), new NumberedShardSpec(1, 0)));
    Assert.assertNull(id4);
    assertSameIdentifier(id5, id2);
    Assert.assertNull(id6);
    assertSameIdentifier(id7, id2);
}

Example 5 with TaskLock

use of io.druid.indexing.common.TaskLock in project druid by druid-io.

the class TaskLockbox method syncFromStorage.

/**
   * Wipe out our current in-memory state and resync it from our bundled {@link io.druid.indexing.overlord.TaskStorage}.
   */
public void syncFromStorage() {
    giant.lock();
    try {
        // Load stuff from taskStorage first. If this fails, we don't want to lose all our locks.
        final Set<String> storedActiveTasks = Sets.newHashSet();
        final List<Pair<Task, TaskLock>> storedLocks = Lists.newArrayList();
        for (final Task task : taskStorage.getActiveTasks()) {
            storedActiveTasks.add(task.getId());
            for (final TaskLock taskLock : taskStorage.getLocks(task.getId())) {
                storedLocks.add(Pair.of(task, taskLock));
            }
        }
        // Sort locks by version, so we add them back in the order they were acquired.
        final Ordering<Pair<Task, TaskLock>> byVersionOrdering = new Ordering<Pair<Task, TaskLock>>() {

            @Override
            public int compare(Pair<Task, TaskLock> left, Pair<Task, TaskLock> right) {
                // The second compare shouldn't be necessary, but, whatever.
                return ComparisonChain.start().compare(left.rhs.getVersion(), right.rhs.getVersion()).compare(left.lhs.getId(), right.lhs.getId()).result();
            }
        };
        running.clear();
        activeTasks.clear();
        activeTasks.addAll(storedActiveTasks);
        // Bookkeeping for a log message at the end
        int taskLockCount = 0;
        for (final Pair<Task, TaskLock> taskAndLock : byVersionOrdering.sortedCopy(storedLocks)) {
            final Task task = taskAndLock.lhs;
            final TaskLock savedTaskLock = taskAndLock.rhs;
            if (savedTaskLock.getInterval().toDurationMillis() <= 0) {
                // "Impossible", but you never know what crazy stuff can be restored from storage.
                log.warn("WTF?! Got lock with empty interval for task: %s", task.getId());
                continue;
            }
            final Optional<TaskLock> acquiredTaskLock = tryLock(task, savedTaskLock.getInterval(), Optional.of(savedTaskLock.getVersion()));
            if (acquiredTaskLock.isPresent() && savedTaskLock.getVersion().equals(acquiredTaskLock.get().getVersion())) {
                taskLockCount++;
                log.info("Reacquired lock on interval[%s] version[%s] for task: %s", savedTaskLock.getInterval(), savedTaskLock.getVersion(), task.getId());
            } else if (acquiredTaskLock.isPresent()) {
                taskLockCount++;
                log.info("Could not reacquire lock on interval[%s] version[%s] (got version[%s] instead) for task: %s", savedTaskLock.getInterval(), savedTaskLock.getVersion(), acquiredTaskLock.get().getVersion(), task.getId());
            } else {
                log.info("Could not reacquire lock on interval[%s] version[%s] for task: %s", savedTaskLock.getInterval(), savedTaskLock.getVersion(), task.getId());
            }
        }
        log.info("Synced %,d locks for %,d activeTasks from storage (%,d locks ignored).", taskLockCount, activeTasks.size(), storedLocks.size() - taskLockCount);
    } finally {
        giant.unlock();
    }
}