Examples with SegmentLock org.apache.druid.indexing.common.SegmentLock used on opensource projects

Example 6 with SegmentLock

use of org.apache.druid.indexing.common.SegmentLock in project druid by druid-io.

the class TaskLockbox method verifyAndCreateOrFindLockPosse.

/**
 * This method is called only in {@link #syncFromStorage()} and verifies the given task and the taskLock have the same
 * groupId, dataSource, and priority.
 */
private TaskLockPosse verifyAndCreateOrFindLockPosse(Task task, TaskLock taskLock) {
    giant.lock();
    try {
        Preconditions.checkArgument(task.getGroupId().equals(taskLock.getGroupId()), "lock groupId[%s] is different from task groupId[%s]", taskLock.getGroupId(), task.getGroupId());
        Preconditions.checkArgument(task.getDataSource().equals(taskLock.getDataSource()), "lock dataSource[%s] is different from task dataSource[%s]", taskLock.getDataSource(), task.getDataSource());
        final int taskPriority = task.getPriority();
        final int lockPriority = taskLock.getNonNullPriority();
        Preconditions.checkArgument(lockPriority == taskPriority, "lock priority[%s] is different from task priority[%s]", lockPriority, taskPriority);
        final LockRequest request;
        switch(taskLock.getGranularity()) {
            case SEGMENT:
                final SegmentLock segmentLock = (SegmentLock) taskLock;
                request = new SpecificSegmentLockRequest(segmentLock.getType(), segmentLock.getGroupId(), segmentLock.getDataSource(), segmentLock.getInterval(), segmentLock.getVersion(), segmentLock.getPartitionId(), taskPriority, segmentLock.isRevoked());
                break;
            case TIME_CHUNK:
                final TimeChunkLock timeChunkLock = (TimeChunkLock) taskLock;
                request = new TimeChunkLockRequest(timeChunkLock.getType(), timeChunkLock.getGroupId(), timeChunkLock.getDataSource(), timeChunkLock.getInterval(), timeChunkLock.getVersion(), taskPriority, timeChunkLock.isRevoked());
                break;
            default:
                throw new ISE("Unknown lockGranularity[%s]", taskLock.getGranularity());
        }
        return createOrFindLockPosse(request);
    } finally {
        giant.unlock();
    }
}

Example 7 with SegmentLock

use of org.apache.druid.indexing.common.SegmentLock in project druid by druid-io.

the class TaskLockbox method unlock.

/**
 * Release lock held for a task on a particular interval. Does nothing if the task does not currently
 * hold the mentioned lock.
 *
 * @param task task to unlock
 * @param interval interval to unlock
 */
public void unlock(final Task task, final Interval interval, @Nullable Integer partitionId) {
    giant.lock();
    try {
        final String dataSource = task.getDataSource();
        final NavigableMap<DateTime, SortedMap<Interval, List<TaskLockPosse>>> dsRunning = running.get(task.getDataSource());
        if (dsRunning == null || dsRunning.isEmpty()) {
            return;
        }
        final SortedMap<Interval, List<TaskLockPosse>> intervalToPosses = dsRunning.get(interval.getStart());
        if (intervalToPosses == null || intervalToPosses.isEmpty()) {
            return;
        }
        final List<TaskLockPosse> possesHolder = intervalToPosses.get(interval);
        if (possesHolder == null || possesHolder.isEmpty()) {
            return;
        }
        final List<TaskLockPosse> posses = possesHolder.stream().filter(posse -> posse.containsTask(task)).collect(Collectors.toList());
        for (TaskLockPosse taskLockPosse : posses) {
            final TaskLock taskLock = taskLockPosse.getTaskLock();
            final boolean match = (partitionId == null && taskLock.getGranularity() == LockGranularity.TIME_CHUNK) || (partitionId != null && taskLock.getGranularity() == LockGranularity.SEGMENT && ((SegmentLock) taskLock).getPartitionId() == partitionId);
            if (match) {
                // Remove task from live list
                log.info("Removing task[%s] from TaskLock[%s]", task.getId(), taskLock);
                final boolean removed = taskLockPosse.removeTask(task);
                if (taskLockPosse.isTasksEmpty()) {
                    log.info("TaskLock is now empty: %s", taskLock);
                    possesHolder.remove(taskLockPosse);
                }
                if (possesHolder.isEmpty()) {
                    intervalToPosses.remove(interval);
                }
                if (intervalToPosses.isEmpty()) {
                    dsRunning.remove(interval.getStart());
                }
                if (running.get(dataSource).size() == 0) {
                    running.remove(dataSource);
                }
                // Wake up blocking-lock waiters
                lockReleaseCondition.signalAll();
                // Remove lock from storage. If it cannot be removed, just ignore the failure.
                try {
                    taskStorage.removeLock(task.getId(), taskLock);
                } catch (Exception e) {
                    log.makeAlert(e, "Failed to clean up lock from storage").addData("task", task.getId()).addData("dataSource", taskLock.getDataSource()).addData("interval", taskLock.getInterval()).addData("version", taskLock.getVersion()).emit();
                }
                if (!removed) {
                    log.makeAlert("Lock release without acquire").addData("task", task.getId()).addData("interval", interval).emit();
                }
            }
        }
    } finally {
        giant.unlock();
    }
}

Example 8 with SegmentLock

use of org.apache.druid.indexing.common.SegmentLock in project druid by druid-io.

the class TaskLocks method isLockCoversSegments.

public static boolean isLockCoversSegments(NavigableMap<DateTime, List<TaskLock>> taskLockMap, Collection<DataSegment> segments) {
    return segments.stream().allMatch(segment -> {
        final Entry<DateTime, List<TaskLock>> entry = taskLockMap.floorEntry(segment.getInterval().getStart());
        if (entry == null) {
            return false;
        }
        final List<TaskLock> locks = entry.getValue();
        return locks.stream().anyMatch(lock -> {
            if (lock.getGranularity() == LockGranularity.TIME_CHUNK) {
                final TimeChunkLock timeChunkLock = (TimeChunkLock) lock;
                return timeChunkLock.getInterval().contains(segment.getInterval()) && timeChunkLock.getDataSource().equals(segment.getDataSource()) && timeChunkLock.getVersion().compareTo(segment.getVersion()) >= 0;
            } else {
                final SegmentLock segmentLock = (SegmentLock) lock;
                return segmentLock.getInterval().contains(segment.getInterval()) && segmentLock.getDataSource().equals(segment.getDataSource()) && segmentLock.getVersion().compareTo(segment.getVersion()) >= 0 && segmentLock.getPartitionId() == segment.getShardSpec().getPartitionNum();
            }
        });
    });
}

Example 9 with SegmentLock

use of org.apache.druid.indexing.common.SegmentLock in project druid by druid-io.

the class TaskLockboxTest method assertAllocatedSegments.

private void assertAllocatedSegments(LockRequestForNewSegment lockRequest, LockResult result) {
    Assert.assertTrue(result.isOk());
    Assert.assertNotNull(result.getTaskLock());
    Assert.assertTrue(result.getTaskLock() instanceof SegmentLock);
    Assert.assertNotNull(result.getNewSegmentId());
    final SegmentLock segmentLock = (SegmentLock) result.getTaskLock();
    final SegmentIdWithShardSpec segmentId = result.getNewSegmentId();
    Assert.assertEquals(lockRequest.getType(), segmentLock.getType());
    Assert.assertEquals(lockRequest.getGroupId(), segmentLock.getGroupId());
    Assert.assertEquals(lockRequest.getDataSource(), segmentLock.getDataSource());
    Assert.assertEquals(lockRequest.getInterval(), segmentLock.getInterval());
    Assert.assertEquals(lockRequest.getPartialShardSpec().getShardSpecClass(), segmentId.getShardSpec().getClass());
    Assert.assertEquals(lockRequest.getPriority(), lockRequest.getPriority());
}