Examples with TimeChunkLock org.apache.druid.indexing.common.TimeChunkLock used on opensource projects

Example 1 with TimeChunkLock

use of org.apache.druid.indexing.common.TimeChunkLock in project druid by druid-io.

the class TaskLocks method findLocksForSegments.

public static List<TaskLock> findLocksForSegments(final Task task, final TaskLockbox taskLockbox, final Collection<DataSegment> segments) {
    final NavigableMap<DateTime, List<TaskLock>> taskLockMap = getTaskLockMap(taskLockbox, task);
    if (taskLockMap.isEmpty()) {
        return Collections.emptyList();
    }
    final List<TaskLock> found = new ArrayList<>();
    segments.forEach(segment -> {
        final Entry<DateTime, List<TaskLock>> entry = taskLockMap.floorEntry(segment.getInterval().getStart());
        if (entry == null) {
            throw new ISE("Can't find lock for the interval of segment[%s]", segment.getId());
        }
        final List<TaskLock> locks = entry.getValue();
        locks.forEach(lock -> {
            if (lock.getGranularity() == LockGranularity.TIME_CHUNK) {
                final TimeChunkLock timeChunkLock = (TimeChunkLock) lock;
                if (timeChunkLock.getInterval().contains(segment.getInterval()) && timeChunkLock.getDataSource().equals(segment.getDataSource()) && timeChunkLock.getVersion().compareTo(segment.getVersion()) >= 0) {
                    found.add(lock);
                }
            } else {
                final SegmentLock segmentLock = (SegmentLock) lock;
                if (segmentLock.getInterval().contains(segment.getInterval()) && segmentLock.getDataSource().equals(segment.getDataSource()) && segmentLock.getVersion().compareTo(segment.getVersion()) >= 0 && segmentLock.getPartitionId() == segment.getShardSpec().getPartitionNum()) {
                    found.add(lock);
                }
            }
        });
    });
    return found;
}

Example 2 with TimeChunkLock

use of org.apache.druid.indexing.common.TimeChunkLock in project druid by druid-io.

the class TaskLockboxTest method testSyncFromStorageWithMissingTaskPriority.

@Test
public void testSyncFromStorageWithMissingTaskPriority() throws EntryExistsException {
    final Task task = NoopTask.create();
    taskStorage.insert(task, TaskStatus.running(task.getId()));
    taskStorage.addLock(task.getId(), new TimeChunkLock(TaskLockType.EXCLUSIVE, task.getGroupId(), task.getDataSource(), Intervals.of("2017/2018"), "v1", task.getPriority()));
    final List<TaskLock> beforeLocksInStorage = taskStorage.getActiveTasks().stream().flatMap(t -> taskStorage.getLocks(t.getId()).stream()).collect(Collectors.toList());
    final TaskLockbox lockbox = new TaskLockbox(taskStorage, metadataStorageCoordinator);
    lockbox.syncFromStorage();
    final List<TaskLock> afterLocksInStorage = taskStorage.getActiveTasks().stream().flatMap(t -> taskStorage.getLocks(t.getId()).stream()).collect(Collectors.toList());
    Assert.assertEquals(beforeLocksInStorage, afterLocksInStorage);
}

Example 3 with TimeChunkLock

use of org.apache.druid.indexing.common.TimeChunkLock in project druid by druid-io.

the class TaskLockboxTest method testSyncFromStorageWithInvalidPriority.

@Test
public void testSyncFromStorageWithInvalidPriority() throws EntryExistsException {
    final Task task = NoopTask.create();
    taskStorage.insert(task, TaskStatus.running(task.getId()));
    taskStorage.addLock(task.getId(), new TimeChunkLock(TaskLockType.EXCLUSIVE, task.getGroupId(), task.getDataSource(), Intervals.of("2017/2018"), "v1", 10));
    final TaskLockbox lockbox = new TaskLockbox(taskStorage, metadataStorageCoordinator);
    expectedException.expect(IllegalArgumentException.class);
    expectedException.expectMessage("lock priority[10] is different from task priority[50]");
    lockbox.syncFromStorage();
}

Example 4 with TimeChunkLock

use of org.apache.druid.indexing.common.TimeChunkLock in project druid by druid-io.

the class TaskLockboxTest method testGetTimeChunkAndSegmentLockForSameGroup.

@Test
public void testGetTimeChunkAndSegmentLockForSameGroup() {
    final Task task1 = NoopTask.withGroupId("groupId");
    final Task task2 = NoopTask.withGroupId("groupId");
    lockbox.add(task1);
    lockbox.add(task2);
    Assert.assertTrue(lockbox.tryLock(task1, new TimeChunkLockRequest(TaskLockType.EXCLUSIVE, task1, Intervals.of("2017/2018"), null)).isOk());
    Assert.assertTrue(lockbox.tryLock(task2, new SpecificSegmentLockRequest(TaskLockType.EXCLUSIVE, task2, Intervals.of("2017/2018"), "version", 0)).isOk());
    final List<TaskLockPosse> posses = lockbox.getAllLocks().get(task1.getDataSource()).get(DateTimes.of("2017")).get(Intervals.of("2017/2018"));
    Assert.assertEquals(2, posses.size());
    Assert.assertEquals(LockGranularity.TIME_CHUNK, posses.get(0).getTaskLock().getGranularity());
    final TimeChunkLock timeChunkLock = (TimeChunkLock) posses.get(0).getTaskLock();
    Assert.assertEquals("none", timeChunkLock.getDataSource());
    Assert.assertEquals("groupId", timeChunkLock.getGroupId());
    Assert.assertEquals(Intervals.of("2017/2018"), timeChunkLock.getInterval());
    Assert.assertEquals(LockGranularity.SEGMENT, posses.get(1).getTaskLock().getGranularity());
    final SegmentLock segmentLock = (SegmentLock) posses.get(1).getTaskLock();
    Assert.assertEquals("none", segmentLock.getDataSource());
    Assert.assertEquals("groupId", segmentLock.getGroupId());
    Assert.assertEquals(Intervals.of("2017/2018"), segmentLock.getInterval());
    Assert.assertEquals(0, segmentLock.getPartitionId());
}

Example 5 with TimeChunkLock

use of org.apache.druid.indexing.common.TimeChunkLock in project druid by druid-io.

the class TaskLockbox method verifyAndCreateOrFindLockPosse.

/**
 * This method is called only in {@link #syncFromStorage()} and verifies the given task and the taskLock have the same
 * groupId, dataSource, and priority.
 */
private TaskLockPosse verifyAndCreateOrFindLockPosse(Task task, TaskLock taskLock) {
    giant.lock();
    try {
        Preconditions.checkArgument(task.getGroupId().equals(taskLock.getGroupId()), "lock groupId[%s] is different from task groupId[%s]", taskLock.getGroupId(), task.getGroupId());
        Preconditions.checkArgument(task.getDataSource().equals(taskLock.getDataSource()), "lock dataSource[%s] is different from task dataSource[%s]", taskLock.getDataSource(), task.getDataSource());
        final int taskPriority = task.getPriority();
        final int lockPriority = taskLock.getNonNullPriority();
        Preconditions.checkArgument(lockPriority == taskPriority, "lock priority[%s] is different from task priority[%s]", lockPriority, taskPriority);
        final LockRequest request;
        switch(taskLock.getGranularity()) {
            case SEGMENT:
                final SegmentLock segmentLock = (SegmentLock) taskLock;
                request = new SpecificSegmentLockRequest(segmentLock.getType(), segmentLock.getGroupId(), segmentLock.getDataSource(), segmentLock.getInterval(), segmentLock.getVersion(), segmentLock.getPartitionId(), taskPriority, segmentLock.isRevoked());
                break;
            case TIME_CHUNK:
                final TimeChunkLock timeChunkLock = (TimeChunkLock) taskLock;
                request = new TimeChunkLockRequest(timeChunkLock.getType(), timeChunkLock.getGroupId(), timeChunkLock.getDataSource(), timeChunkLock.getInterval(), timeChunkLock.getVersion(), taskPriority, timeChunkLock.isRevoked());
                break;
            default:
                throw new ISE("Unknown lockGranularity[%s]", taskLock.getGranularity());
        }
        return createOrFindLockPosse(request);
    } finally {
        giant.unlock();
    }
}