Examples with NumberedShardSpec io.druid.timeline.partition.NumberedShardSpec used on opensource projects

Example 1 with NumberedShardSpec

use of io.druid.timeline.partition.NumberedShardSpec in project druid by druid-io.

the class SegmentAllocateActionTest method testResumeSequence.

@Test
public void testResumeSequence() throws Exception {
    final Task task = new NoopTask(null, 0, 0, null, null, null);
    taskActionTestKit.getTaskLockbox().add(task);
    final SegmentIdentifier id1 = allocate(task, PARTY_TIME, Granularities.NONE, Granularities.HOUR, "s1", null);
    final SegmentIdentifier id2 = allocate(task, THE_DISTANT_FUTURE, Granularities.NONE, Granularities.HOUR, "s1", id1.getIdentifierAsString());
    final SegmentIdentifier id3 = allocate(task, PARTY_TIME, Granularities.NONE, Granularities.HOUR, "s1", id2.getIdentifierAsString());
    final SegmentIdentifier id4 = allocate(task, PARTY_TIME, Granularities.NONE, Granularities.HOUR, "s1", id1.getIdentifierAsString());
    final SegmentIdentifier id5 = allocate(task, THE_DISTANT_FUTURE, Granularities.NONE, Granularities.HOUR, "s1", id1.getIdentifierAsString());
    final SegmentIdentifier id6 = allocate(task, THE_DISTANT_FUTURE, Granularities.NONE, Granularities.MINUTE, "s1", id1.getIdentifierAsString());
    final SegmentIdentifier id7 = allocate(task, THE_DISTANT_FUTURE, Granularities.NONE, Granularities.DAY, "s1", id1.getIdentifierAsString());
    final TaskLock partyLock = Iterables.getOnlyElement(FluentIterable.from(taskActionTestKit.getTaskLockbox().findLocksForTask(task)).filter(new Predicate<TaskLock>() {

        @Override
        public boolean apply(TaskLock input) {
            return input.getInterval().contains(PARTY_TIME);
        }
    }));
    final TaskLock futureLock = Iterables.getOnlyElement(FluentIterable.from(taskActionTestKit.getTaskLockbox().findLocksForTask(task)).filter(new Predicate<TaskLock>() {

        @Override
        public boolean apply(TaskLock input) {
            return input.getInterval().contains(THE_DISTANT_FUTURE);
        }
    }));
    assertSameIdentifier(id1, new SegmentIdentifier(DATA_SOURCE, Granularities.HOUR.bucket(PARTY_TIME), partyLock.getVersion(), new NumberedShardSpec(0, 0)));
    assertSameIdentifier(id2, new SegmentIdentifier(DATA_SOURCE, Granularities.HOUR.bucket(THE_DISTANT_FUTURE), futureLock.getVersion(), new NumberedShardSpec(0, 0)));
    assertSameIdentifier(id3, new SegmentIdentifier(DATA_SOURCE, Granularities.HOUR.bucket(PARTY_TIME), partyLock.getVersion(), new NumberedShardSpec(1, 0)));
    Assert.assertNull(id4);
    assertSameIdentifier(id5, id2);
    Assert.assertNull(id6);
    assertSameIdentifier(id7, id2);
}

Example 2 with NumberedShardSpec

use of io.druid.timeline.partition.NumberedShardSpec in project druid by druid-io.

the class SegmentAllocateActionTest method testAddToExistingNumberedShardSpecsCoarserPreferredGranularity.

@Test
public void testAddToExistingNumberedShardSpecsCoarserPreferredGranularity() throws Exception {
    final Task task = new NoopTask(null, 0, 0, null, null, null);
    taskActionTestKit.getMetadataStorageCoordinator().announceHistoricalSegments(ImmutableSet.of(DataSegment.builder().dataSource(DATA_SOURCE).interval(Granularities.HOUR.bucket(PARTY_TIME)).version(PARTY_TIME.toString()).shardSpec(new NumberedShardSpec(0, 2)).build(), DataSegment.builder().dataSource(DATA_SOURCE).interval(Granularities.HOUR.bucket(PARTY_TIME)).version(PARTY_TIME.toString()).shardSpec(new NumberedShardSpec(1, 2)).build()));
    taskActionTestKit.getTaskLockbox().add(task);
    final SegmentIdentifier id1 = allocate(task, PARTY_TIME, Granularities.NONE, Granularities.DAY, "s1", null);
    assertSameIdentifier(id1, new SegmentIdentifier(DATA_SOURCE, Granularities.HOUR.bucket(PARTY_TIME), PARTY_TIME.toString(), new NumberedShardSpec(2, 2)));
}

Example 3 with NumberedShardSpec

use of io.druid.timeline.partition.NumberedShardSpec in project druid by druid-io.

the class IndexTaskTest method runTask.

private final List<DataSegment> runTask(final IndexTask indexTask) throws Exception {
    final List<DataSegment> segments = Lists.newArrayList();
    indexTask.run(new TaskToolbox(null, null, new TaskActionClient() {

        @Override
        public <RetType> RetType submit(TaskAction<RetType> taskAction) throws IOException {
            if (taskAction instanceof LockListAction) {
                return (RetType) Arrays.asList(new TaskLock("", "", null, new DateTime().toString()));
            }
            if (taskAction instanceof LockAcquireAction) {
                return (RetType) new TaskLock("groupId", "test", ((LockAcquireAction) taskAction).getInterval(), new DateTime().toString());
            }
            if (taskAction instanceof SegmentTransactionalInsertAction) {
                return (RetType) new SegmentPublishResult(((SegmentTransactionalInsertAction) taskAction).getSegments(), true);
            }
            if (taskAction instanceof SegmentAllocateAction) {
                SegmentAllocateAction action = (SegmentAllocateAction) taskAction;
                Interval interval = action.getPreferredSegmentGranularity().bucket(action.getTimestamp());
                ShardSpec shardSpec = new NumberedShardSpec(segmentAllocatePartitionCounter++, 0);
                return (RetType) new SegmentIdentifier(action.getDataSource(), interval, "latestVersion", shardSpec);
            }
            return null;
        }
    }, null, new DataSegmentPusher() {

        @Deprecated
        @Override
        public String getPathForHadoop(String dataSource) {
            return getPathForHadoop();
        }

        @Override
        public String getPathForHadoop() {
            return null;
        }

        @Override
        public DataSegment push(File file, DataSegment segment) throws IOException {
            segments.add(segment);
            return segment;
        }
    }, null, null, null, null, null, null, null, null, null, jsonMapper, temporaryFolder.newFolder(), indexMerger, indexIO, null, null, indexMergerV9));
    Collections.sort(segments);
    return segments;
}

Example 4 with NumberedShardSpec

use of io.druid.timeline.partition.NumberedShardSpec in project druid by druid-io.

the class SegmentAllocateActionTest method testCannotAddToExistingNumberedShardSpecsWithCoarserQueryGranularity.

@Test
public void testCannotAddToExistingNumberedShardSpecsWithCoarserQueryGranularity() throws Exception {
    final Task task = new NoopTask(null, 0, 0, null, null, null);
    taskActionTestKit.getMetadataStorageCoordinator().announceHistoricalSegments(ImmutableSet.of(DataSegment.builder().dataSource(DATA_SOURCE).interval(Granularities.HOUR.bucket(PARTY_TIME)).version(PARTY_TIME.toString()).shardSpec(new NumberedShardSpec(0, 2)).build(), DataSegment.builder().dataSource(DATA_SOURCE).interval(Granularities.HOUR.bucket(PARTY_TIME)).version(PARTY_TIME.toString()).shardSpec(new NumberedShardSpec(1, 2)).build()));
    taskActionTestKit.getTaskLockbox().add(task);
    final SegmentIdentifier id1 = allocate(task, PARTY_TIME, Granularities.DAY, Granularities.DAY, "s1", null);
    Assert.assertNull(id1);
}

Example 5 with NumberedShardSpec

use of io.druid.timeline.partition.NumberedShardSpec in project druid by druid-io.

the class SegmentAllocateActionTest method assertSameIdentifier.

private void assertSameIdentifier(final SegmentIdentifier one, final SegmentIdentifier other) {
    Assert.assertEquals(one, other);
    Assert.assertEquals(one.getShardSpec().getPartitionNum(), other.getShardSpec().getPartitionNum());
    if (one.getShardSpec().getClass() == NumberedShardSpec.class && other.getShardSpec().getClass() == NumberedShardSpec.class) {
        Assert.assertEquals(((NumberedShardSpec) one.getShardSpec()).getPartitions(), ((NumberedShardSpec) other.getShardSpec()).getPartitions());
    } else if (one.getShardSpec().getClass() == LinearShardSpec.class && other.getShardSpec().getClass() == LinearShardSpec.class) {
    // do nothing
    } else {
        throw new ISE("Unexpected shardSpecs [%s] and [%s]", one.getShardSpec().getClass(), other.getShardSpec().getClass());
    }
}