Examples with NumberedShardSpec io.druid.timeline.partition.NumberedShardSpec used on opensource projects

Example 6 with NumberedShardSpec

use of io.druid.timeline.partition.NumberedShardSpec in project druid by druid-io.

the class SegmentAllocateActionTest method testManySegmentsSameInterval.

@Test
public void testManySegmentsSameInterval() throws Exception {
    final Task task = new NoopTask(null, 0, 0, null, null, null);
    taskActionTestKit.getTaskLockbox().add(task);
    final SegmentIdentifier id1 = allocate(task, PARTY_TIME, Granularities.NONE, Granularities.HOUR, "s1", null);
    final SegmentIdentifier id2 = allocate(task, PARTY_TIME, Granularities.NONE, Granularities.HOUR, "s1", id1.getIdentifierAsString());
    final SegmentIdentifier id3 = allocate(task, PARTY_TIME, Granularities.NONE, Granularities.HOUR, "s1", id2.getIdentifierAsString());
    final TaskLock partyLock = Iterables.getOnlyElement(FluentIterable.from(taskActionTestKit.getTaskLockbox().findLocksForTask(task)).filter(new Predicate<TaskLock>() {

        @Override
        public boolean apply(TaskLock input) {
            return input.getInterval().contains(PARTY_TIME);
        }
    }));
    assertSameIdentifier(id1, new SegmentIdentifier(DATA_SOURCE, Granularities.HOUR.bucket(PARTY_TIME), partyLock.getVersion(), new NumberedShardSpec(0, 0)));
    assertSameIdentifier(id2, new SegmentIdentifier(DATA_SOURCE, Granularities.HOUR.bucket(PARTY_TIME), partyLock.getVersion(), new NumberedShardSpec(1, 0)));
    assertSameIdentifier(id3, new SegmentIdentifier(DATA_SOURCE, Granularities.HOUR.bucket(PARTY_TIME), partyLock.getVersion(), new NumberedShardSpec(2, 0)));
}

Example 7 with NumberedShardSpec

use of io.druid.timeline.partition.NumberedShardSpec in project druid by druid-io.

the class SegmentAllocateActionTest method testAddToExistingNumberedShardSpecsSameGranularity.

@Test
public void testAddToExistingNumberedShardSpecsSameGranularity() throws Exception {
    final Task task = new NoopTask(null, 0, 0, null, null, null);
    taskActionTestKit.getMetadataStorageCoordinator().announceHistoricalSegments(ImmutableSet.of(DataSegment.builder().dataSource(DATA_SOURCE).interval(Granularities.HOUR.bucket(PARTY_TIME)).version(PARTY_TIME.toString()).shardSpec(new NumberedShardSpec(0, 2)).build(), DataSegment.builder().dataSource(DATA_SOURCE).interval(Granularities.HOUR.bucket(PARTY_TIME)).version(PARTY_TIME.toString()).shardSpec(new NumberedShardSpec(1, 2)).build()));
    taskActionTestKit.getTaskLockbox().add(task);
    final SegmentIdentifier id1 = allocate(task, PARTY_TIME, Granularities.NONE, Granularities.HOUR, "s1", null);
    final SegmentIdentifier id2 = allocate(task, PARTY_TIME, Granularities.NONE, Granularities.HOUR, "s1", id1.getIdentifierAsString());
    assertSameIdentifier(id1, new SegmentIdentifier(DATA_SOURCE, Granularities.HOUR.bucket(PARTY_TIME), PARTY_TIME.toString(), new NumberedShardSpec(2, 2)));
    assertSameIdentifier(id2, new SegmentIdentifier(DATA_SOURCE, Granularities.HOUR.bucket(PARTY_TIME), PARTY_TIME.toString(), new NumberedShardSpec(3, 2)));
}

Example 8 with NumberedShardSpec

use of io.druid.timeline.partition.NumberedShardSpec in project druid by druid-io.

the class IngestSegmentFirehoseFactoryTest method buildSegment.

private static DataSegment buildSegment(Integer shardNumber) {
    Preconditions.checkArgument(shardNumber < MAX_SHARD_NUMBER);
    Preconditions.checkArgument(shardNumber >= 0);
    return new DataSegment(DATA_SOURCE_NAME, FOREVER, DATA_SOURCE_VERSION, ImmutableMap.<String, Object>of("type", "local", "path", persistDir.getAbsolutePath()), ImmutableList.of(DIM_NAME), ImmutableList.of(METRIC_LONG_NAME, METRIC_FLOAT_NAME), new NumberedShardSpec(shardNumber, MAX_SHARD_NUMBER), BINARY_VERSION, 0L);
}

Example 9 with NumberedShardSpec

use of io.druid.timeline.partition.NumberedShardSpec in project druid by druid-io.

the class CoordinatorBasedSegmentHandoffNotifierTest method testHandoffCallbackCalled.

@Test
public void testHandoffCallbackCalled() throws IOException, InterruptedException {
    Interval interval = new Interval("2011-04-01/2011-04-02");
    SegmentDescriptor descriptor = new SegmentDescriptor(interval, "v1", 2);
    DataSegment segment = new DataSegment("test_ds", interval, "v1", null, null, null, new NumberedShardSpec(2, 3), 0, 0);
    final AtomicBoolean callbackCalled = new AtomicBoolean(false);
    CoordinatorClient coordinatorClient = EasyMock.createMock(CoordinatorClient.class);
    EasyMock.expect(coordinatorClient.fetchServerView("test_ds", interval, true)).andReturn(Lists.newArrayList(new ImmutableSegmentLoadInfo(segment, Sets.newHashSet(createHistoricalServerMetadata("a1"))))).anyTimes();
    EasyMock.replay(coordinatorClient);
    CoordinatorBasedSegmentHandoffNotifier notifier = new CoordinatorBasedSegmentHandoffNotifier("test_ds", coordinatorClient, notifierConfig);
    notifier.registerSegmentHandoffCallback(descriptor, MoreExecutors.sameThreadExecutor(), new Runnable() {

        @Override
        public void run() {
            callbackCalled.set(true);
        }
    });
    Assert.assertEquals(1, notifier.getHandOffCallbacks().size());
    Assert.assertTrue(notifier.getHandOffCallbacks().containsKey(descriptor));
    notifier.checkForSegmentHandoffs();
    // callback should have been removed
    Assert.assertTrue(notifier.getHandOffCallbacks().isEmpty());
    Assert.assertTrue(callbackCalled.get());
    EasyMock.verify(coordinatorClient);
}

Example 10 with NumberedShardSpec

use of io.druid.timeline.partition.NumberedShardSpec in project druid by druid-io.

the class CoordinatorBasedSegmentHandoffNotifierTest method testHandoffCallbackNotCalled.

@Test
public void testHandoffCallbackNotCalled() throws IOException, InterruptedException {
    Interval interval = new Interval("2011-04-01/2011-04-02");
    SegmentDescriptor descriptor = new SegmentDescriptor(interval, "v1", 2);
    DataSegment segment = new DataSegment("test_ds", interval, "v1", null, null, null, new NumberedShardSpec(2, 3), 0, 0);
    CoordinatorClient coordinatorClient = EasyMock.createMock(CoordinatorClient.class);
    EasyMock.expect(coordinatorClient.fetchServerView("test_ds", interval, true)).andReturn(Lists.newArrayList(new ImmutableSegmentLoadInfo(segment, Sets.newHashSet(createRealtimeServerMetadata("a1"))))).anyTimes();
    EasyMock.replay(coordinatorClient);
    CoordinatorBasedSegmentHandoffNotifier notifier = new CoordinatorBasedSegmentHandoffNotifier("test_ds", coordinatorClient, notifierConfig);
    final AtomicBoolean callbackCalled = new AtomicBoolean(false);
    notifier.registerSegmentHandoffCallback(descriptor, MoreExecutors.sameThreadExecutor(), new Runnable() {

        @Override
        public void run() {
            callbackCalled.set(true);
        }
    });
    notifier.checkForSegmentHandoffs();
    // callback should have registered
    Assert.assertEquals(1, notifier.getHandOffCallbacks().size());
    Assert.assertTrue(notifier.getHandOffCallbacks().containsKey(descriptor));
    Assert.assertFalse(callbackCalled.get());
    EasyMock.verify(coordinatorClient);
}