Examples with NumberedShardSpec io.druid.timeline.partition.NumberedShardSpec used on opensource projects

Example 16 with NumberedShardSpec

use of io.druid.timeline.partition.NumberedShardSpec in project druid by druid-io.

the class HdfsDataSegmentPusherTest method testUsingSchemeForMultipleSegments.

private void testUsingSchemeForMultipleSegments(final String scheme, final int numberOfSegments) throws Exception {
    Configuration conf = new Configuration(true);
    DataSegment[] segments = new DataSegment[numberOfSegments];
    // Create a mock segment on disk
    File segmentDir = tempFolder.newFolder();
    File tmp = new File(segmentDir, "version.bin");
    final byte[] data = new byte[] { 0x0, 0x0, 0x0, 0x1 };
    Files.write(data, tmp);
    final long size = data.length;
    HdfsDataSegmentPusherConfig config = new HdfsDataSegmentPusherConfig();
    final File storageDirectory = tempFolder.newFolder();
    config.setStorageDirectory(scheme != null ? String.format("%s://%s", scheme, storageDirectory.getAbsolutePath()) : storageDirectory.getAbsolutePath());
    HdfsDataSegmentPusher pusher = new HdfsDataSegmentPusher(config, conf, new DefaultObjectMapper());
    for (int i = 0; i < numberOfSegments; i++) {
        segments[i] = new DataSegment("foo", new Interval("2015/2016"), "0", Maps.<String, Object>newHashMap(), Lists.<String>newArrayList(), Lists.<String>newArrayList(), new NumberedShardSpec(i, i), 0, size);
    }
    for (int i = 0; i < numberOfSegments; i++) {
        final DataSegment pushedSegment = pusher.push(segmentDir, segments[i]);
        String indexUri = String.format("%s/%s/%d_index.zip", FileSystem.newInstance(conf).makeQualified(new Path(config.getStorageDirectory())).toUri().toString(), DataSegmentPusherUtil.getHdfsStorageDir(segments[i]), segments[i].getShardSpec().getPartitionNum());
        Assert.assertEquals(segments[i].getSize(), pushedSegment.getSize());
        Assert.assertEquals(segments[i], pushedSegment);
        Assert.assertEquals(ImmutableMap.of("type", "hdfs", "path", indexUri), pushedSegment.getLoadSpec());
        // rename directory after push
        String segmentPath = DataSegmentPusherUtil.getHdfsStorageDir(pushedSegment);
        File indexFile = new File(String.format("%s/%s/%d_index.zip", storageDirectory, segmentPath, pushedSegment.getShardSpec().getPartitionNum()));
        Assert.assertTrue(indexFile.exists());
        File descriptorFile = new File(String.format("%s/%s/%d_descriptor.json", storageDirectory, segmentPath, pushedSegment.getShardSpec().getPartitionNum()));
        Assert.assertTrue(descriptorFile.exists());
        //read actual data from descriptor file.
        DataSegment fromDescriptorFileDataSegment = objectMapper.readValue(descriptorFile, DataSegment.class);
        Assert.assertEquals(segments[i].getSize(), pushedSegment.getSize());
        Assert.assertEquals(segments[i], pushedSegment);
        Assert.assertEquals(ImmutableMap.of("type", "hdfs", "path", indexUri), fromDescriptorFileDataSegment.getLoadSpec());
        // rename directory after push
        segmentPath = DataSegmentPusherUtil.getHdfsStorageDir(fromDescriptorFileDataSegment);
        indexFile = new File(String.format("%s/%s/%d_index.zip", storageDirectory, segmentPath, fromDescriptorFileDataSegment.getShardSpec().getPartitionNum()));
        Assert.assertTrue(indexFile.exists());
        // push twice will fail and temp dir cleaned
        File outDir = new File(String.format("%s/%s", config.getStorageDirectory(), segmentPath));
        outDir.setReadOnly();
        try {
            pusher.push(segmentDir, segments[i]);
        } catch (IOException e) {
            Assert.fail("should not throw exception");
        }
    }
}

Example 17 with NumberedShardSpec

use of io.druid.timeline.partition.NumberedShardSpec in project druid by druid-io.

the class ClientInfoResourceTest method setup.

@Before
public void setup() {
    VersionedIntervalTimeline<String, ServerSelector> timeline = new VersionedIntervalTimeline<>(Ordering.<String>natural());
    DruidServer server = new DruidServer("name", "host", 1234, "type", "tier", 0);
    addSegment(timeline, server, "1960-02-13/1961-02-14", ImmutableList.of("d5"), ImmutableList.of("m5"), "v0");
    // segments within [2014-02-13, 2014-02-18]
    addSegment(timeline, server, "2014-02-13/2014-02-14", ImmutableList.of("d1"), ImmutableList.of("m1"), "v0");
    addSegment(timeline, server, "2014-02-14/2014-02-15", ImmutableList.of("d1"), ImmutableList.of("m1"), "v0");
    addSegment(timeline, server, "2014-02-16/2014-02-17", ImmutableList.of("d1"), ImmutableList.of("m1"), "v0");
    addSegment(timeline, server, "2014-02-17/2014-02-18", ImmutableList.of("d2"), ImmutableList.of("m2"), "v0");
    // segments within [2015-02-01, 2015-02-13]
    addSegment(timeline, server, "2015-02-01/2015-02-07", ImmutableList.of("d1"), ImmutableList.of("m1"), "v1");
    addSegment(timeline, server, "2015-02-07/2015-02-13", ImmutableList.of("d1"), ImmutableList.of("m1"), "v1");
    addSegmentWithShardSpec(timeline, server, "2015-02-03/2015-02-05", ImmutableList.of("d1", "d2"), ImmutableList.of("m1", "m2"), "v2", new NumberedShardSpec(0, 2));
    addSegmentWithShardSpec(timeline, server, "2015-02-03/2015-02-05", ImmutableList.of("d1", "d2", "d3"), ImmutableList.of("m1", "m2", "m3"), "v2", new NumberedShardSpec(1, 2));
    addSegment(timeline, server, "2015-02-09/2015-02-10", ImmutableList.of("d1", "d3"), ImmutableList.of("m1", "m3"), "v2");
    addSegment(timeline, server, "2015-02-11/2015-02-12", ImmutableList.of("d3"), ImmutableList.of("m3"), "v2");
    // segments within [2015-03-13, 2015-03-19]
    addSegment(timeline, server, "2015-03-13/2015-03-19", ImmutableList.of("d1"), ImmutableList.of("m1"), "v3");
    addSegment(timeline, server, "2015-03-13/2015-03-14", ImmutableList.of("d1"), ImmutableList.of("m1"), "v4");
    addSegment(timeline, server, "2015-03-14/2015-03-15", ImmutableList.of("d1"), ImmutableList.of("m1"), "v5");
    addSegment(timeline, server, "2015-03-15/2015-03-16", ImmutableList.of("d1"), ImmutableList.of("m1"), "v6");
    // imcomplete segment
    addSegmentWithShardSpec(timeline, server, "2015-04-03/2015-04-05", ImmutableList.of("d4"), ImmutableList.of("m4"), "v7", new NumberedShardSpec(0, 2));
    serverInventoryView = EasyMock.createMock(FilteredServerInventoryView.class);
    EasyMock.expect(serverInventoryView.getInventory()).andReturn(ImmutableList.of(server)).anyTimes();
    timelineServerView = EasyMock.createMock(TimelineServerView.class);
    EasyMock.expect(timelineServerView.getTimeline(EasyMock.anyObject(TableDataSource.class))).andReturn(timeline);
    EasyMock.replay(serverInventoryView, timelineServerView);
    resource = getResourceTestHelper(serverInventoryView, timelineServerView, new SegmentMetadataQueryConfig());
}

Example 18 with NumberedShardSpec

use of io.druid.timeline.partition.NumberedShardSpec in project druid by druid-io.

the class NumberedShardSpecTest method testVersionedIntervalTimelineBehaviorForNumberedShardSpec.

@Test
public void testVersionedIntervalTimelineBehaviorForNumberedShardSpec() {
    //core partition chunks
    PartitionChunk<String> chunk0 = new NumberedShardSpec(0, 2).createChunk("0");
    PartitionChunk<String> chunk1 = new NumberedShardSpec(1, 2).createChunk("1");
    //appended partition chunk
    PartitionChunk<String> chunk4 = new NumberedShardSpec(4, 2).createChunk("4");
    //incomplete partition sets
    testVersionedIntervalTimelineBehaviorForNumberedShardSpec(ImmutableList.of(chunk0), Collections.EMPTY_SET);
    testVersionedIntervalTimelineBehaviorForNumberedShardSpec(ImmutableList.of(chunk1), Collections.EMPTY_SET);
    testVersionedIntervalTimelineBehaviorForNumberedShardSpec(ImmutableList.of(chunk4), Collections.EMPTY_SET);
    testVersionedIntervalTimelineBehaviorForNumberedShardSpec(ImmutableList.of(chunk0, chunk4), Collections.EMPTY_SET);
    testVersionedIntervalTimelineBehaviorForNumberedShardSpec(ImmutableList.of(chunk1, chunk4), Collections.EMPTY_SET);
    //complete partition sets
    testVersionedIntervalTimelineBehaviorForNumberedShardSpec(ImmutableList.of(chunk1, chunk0), ImmutableSet.of("0", "1"));
    testVersionedIntervalTimelineBehaviorForNumberedShardSpec(ImmutableList.of(chunk4, chunk1, chunk0), ImmutableSet.of("0", "1", "4"));
    // a partition set with 0 core partitions
    chunk0 = new NumberedShardSpec(0, 0).createChunk("0");
    chunk4 = new NumberedShardSpec(4, 0).createChunk("4");
    testVersionedIntervalTimelineBehaviorForNumberedShardSpec(ImmutableList.of(chunk0), ImmutableSet.of("0"));
    testVersionedIntervalTimelineBehaviorForNumberedShardSpec(ImmutableList.of(chunk4), ImmutableSet.of("4"));
    testVersionedIntervalTimelineBehaviorForNumberedShardSpec(ImmutableList.of(chunk4, chunk0), ImmutableSet.of("0", "4"));
}

Example 19 with NumberedShardSpec

use of io.druid.timeline.partition.NumberedShardSpec in project druid by druid-io.

the class IndexTaskTest method testForceExtendableShardSpecs.

@Test
public void testForceExtendableShardSpecs() throws Exception {
    File tmpDir = temporaryFolder.newFolder();
    File tmpFile = File.createTempFile("druid", "index", tmpDir);
    PrintWriter writer = new PrintWriter(tmpFile);
    writer.println("2014-01-01T00:00:10Z,a,1");
    writer.println("2014-01-01T01:00:20Z,b,1");
    writer.println("2014-01-01T02:00:30Z,c,1");
    writer.close();
    IndexTask indexTask = new IndexTask(null, null, createIngestionSpec(tmpDir, null, 2, null, true, false), null, jsonMapper);
    final List<DataSegment> segments = runTask(indexTask);
    Assert.assertEquals(2, segments.size());
    Assert.assertEquals("test", segments.get(0).getDataSource());
    Assert.assertEquals(new Interval("2014/P1D"), segments.get(0).getInterval());
    Assert.assertTrue(segments.get(0).getShardSpec().getClass().equals(NumberedShardSpec.class));
    Assert.assertEquals(0, segments.get(0).getShardSpec().getPartitionNum());
    Assert.assertEquals(2, ((NumberedShardSpec) segments.get(0).getShardSpec()).getPartitions());
    Assert.assertEquals("test", segments.get(1).getDataSource());
    Assert.assertEquals(new Interval("2014/P1D"), segments.get(1).getInterval());
    Assert.assertTrue(segments.get(1).getShardSpec().getClass().equals(NumberedShardSpec.class));
    Assert.assertEquals(1, segments.get(1).getShardSpec().getPartitionNum());
    Assert.assertEquals(2, ((NumberedShardSpec) segments.get(1).getShardSpec()).getPartitions());
}

Example 20 with NumberedShardSpec

use of io.druid.timeline.partition.NumberedShardSpec in project druid by druid-io.

the class SegmentAllocateActionTest method testMultipleSequences.

@Test
public void testMultipleSequences() throws Exception {
    final Task task = new NoopTask(null, 0, 0, null, null, null);
    taskActionTestKit.getTaskLockbox().add(task);
    final SegmentIdentifier id1 = allocate(task, PARTY_TIME, Granularities.NONE, Granularities.HOUR, "s1", null);
    final SegmentIdentifier id2 = allocate(task, PARTY_TIME, Granularities.NONE, Granularities.HOUR, "s2", null);
    final SegmentIdentifier id3 = allocate(task, PARTY_TIME, Granularities.NONE, Granularities.HOUR, "s1", id1.getIdentifierAsString());
    final SegmentIdentifier id4 = allocate(task, THE_DISTANT_FUTURE, Granularities.NONE, Granularities.HOUR, "s1", id3.getIdentifierAsString());
    final SegmentIdentifier id5 = allocate(task, THE_DISTANT_FUTURE, Granularities.NONE, Granularities.HOUR, "s2", id2.getIdentifierAsString());
    final SegmentIdentifier id6 = allocate(task, PARTY_TIME, Granularities.NONE, Granularities.HOUR, "s1", null);
    final TaskLock partyLock = Iterables.getOnlyElement(FluentIterable.from(taskActionTestKit.getTaskLockbox().findLocksForTask(task)).filter(new Predicate<TaskLock>() {

        @Override
        public boolean apply(TaskLock input) {
            return input.getInterval().contains(PARTY_TIME);
        }
    }));
    final TaskLock futureLock = Iterables.getOnlyElement(FluentIterable.from(taskActionTestKit.getTaskLockbox().findLocksForTask(task)).filter(new Predicate<TaskLock>() {

        @Override
        public boolean apply(TaskLock input) {
            return input.getInterval().contains(THE_DISTANT_FUTURE);
        }
    }));
    assertSameIdentifier(id1, new SegmentIdentifier(DATA_SOURCE, Granularities.HOUR.bucket(PARTY_TIME), partyLock.getVersion(), new NumberedShardSpec(0, 0)));
    assertSameIdentifier(id2, new SegmentIdentifier(DATA_SOURCE, Granularities.HOUR.bucket(PARTY_TIME), partyLock.getVersion(), new NumberedShardSpec(1, 0)));
    assertSameIdentifier(id3, new SegmentIdentifier(DATA_SOURCE, Granularities.HOUR.bucket(PARTY_TIME), partyLock.getVersion(), new NumberedShardSpec(2, 0)));
    assertSameIdentifier(id4, new SegmentIdentifier(DATA_SOURCE, Granularities.HOUR.bucket(THE_DISTANT_FUTURE), futureLock.getVersion(), new NumberedShardSpec(0, 0)));
    assertSameIdentifier(id5, new SegmentIdentifier(DATA_SOURCE, Granularities.HOUR.bucket(THE_DISTANT_FUTURE), futureLock.getVersion(), new NumberedShardSpec(1, 0)));
    assertSameIdentifier(id6, id1);
}