Examples with SingleDimensionShardSpec io.druid.timeline.partition.SingleDimensionShardSpec used on opensource projects

Example 6 with SingleDimensionShardSpec

use of io.druid.timeline.partition.SingleDimensionShardSpec in project druid by druid-io.

the class IndexGeneratorJobTest method verifyJob.

private void verifyJob(IndexGeneratorJob job) throws IOException {
    JobHelper.runJobs(ImmutableList.<Jobby>of(job), config);
    int segmentNum = 0;
    for (DateTime currTime = interval.getStart(); currTime.isBefore(interval.getEnd()); currTime = currTime.plusDays(1)) {
        Object[][] shardInfo = shardInfoForEachSegment[segmentNum++];
        File segmentOutputFolder = new File(String.format("%s/%s/%s_%s/%s", config.getSchema().getIOConfig().getSegmentOutputPath(), config.getSchema().getDataSchema().getDataSource(), currTime.toString(), currTime.plusDays(1).toString(), config.getSchema().getTuningConfig().getVersion()));
        Assert.assertTrue(segmentOutputFolder.exists());
        Assert.assertEquals(shardInfo.length, segmentOutputFolder.list().length);
        for (int partitionNum = 0; partitionNum < shardInfo.length; ++partitionNum) {
            File individualSegmentFolder = new File(segmentOutputFolder, Integer.toString(partitionNum));
            Assert.assertTrue(individualSegmentFolder.exists());
            File descriptor = new File(individualSegmentFolder, "descriptor.json");
            File indexZip = new File(individualSegmentFolder, "index.zip");
            Assert.assertTrue(descriptor.exists());
            Assert.assertTrue(indexZip.exists());
            DataSegment dataSegment = mapper.readValue(descriptor, DataSegment.class);
            Assert.assertEquals(config.getSchema().getTuningConfig().getVersion(), dataSegment.getVersion());
            Assert.assertEquals(new Interval(currTime, currTime.plusDays(1)), dataSegment.getInterval());
            Assert.assertEquals("local", dataSegment.getLoadSpec().get("type"));
            Assert.assertEquals(indexZip.getCanonicalPath(), dataSegment.getLoadSpec().get("path"));
            Assert.assertEquals(Integer.valueOf(9), dataSegment.getBinaryVersion());
            if (datasourceName.equals("website")) {
                Assert.assertEquals("website", dataSegment.getDataSource());
                Assert.assertEquals("host", dataSegment.getDimensions().get(0));
                Assert.assertEquals("visited_num", dataSegment.getMetrics().get(0));
                Assert.assertEquals("unique_hosts", dataSegment.getMetrics().get(1));
            } else if (datasourceName.equals("inherit_dims")) {
                Assert.assertEquals("inherit_dims", dataSegment.getDataSource());
                Assert.assertEquals(ImmutableList.of("X", "Y", "M", "Q", "B", "F"), dataSegment.getDimensions());
                Assert.assertEquals("count", dataSegment.getMetrics().get(0));
            } else if (datasourceName.equals("inherit_dims2")) {
                Assert.assertEquals("inherit_dims2", dataSegment.getDataSource());
                Assert.assertEquals(ImmutableList.of("B", "F", "M", "Q", "X", "Y"), dataSegment.getDimensions());
                Assert.assertEquals("count", dataSegment.getMetrics().get(0));
            } else {
                Assert.fail("Test did not specify supported datasource name");
            }
            if (forceExtendableShardSpecs) {
                NumberedShardSpec spec = (NumberedShardSpec) dataSegment.getShardSpec();
                Assert.assertEquals(partitionNum, spec.getPartitionNum());
                Assert.assertEquals(shardInfo.length, spec.getPartitions());
            } else if (partitionType.equals("hashed")) {
                Integer[] hashShardInfo = (Integer[]) shardInfo[partitionNum];
                HashBasedNumberedShardSpec spec = (HashBasedNumberedShardSpec) dataSegment.getShardSpec();
                Assert.assertEquals((int) hashShardInfo[0], spec.getPartitionNum());
                Assert.assertEquals((int) hashShardInfo[1], spec.getPartitions());
            } else if (partitionType.equals("single")) {
                String[] singleDimensionShardInfo = (String[]) shardInfo[partitionNum];
                SingleDimensionShardSpec spec = (SingleDimensionShardSpec) dataSegment.getShardSpec();
                Assert.assertEquals(singleDimensionShardInfo[0], spec.getStart());
                Assert.assertEquals(singleDimensionShardInfo[1], spec.getEnd());
            } else {
                throw new RuntimeException(String.format("Invalid partition type:[%s]", partitionType));
            }
        }
    }
}

Example 7 with SingleDimensionShardSpec

use of io.druid.timeline.partition.SingleDimensionShardSpec in project druid by druid-io.

the class SegmentAllocateActionTest method testCannotAddToExistingSingleDimensionShardSpecs.

@Test
public void testCannotAddToExistingSingleDimensionShardSpecs() throws Exception {
    final Task task = new NoopTask(null, 0, 0, null, null, null);
    taskActionTestKit.getMetadataStorageCoordinator().announceHistoricalSegments(ImmutableSet.of(DataSegment.builder().dataSource(DATA_SOURCE).interval(Granularities.HOUR.bucket(PARTY_TIME)).version(PARTY_TIME.toString()).shardSpec(new SingleDimensionShardSpec("foo", null, "bar", 0)).build(), DataSegment.builder().dataSource(DATA_SOURCE).interval(Granularities.HOUR.bucket(PARTY_TIME)).version(PARTY_TIME.toString()).shardSpec(new SingleDimensionShardSpec("foo", "bar", null, 1)).build()));
    taskActionTestKit.getTaskLockbox().add(task);
    final SegmentIdentifier id1 = allocate(task, PARTY_TIME, Granularities.NONE, Granularities.HOUR, "s1", null);
    Assert.assertNull(id1);
}

Example 8 with SingleDimensionShardSpec

use of io.druid.timeline.partition.SingleDimensionShardSpec in project druid by druid-io.

the class DataSegmentTest method testIdentifierWithNonzeroPartition.

@Test
public void testIdentifierWithNonzeroPartition() {
    final DataSegment segment = DataSegment.builder().dataSource("foo").interval(new Interval("2012-01-01/2012-01-02")).version(new DateTime("2012-01-01T11:22:33.444Z").toString()).shardSpec(new SingleDimensionShardSpec("bar", "abc", "def", 1)).build();
    Assert.assertEquals("foo_2012-01-01T00:00:00.000Z_2012-01-02T00:00:00.000Z_2012-01-01T11:22:33.444Z_1", segment.getIdentifier());
}