Examples with UniformGranularitySpec org.apache.druid.segment.indexing.granularity.UniformGranularitySpec used on opensource projects

Example 56 with UniformGranularitySpec

use of org.apache.druid.segment.indexing.granularity.UniformGranularitySpec in project druid by druid-io.

the class IndexTaskTest method testWaitForSegmentAvailabilityEmitsExpectedMetric.

@Test
public void testWaitForSegmentAvailabilityEmitsExpectedMetric() throws IOException, InterruptedException {
    final File tmpDir = temporaryFolder.newFolder();
    LatchableServiceEmitter latchEmitter = new LatchableServiceEmitter();
    latchEmitter.latch = new CountDownLatch(1);
    TaskToolbox mockToolbox = EasyMock.createMock(TaskToolbox.class);
    DataSegment mockDataSegment1 = EasyMock.createMock(DataSegment.class);
    DataSegment mockDataSegment2 = EasyMock.createMock(DataSegment.class);
    List<DataSegment> segmentsToWaitFor = new ArrayList<>();
    segmentsToWaitFor.add(mockDataSegment1);
    segmentsToWaitFor.add(mockDataSegment2);
    IndexTask indexTask = new IndexTask(null, null, createDefaultIngestionSpec(jsonMapper, tmpDir, new UniformGranularitySpec(Granularities.HOUR, Granularities.MINUTE, null), null, createTuningConfigWithMaxRowsPerSegment(2, true), false, false), null);
    EasyMock.expect(mockDataSegment1.getInterval()).andReturn(Intervals.of("1970-01-01/1971-01-01")).once();
    EasyMock.expect(mockDataSegment1.getVersion()).andReturn("dummyString").once();
    EasyMock.expect(mockDataSegment1.getShardSpec()).andReturn(EasyMock.createMock(ShardSpec.class)).once();
    EasyMock.expect(mockDataSegment1.getId()).andReturn(SegmentId.dummy("MockDataSource")).once();
    EasyMock.expect(mockDataSegment2.getInterval()).andReturn(Intervals.of("1971-01-01/1972-01-01")).once();
    EasyMock.expect(mockDataSegment2.getVersion()).andReturn("dummyString").once();
    EasyMock.expect(mockDataSegment2.getShardSpec()).andReturn(EasyMock.createMock(ShardSpec.class)).once();
    EasyMock.expect(mockDataSegment2.getId()).andReturn(SegmentId.dummy("MockDataSource")).once();
    EasyMock.expect(mockToolbox.getSegmentHandoffNotifierFactory()).andReturn(new NoopSegmentHandoffNotifierFactory()).once();
    EasyMock.expect(mockToolbox.getEmitter()).andReturn(latchEmitter).anyTimes();
    EasyMock.expect(mockDataSegment1.getDataSource()).andReturn("MockDataSource").once();
    EasyMock.replay(mockToolbox);
    EasyMock.replay(mockDataSegment1, mockDataSegment2);
    Assert.assertTrue(indexTask.waitForSegmentAvailability(mockToolbox, segmentsToWaitFor, 30000));
    latchEmitter.latch.await(300000, TimeUnit.MILLISECONDS);
    EasyMock.verify(mockToolbox);
    EasyMock.verify(mockDataSegment1, mockDataSegment2);
}

Example 57 with UniformGranularitySpec

use of org.apache.druid.segment.indexing.granularity.UniformGranularitySpec in project druid by druid-io.

the class IndexTaskTest method testOldSegmentNotDropWhenDropFlagTrueAndIngestionIntervalEmpty.

@Test
public void testOldSegmentNotDropWhenDropFlagTrueAndIngestionIntervalEmpty() throws Exception {
    File tmpDir = temporaryFolder.newFolder();
    File tmpFile = File.createTempFile("druid", "index", tmpDir);
    try (BufferedWriter writer = Files.newWriter(tmpFile, StandardCharsets.UTF_8)) {
        writer.write("2014-01-01T00:00:10Z,a,1\n");
        writer.write("2014-01-01T01:00:20Z,b,1\n");
        writer.write("2014-01-01T02:00:30Z,c,1\n");
    }
    IndexTask indexTask = new IndexTask(null, null, createDefaultIngestionSpec(jsonMapper, tmpDir, new UniformGranularitySpec(Granularities.YEAR, Granularities.MINUTE, Collections.singletonList(Intervals.of("2014-01-01/2014-01-02"))), null, createTuningConfigWithMaxRowsPerSegment(10, true), false, false), null);
    // Ingest data with YEAR segment granularity
    List<DataSegment> segments = runTask(indexTask).rhs;
    Assert.assertEquals(1, segments.size());
    Set<DataSegment> usedSegmentsBeforeOverwrite = Sets.newHashSet(getSegmentsMetadataManager().iterateAllUsedNonOvershadowedSegmentsForDatasourceInterval(DATASOURCE, Intervals.ETERNITY, true).get());
    Assert.assertEquals(1, usedSegmentsBeforeOverwrite.size());
    for (DataSegment segment : usedSegmentsBeforeOverwrite) {
        Assert.assertTrue(Granularities.YEAR.isAligned(segment.getInterval()));
    }
    indexTask = new IndexTask(null, null, createDefaultIngestionSpec(jsonMapper, tmpDir, new UniformGranularitySpec(Granularities.MINUTE, Granularities.MINUTE, null), null, createTuningConfigWithMaxRowsPerSegment(10, true), false, true), null);
    // Ingest data with overwrite and MINUTE segment granularity
    segments = runTask(indexTask).rhs;
    Assert.assertEquals(3, segments.size());
    Set<DataSegment> usedSegmentsBeforeAfterOverwrite = Sets.newHashSet(getSegmentsMetadataManager().iterateAllUsedNonOvershadowedSegmentsForDatasourceInterval(DATASOURCE, Intervals.ETERNITY, true).get());
    Assert.assertEquals(4, usedSegmentsBeforeAfterOverwrite.size());
    int yearSegmentFound = 0;
    int minuteSegmentFound = 0;
    for (DataSegment segment : usedSegmentsBeforeAfterOverwrite) {
        // and 3 new segments with MINUTE segmentGranularity (from second ingestion)
        if (usedSegmentsBeforeOverwrite.contains(segment)) {
            Assert.assertTrue(Granularities.YEAR.isAligned(segment.getInterval()));
            yearSegmentFound++;
        } else {
            Assert.assertTrue(Granularities.MINUTE.isAligned(segment.getInterval()));
            minuteSegmentFound++;
        }
    }
    Assert.assertEquals(1, yearSegmentFound);
    Assert.assertEquals(3, minuteSegmentFound);
}

Example 58 with UniformGranularitySpec

use of org.apache.druid.segment.indexing.granularity.UniformGranularitySpec in project druid by druid-io.

the class IndexTaskTest method testWithSmallMaxTotalRows.

@Test
public void testWithSmallMaxTotalRows() throws Exception {
    File tmpDir = temporaryFolder.newFolder();
    File tmpFile = File.createTempFile("druid", "index", tmpDir);
    try (BufferedWriter writer = Files.newWriter(tmpFile, StandardCharsets.UTF_8)) {
        writer.write("2014-01-01T00:00:10Z,a,1\n");
        writer.write("2014-01-01T00:00:10Z,b,2\n");
        writer.write("2014-01-01T00:00:10Z,c,3\n");
        writer.write("2014-01-01T01:00:20Z,a,1\n");
        writer.write("2014-01-01T01:00:20Z,b,2\n");
        writer.write("2014-01-01T01:00:20Z,c,3\n");
        writer.write("2014-01-01T02:00:30Z,a,1\n");
        writer.write("2014-01-01T02:00:30Z,b,2\n");
        writer.write("2014-01-01T02:00:30Z,c,3\n");
    }
    IndexTask indexTask = new IndexTask(null, null, createDefaultIngestionSpec(jsonMapper, tmpDir, new UniformGranularitySpec(Granularities.HOUR, Granularities.MINUTE, null), null, createTuningConfig(2, 2, null, 2L, null, false, true), false, false), null);
    final List<DataSegment> segments = runTask(indexTask).rhs;
    Assert.assertEquals(6, segments.size());
    for (int i = 0; i < 6; i++) {
        final DataSegment segment = segments.get(i);
        final Interval expectedInterval = Intervals.of(StringUtils.format("2014-01-01T0%d/PT1H", (i / 2)));
        final int expectedPartitionNum = i % 2;
        Assert.assertEquals(DATASOURCE, segment.getDataSource());
        Assert.assertEquals(expectedInterval, segment.getInterval());
        Assert.assertEquals(NumberedShardSpec.class, segment.getShardSpec().getClass());
        Assert.assertEquals(expectedPartitionNum, segment.getShardSpec().getPartitionNum());
    }
}

Example 59 with UniformGranularitySpec

use of org.apache.druid.segment.indexing.granularity.UniformGranularitySpec in project druid by druid-io.

the class IndexTaskTest method testOldSegmentNotDropWhenDropFlagFalse.

@Test
public void testOldSegmentNotDropWhenDropFlagFalse() throws Exception {
    File tmpDir = temporaryFolder.newFolder();
    File tmpFile = File.createTempFile("druid", "index", tmpDir);
    try (BufferedWriter writer = Files.newWriter(tmpFile, StandardCharsets.UTF_8)) {
        writer.write("2014-01-01T00:00:10Z,a,1\n");
        writer.write("2014-01-01T01:00:20Z,b,1\n");
        writer.write("2014-01-01T02:00:30Z,c,1\n");
    }
    IndexTask indexTask = new IndexTask(null, null, createDefaultIngestionSpec(jsonMapper, tmpDir, new UniformGranularitySpec(Granularities.YEAR, Granularities.MINUTE, Collections.singletonList(Intervals.of("2014-01-01/2014-01-02"))), null, createTuningConfigWithMaxRowsPerSegment(10, true), false, false), null);
    // Ingest data with YEAR segment granularity
    List<DataSegment> segments = runTask(indexTask).rhs;
    Assert.assertEquals(1, segments.size());
    Set<DataSegment> usedSegmentsBeforeOverwrite = Sets.newHashSet(getSegmentsMetadataManager().iterateAllUsedNonOvershadowedSegmentsForDatasourceInterval(DATASOURCE, Intervals.ETERNITY, true).get());
    Assert.assertEquals(1, usedSegmentsBeforeOverwrite.size());
    for (DataSegment segment : usedSegmentsBeforeOverwrite) {
        Assert.assertTrue(Granularities.YEAR.isAligned(segment.getInterval()));
    }
    indexTask = new IndexTask(null, null, createDefaultIngestionSpec(jsonMapper, tmpDir, new UniformGranularitySpec(Granularities.MINUTE, Granularities.MINUTE, Collections.singletonList(Intervals.of("2014-01-01/2014-01-02"))), null, createTuningConfigWithMaxRowsPerSegment(10, true), false, false), null);
    // Ingest data with overwrite and MINUTE segment granularity
    segments = runTask(indexTask).rhs;
    Assert.assertEquals(3, segments.size());
    Set<DataSegment> usedSegmentsBeforeAfterOverwrite = Sets.newHashSet(getSegmentsMetadataManager().iterateAllUsedNonOvershadowedSegmentsForDatasourceInterval(DATASOURCE, Intervals.ETERNITY, true).get());
    Assert.assertEquals(4, usedSegmentsBeforeAfterOverwrite.size());
    int yearSegmentFound = 0;
    int minuteSegmentFound = 0;
    for (DataSegment segment : usedSegmentsBeforeAfterOverwrite) {
        // and 3 new segments with MINUTE segmentGranularity (from second ingestion)
        if (usedSegmentsBeforeOverwrite.contains(segment)) {
            Assert.assertTrue(Granularities.YEAR.isAligned(segment.getInterval()));
            yearSegmentFound++;
        } else {
            Assert.assertTrue(Granularities.MINUTE.isAligned(segment.getInterval()));
            minuteSegmentFound++;
        }
    }
    Assert.assertEquals(1, yearSegmentFound);
    Assert.assertEquals(3, minuteSegmentFound);
}

Example 60 with UniformGranularitySpec

use of org.apache.druid.segment.indexing.granularity.UniformGranularitySpec in project druid by druid-io.

the class IndexTaskTest method testIntervalBucketing.

@Test
public void testIntervalBucketing() throws Exception {
    File tmpDir = temporaryFolder.newFolder();
    File tmpFile = File.createTempFile("druid", "index", tmpDir);
    try (BufferedWriter writer = Files.newWriter(tmpFile, StandardCharsets.UTF_8)) {
        writer.write("2014-01-01T07:59:59.977Z,a,1\n");
        writer.write("2014-01-01T08:00:00.000Z,b,1\n");
    }
    IndexTask indexTask = new IndexTask(null, null, createDefaultIngestionSpec(jsonMapper, tmpDir, new UniformGranularitySpec(Granularities.HOUR, Granularities.HOUR, Collections.singletonList(Intervals.of("2014-01-01T08:00:00Z/2014-01-01T09:00:00Z"))), null, createTuningConfigWithMaxRowsPerSegment(50, true), false, false), null);
    final List<DataSegment> segments = runTask(indexTask).rhs;
    Assert.assertEquals(1, segments.size());
}