Examples with DataSegment org.apache.druid.timeline.DataSegment used on opensource projects

Example 81 with DataSegment

use of org.apache.druid.timeline.DataSegment in project druid by druid-io.

the class ExportMetadata method makePayloadWithConvertedLoadSpec.

/**
 * Returns a new load spec in escaped JSON form, with the new deep storage location if configured.
 */
private String makePayloadWithConvertedLoadSpec(String payload) throws IOException {
    DataSegment segment = JSON_MAPPER.readValue(DatatypeConverter.parseHexBinary(payload), DataSegment.class);
    String uniqueId = getUniqueIDFromLocalLoadSpec(segment.getLoadSpec());
    String segmentPath = DataSegmentPusher.getDefaultStorageDirWithExistingUniquePath(segment, uniqueId);
    Map<String, Object> newLoadSpec = null;
    if (s3Bucket != null) {
        newLoadSpec = makeS3LoadSpec(segmentPath);
    } else if (hadoopStorageDirectory != null) {
        newLoadSpec = makeHDFSLoadSpec(segmentPath);
    } else if (newLocalPath != null) {
        newLoadSpec = makeLocalLoadSpec(segmentPath);
    }
    if (newLoadSpec != null) {
        segment = new DataSegment(segment.getDataSource(), segment.getInterval(), segment.getVersion(), newLoadSpec, segment.getDimensions(), segment.getMetrics(), segment.getShardSpec(), segment.getBinaryVersion(), segment.getSize());
    }
    String serialized = JSON_MAPPER.writeValueAsString(segment);
    if (useHexBlobs) {
        return DatatypeConverter.printHexBinary(StringUtils.toUtf8(serialized));
    } else {
        return escapeJSONForCSV(serialized);
    }
}

Example 82 with DataSegment

use of org.apache.druid.timeline.DataSegment in project druid by druid-io.

the class HistoricalMetricsMonitorTest method testSimple.

@Test
public void testSimple() {
    final long size = 5;
    final String dataSource = "dataSource";
    final DataSegment dataSegment = new DataSegment(dataSource, Intervals.of("2014/2015"), "version", ImmutableMap.of(), ImmutableList.of(), ImmutableList.of(), null, 1, size);
    final long maxSize = 10;
    final int priority = 111;
    final String tier = "tier";
    EasyMock.expect(druidServerConfig.getMaxSize()).andReturn(maxSize).once();
    EasyMock.expect(segmentLoadDropMgr.getPendingDeleteSnapshot()).andReturn(ImmutableList.of(dataSegment)).once();
    EasyMock.expect(druidServerConfig.getTier()).andReturn(tier).once();
    EasyMock.expect(druidServerConfig.getPriority()).andReturn(priority).once();
    EasyMock.expect(segmentManager.getDataSourceSizes()).andReturn(ImmutableMap.of(dataSource, size));
    EasyMock.expect(druidServerConfig.getTier()).andReturn(tier).once();
    EasyMock.expect(druidServerConfig.getPriority()).andReturn(priority).once();
    EasyMock.expect(druidServerConfig.getMaxSize()).andReturn(maxSize).times(2);
    EasyMock.expect(segmentManager.getDataSourceCounts()).andReturn(ImmutableMap.of(dataSource, 1L));
    EasyMock.expect(druidServerConfig.getTier()).andReturn(tier).once();
    EasyMock.expect(druidServerConfig.getPriority()).andReturn(priority).once();
    final HistoricalMetricsMonitor monitor = new HistoricalMetricsMonitor(druidServerConfig, segmentManager, segmentLoadDropMgr);
    final Capture<ServiceEventBuilder<ServiceMetricEvent>> eventCapture = EasyMock.newCapture(CaptureType.ALL);
    serviceEmitter.emit(EasyMock.capture(eventCapture));
    EasyMock.expectLastCall().times(5);
    EasyMock.replay(druidServerConfig, segmentManager, segmentLoadDropMgr, serviceEmitter);
    monitor.doMonitor(serviceEmitter);
    EasyMock.verify(druidServerConfig, segmentManager, segmentLoadDropMgr, serviceEmitter);
    final String host = "host";
    final String service = "service";
    Assert.assertTrue(eventCapture.hasCaptured());
    final List<Map<String, Object>> events = Lists.transform(eventCapture.getValues(), new Function<ServiceEventBuilder<ServiceMetricEvent>, Map<String, Object>>() {

        @Nullable
        @Override
        public Map<String, Object> apply(@Nullable ServiceEventBuilder<ServiceMetricEvent> input) {
            final HashMap<String, Object> map = new HashMap<>(input.build(service, host).toMap());
            Assert.assertNotNull(map.remove("feed"));
            Assert.assertNotNull(map.remove("timestamp"));
            Assert.assertNotNull(map.remove("service"));
            Assert.assertNotNull(map.remove("host"));
            return map;
        }
    });
    Assert.assertEquals(ImmutableMap.<String, Object>of("metric", "segment/max", "value", maxSize), events.get(0));
    Assert.assertEquals(ImmutableMap.<String, Object>of("dataSource", dataSource, "metric", "segment/pendingDelete", "priority", String.valueOf(priority), "tier", tier, "value", dataSegment.getSize()), events.get(1));
    Assert.assertEquals(ImmutableMap.<String, Object>of("metric", "segment/used", "value", dataSegment.getSize(), "tier", tier, "priority", String.valueOf(priority), "dataSource", dataSource), events.get(2));
    Assert.assertEquals(ImmutableMap.<String, Object>of("metric", "segment/usedPercent", "value", dataSegment.getSize() * 1.0D / maxSize, "tier", tier, "priority", String.valueOf(priority), "dataSource", dataSource), events.get(3));
    Assert.assertEquals(ImmutableMap.<String, Object>of("metric", "segment/count", "value", 1L, "tier", tier, "priority", String.valueOf(priority), "dataSource", dataSource), events.get(4));
}

Example 83 with DataSegment

use of org.apache.druid.timeline.DataSegment in project druid by druid-io.

the class CachingClusteredClientTest method testEtagforDifferentQueryInterval.

@Test
public void testEtagforDifferentQueryInterval() {
    final Interval interval = Intervals.of("2016-01-01/2016-01-02");
    final Interval queryInterval = Intervals.of("2016-01-01T14:00:00/2016-01-02T14:00:00");
    final Interval queryInterval2 = Intervals.of("2016-01-01T18:00:00/2016-01-02T18:00:00");
    final DataSegment dataSegment = new DataSegment("dataSource", interval, "ver", ImmutableMap.of("type", "hdfs", "path", "/tmp"), ImmutableList.of("product"), ImmutableList.of("visited_sum"), NoneShardSpec.instance(), 9, 12334);
    final ServerSelector selector = new ServerSelector(dataSegment, new HighestPriorityTierSelectorStrategy(new RandomServerSelectorStrategy()));
    selector.addServerAndUpdateSegment(new QueryableDruidServer(servers[0], null), dataSegment);
    timeline.add(interval, "ver", new SingleElementPartitionChunk<>(selector));
    final TimeBoundaryQuery query = Druids.newTimeBoundaryQueryBuilder().dataSource(DATA_SOURCE).intervals(new MultipleIntervalSegmentSpec(ImmutableList.of(queryInterval))).context(ImmutableMap.of("If-None-Match", "aVJV29CJY93rszVW/QBy0arWZo0=")).randomQueryId().build();
    final TimeBoundaryQuery query2 = Druids.newTimeBoundaryQueryBuilder().dataSource(DATA_SOURCE).intervals(new MultipleIntervalSegmentSpec(ImmutableList.of(queryInterval2))).context(ImmutableMap.of("If-None-Match", "aVJV29CJY93rszVW/QBy0arWZo0=")).randomQueryId().build();
    final ResponseContext responseContext = initializeResponseContext();
    getDefaultQueryRunner().run(QueryPlus.wrap(query), responseContext);
    final String etag1 = responseContext.getEntityTag();
    getDefaultQueryRunner().run(QueryPlus.wrap(query2), responseContext);
    final String etag2 = responseContext.getEntityTag();
    Assert.assertNotEquals(etag1, etag2);
}

Example 84 with DataSegment

use of org.apache.druid.timeline.DataSegment in project druid by druid-io.

the class CachingClusteredClientTest method testCachingOverBulkLimitEnforcesLimit.

@Test
@SuppressWarnings("unchecked")
public void testCachingOverBulkLimitEnforcesLimit() {
    final int limit = 10;
    final Interval interval = Intervals.of("2011-01-01/2011-01-02");
    final TimeseriesQuery query = Druids.newTimeseriesQueryBuilder().dataSource(DATA_SOURCE).intervals(new MultipleIntervalSegmentSpec(ImmutableList.of(interval))).filters(DIM_FILTER).granularity(GRANULARITY).aggregators(AGGS).postAggregators(POST_AGGS).context(CONTEXT).randomQueryId().build();
    final ResponseContext context = initializeResponseContext();
    final Cache cache = EasyMock.createStrictMock(Cache.class);
    final Capture<Iterable<Cache.NamedKey>> cacheKeyCapture = EasyMock.newCapture();
    EasyMock.expect(cache.getBulk(EasyMock.capture(cacheKeyCapture))).andReturn(ImmutableMap.of()).once();
    EasyMock.replay(cache);
    client = makeClient(new ForegroundCachePopulator(JSON_MAPPER, new CachePopulatorStats(), -1), cache, limit);
    final DruidServer lastServer = servers[random.nextInt(servers.length)];
    final DataSegment dataSegment = EasyMock.createNiceMock(DataSegment.class);
    EasyMock.expect(dataSegment.getId()).andReturn(SegmentId.dummy(DATA_SOURCE)).anyTimes();
    EasyMock.replay(dataSegment);
    final ServerSelector selector = new ServerSelector(dataSegment, new HighestPriorityTierSelectorStrategy(new RandomServerSelectorStrategy()));
    selector.addServerAndUpdateSegment(new QueryableDruidServer(lastServer, null), dataSegment);
    timeline.add(interval, "v", new SingleElementPartitionChunk<>(selector));
    getDefaultQueryRunner().run(QueryPlus.wrap(query), context);
    Assert.assertTrue("Capture cache keys", cacheKeyCapture.hasCaptured());
    Assert.assertTrue("Cache key below limit", ImmutableList.copyOf(cacheKeyCapture.getValue()).size() <= limit);
    EasyMock.verify(cache);
    EasyMock.reset(cache);
    cacheKeyCapture.reset();
    EasyMock.expect(cache.getBulk(EasyMock.capture(cacheKeyCapture))).andReturn(ImmutableMap.of()).once();
    EasyMock.replay(cache);
    client = makeClient(new ForegroundCachePopulator(JSON_MAPPER, new CachePopulatorStats(), -1), cache, 0);
    getDefaultQueryRunner().run(QueryPlus.wrap(query), context);
    EasyMock.verify(cache);
    EasyMock.verify(dataSegment);
    Assert.assertTrue("Capture cache keys", cacheKeyCapture.hasCaptured());
    Assert.assertTrue("Cache Keys empty", ImmutableList.copyOf(cacheKeyCapture.getValue()).isEmpty());
}

Example 85 with DataSegment

use of org.apache.druid.timeline.DataSegment in project druid by druid-io.

the class ImmutableDruidDataSourceTest method testSerde.

@Test
public void testSerde() throws IOException {
    final DataSegment segment = getTestSegment();
    final ImmutableDruidDataSource dataSource = getImmutableDruidDataSource(segment);
    final ObjectMapper objectMapper = new DefaultObjectMapper().setInjectableValues(new Std().addValue(PruneSpecsHolder.class, PruneSpecsHolder.DEFAULT));
    final String json = objectMapper.writeValueAsString(dataSource);
    ImmutableDruidDataSourceTestUtils.assertEquals(dataSource, objectMapper.readValue(json, ImmutableDruidDataSource.class));
}