Examples with SegmentId org.apache.druid.timeline.SegmentId used on opensource projects

Example 36 with SegmentId

use of org.apache.druid.timeline.SegmentId in project druid by druid-io.

the class SqlSegmentsMetadataQuery method markSegments.

/**
 * Marks the provided segments as either used or unused.
 *
 * Returns the number of segments actually modified.
 */
public int markSegments(final Collection<SegmentId> segmentIds, final boolean used) {
    final String dataSource;
    if (segmentIds.isEmpty()) {
        return 0;
    } else {
        dataSource = segmentIds.iterator().next().getDataSource();
        if (segmentIds.stream().anyMatch(segment -> !dataSource.equals(segment.getDataSource()))) {
            throw new IAE("Segments to drop must all be part of the same datasource");
        }
    }
    final PreparedBatch batch = handle.prepareBatch(StringUtils.format("UPDATE %s SET used = ? WHERE datasource = ? AND id = ?", dbTables.getSegmentsTable()));
    for (SegmentId segmentId : segmentIds) {
        batch.add(used, dataSource, segmentId.toString());
    }
    final int[] segmentChanges = batch.execute();
    return computeNumChangedSegments(segmentIds.stream().map(SegmentId::toString).collect(Collectors.toList()), segmentChanges);
}

Example 37 with SegmentId

use of org.apache.druid.timeline.SegmentId in project druid by druid-io.

the class DataSourcesResource method getSimpleDatasource.

private Map<String, Map<String, Object>> getSimpleDatasource(String dataSourceName) {
    Map<String, Object> tiers = new HashMap<>();
    Map<String, Object> segments = new HashMap<>();
    Map<String, Map<String, Object>> retVal = ImmutableMap.of("tiers", tiers, "segments", segments);
    Set<SegmentId> totalDistinctSegments = new HashSet<>();
    Map<String, HashSet<Object>> tierDistinctSegments = new HashMap<>();
    long totalSegmentSize = 0;
    long totalReplicatedSize = 0;
    DateTime minTime = DateTimes.MAX;
    DateTime maxTime = DateTimes.MIN;
    String tier;
    for (DruidServer druidServer : serverInventoryView.getInventory()) {
        DruidDataSource druidDataSource = druidServer.getDataSource(dataSourceName);
        tier = druidServer.getTier();
        if (druidDataSource == null) {
            continue;
        }
        tierDistinctSegments.computeIfAbsent(tier, t -> new HashSet<>());
        long dataSourceSegmentSize = 0;
        long replicatedSegmentSize = 0;
        for (DataSegment dataSegment : druidDataSource.getSegments()) {
            // tier segments stats
            if (!tierDistinctSegments.get(tier).contains(dataSegment.getId())) {
                dataSourceSegmentSize += dataSegment.getSize();
                tierDistinctSegments.get(tier).add(dataSegment.getId());
            }
            // total segments stats
            if (totalDistinctSegments.add(dataSegment.getId())) {
                totalSegmentSize += dataSegment.getSize();
                minTime = DateTimes.min(minTime, dataSegment.getInterval().getStart());
                maxTime = DateTimes.max(maxTime, dataSegment.getInterval().getEnd());
            }
            totalReplicatedSize += dataSegment.getSize();
            replicatedSegmentSize += dataSegment.getSize();
        }
        // tier stats
        Map<String, Object> tierStats = (Map) tiers.get(tier);
        if (tierStats == null) {
            tierStats = new HashMap<>();
            tiers.put(druidServer.getTier(), tierStats);
        }
        tierStats.put("segmentCount", tierDistinctSegments.get(tier).size());
        long segmentSize = MapUtils.getLong(tierStats, "size", 0L);
        tierStats.put("size", segmentSize + dataSourceSegmentSize);
        long replicatedSize = MapUtils.getLong(tierStats, "replicatedSize", 0L);
        tierStats.put("replicatedSize", replicatedSize + replicatedSegmentSize);
    }
    segments.put("count", totalDistinctSegments.size());
    segments.put("size", totalSegmentSize);
    segments.put("replicatedSize", totalReplicatedSize);
    segments.put("minTime", minTime);
    segments.put("maxTime", maxTime);
    return retVal;
}

Example 38 with SegmentId

use of org.apache.druid.timeline.SegmentId in project druid by druid-io.

the class DataSourcesResource method getServedSegmentsInInterval.

private Response getServedSegmentsInInterval(String dataSourceName, boolean full, Predicate<Interval> intervalFilter) {
    final ImmutableDruidDataSource dataSource = getDataSource(dataSourceName);
    if (dataSource == null) {
        return logAndCreateDataSourceNotFoundResponse(dataSourceName);
    }
    final Comparator<Interval> comparator = Comparators.intervalsByStartThenEnd().reversed();
    if (full) {
        final Map<Interval, Map<SegmentId, Object>> retVal = new TreeMap<>(comparator);
        for (DataSegment dataSegment : dataSource.getSegments()) {
            if (intervalFilter.test(dataSegment.getInterval())) {
                Map<SegmentId, Object> segments = retVal.computeIfAbsent(dataSegment.getInterval(), i -> new HashMap<>());
                Pair<DataSegment, Set<String>> segmentAndServers = getServersWhereSegmentIsServed(dataSegment.getId());
                if (segmentAndServers != null) {
                    segments.put(dataSegment.getId(), ImmutableMap.of("metadata", segmentAndServers.lhs, "servers", segmentAndServers.rhs));
                }
            }
        }
        return Response.ok(retVal).build();
    } else {
        final Map<Interval, Map<SimpleProperties, Object>> statsPerInterval = new TreeMap<>(comparator);
        for (DataSegment dataSegment : dataSource.getSegments()) {
            if (intervalFilter.test(dataSegment.getInterval())) {
                Map<SimpleProperties, Object> properties = statsPerInterval.computeIfAbsent(dataSegment.getInterval(), i -> new EnumMap<>(SimpleProperties.class));
                properties.merge(SimpleProperties.size, dataSegment.getSize(), (a, b) -> (Long) a + (Long) b);
                properties.merge(SimpleProperties.count, 1, (a, b) -> (Integer) a + (Integer) b);
            }
        }
        return Response.ok(statsPerInterval).build();
    }
}

Example 39 with SegmentId

use of org.apache.druid.timeline.SegmentId in project druid by druid-io.

the class DataSourcesResource method getServedSegment.

@GET
@Path("/{dataSourceName}/segments/{segmentId}")
@Produces(MediaType.APPLICATION_JSON)
@ResourceFilters(DatasourceResourceFilter.class)
public Response getServedSegment(@PathParam("dataSourceName") String dataSourceName, @PathParam("segmentId") String segmentId) {
    ImmutableDruidDataSource dataSource = getDataSource(dataSourceName);
    if (dataSource == null) {
        return logAndCreateDataSourceNotFoundResponse(dataSourceName);
    }
    for (SegmentId possibleSegmentId : SegmentId.iteratePossibleParsingsWithDataSource(dataSourceName, segmentId)) {
        Pair<DataSegment, Set<String>> retVal = getServersWhereSegmentIsServed(possibleSegmentId);
        if (retVal != null) {
            return Response.ok(ImmutableMap.of("metadata", retVal.lhs, "servers", retVal.rhs)).build();
        }
    }
    log.warn("Segment id [%s] is unknown", segmentId);
    return Response.noContent().build();
}

Example 40 with SegmentId

use of org.apache.druid.timeline.SegmentId in project druid by druid-io.

the class DataSourcesResource method getServedSegmentsInInterval.

@GET
@Path("/{dataSourceName}/intervals/{interval}")
@Produces(MediaType.APPLICATION_JSON)
@ResourceFilters(DatasourceResourceFilter.class)
public Response getServedSegmentsInInterval(@PathParam("dataSourceName") String dataSourceName, @PathParam("interval") String interval, @QueryParam("simple") String simple, @QueryParam("full") String full) {
    final Interval theInterval = Intervals.of(interval.replace('_', '/'));
    if (simple == null && full == null) {
        final ImmutableDruidDataSource dataSource = getDataSource(dataSourceName);
        if (dataSource == null) {
            return logAndCreateDataSourceNotFoundResponse(dataSourceName);
        }
        final Set<SegmentId> segmentIds = new TreeSet<>();
        for (DataSegment dataSegment : dataSource.getSegments()) {
            if (theInterval.contains(dataSegment.getInterval())) {
                segmentIds.add(dataSegment.getId());
            }
        }
        return Response.ok(segmentIds).build();
    }
    return getServedSegmentsInInterval(dataSourceName, full != null, theInterval::contains);
}