Examples with DataSegment io.druid.timeline.DataSegment used on opensource projects

Example 31 with DataSegment

use of io.druid.timeline.DataSegment in project druid by druid-io.

the class ServersResource method getServerSegment.

@GET
@Path("/{serverName}/segments/{segmentId}")
@Produces(MediaType.APPLICATION_JSON)
public Response getServerSegment(@PathParam("serverName") String serverName, @PathParam("segmentId") String segmentId) {
    DruidServer server = serverInventoryView.getInventoryValue(serverName);
    if (server == null) {
        return Response.status(Response.Status.NOT_FOUND).build();
    }
    DataSegment segment = server.getSegment(segmentId);
    if (segment == null) {
        return Response.status(Response.Status.NOT_FOUND).build();
    }
    return Response.status(Response.Status.OK).entity(segment).build();
}

Example 32 with DataSegment

use of io.druid.timeline.DataSegment in project druid by druid-io.

the class TiersResource method getTierDatasources.

@GET
@Path("/{tierName}")
@Produces(MediaType.APPLICATION_JSON)
public Response getTierDatasources(@PathParam("tierName") String tierName, @QueryParam("simple") String simple) {
    if (simple != null) {
        Table<String, Interval, Map<String, Object>> retVal = HashBasedTable.create();
        for (DruidServer druidServer : serverInventoryView.getInventory()) {
            if (druidServer.getTier().equalsIgnoreCase(tierName)) {
                for (DataSegment dataSegment : druidServer.getSegments().values()) {
                    Map<String, Object> properties = retVal.get(dataSegment.getDataSource(), dataSegment.getInterval());
                    if (properties == null) {
                        properties = Maps.newHashMap();
                        retVal.put(dataSegment.getDataSource(), dataSegment.getInterval(), properties);
                    }
                    properties.put("size", MapUtils.getLong(properties, "size", 0L) + dataSegment.getSize());
                    properties.put("count", MapUtils.getInt(properties, "count", 0) + 1);
                }
            }
        }
        return Response.ok(retVal.rowMap()).build();
    }
    Set<String> retVal = Sets.newHashSet();
    for (DruidServer druidServer : serverInventoryView.getInventory()) {
        if (druidServer.getTier().equalsIgnoreCase(tierName)) {
            retVal.addAll(Lists.newArrayList(Iterables.transform(druidServer.getDataSources(), new Function<DruidDataSource, String>() {

                @Override
                public String apply(DruidDataSource input) {
                    return input.getName();
                }
            })));
        }
    }
    return Response.ok(retVal).build();
}

Example 33 with DataSegment

use of io.druid.timeline.DataSegment in project druid by druid-io.

the class DruidCoordinatorCleanupUnneeded method run.

@Override
public DruidCoordinatorRuntimeParams run(DruidCoordinatorRuntimeParams params) {
    CoordinatorStats stats = new CoordinatorStats();
    Set<DataSegment> availableSegments = params.getAvailableSegments();
    DruidCluster cluster = params.getDruidCluster();
    // cleanup before it finished polling the metadata storage for available segments for the first time.
    if (!availableSegments.isEmpty()) {
        for (MinMaxPriorityQueue<ServerHolder> serverHolders : cluster.getSortedServersByTier()) {
            for (ServerHolder serverHolder : serverHolders) {
                ImmutableDruidServer server = serverHolder.getServer();
                for (ImmutableDruidDataSource dataSource : server.getDataSources()) {
                    for (DataSegment segment : dataSource.getSegments()) {
                        if (!availableSegments.contains(segment)) {
                            LoadQueuePeon queuePeon = params.getLoadManagementPeons().get(server.getName());
                            if (!queuePeon.getSegmentsToDrop().contains(segment)) {
                                queuePeon.dropSegment(segment, new LoadPeonCallback() {

                                    @Override
                                    public void execute() {
                                    }
                                });
                                stats.addToTieredStat("unneededCount", server.getTier(), 1);
                            }
                        }
                    }
                }
            }
        }
    } else {
        log.info("Found 0 availableSegments, skipping the cleanup of segments from historicals. This is done to prevent a race condition in which the coordinator would drop all segments if it started running cleanup before it finished polling the metadata storage for available segments for the first time.");
    }
    return params.buildFromExisting().withCoordinatorStats(stats).build();
}

Example 34 with DataSegment

use of io.druid.timeline.DataSegment in project druid by druid-io.

the class DruidCoordinatorRuleRunner method run.

@Override
public DruidCoordinatorRuntimeParams run(DruidCoordinatorRuntimeParams params) {
    replicatorThrottler.updateParams(coordinator.getDynamicConfigs().getReplicationThrottleLimit(), coordinator.getDynamicConfigs().getReplicantLifetime());
    CoordinatorStats stats = new CoordinatorStats();
    DruidCluster cluster = params.getDruidCluster();
    if (cluster.isEmpty()) {
        log.warn("Uh... I have no servers. Not assigning anything...");
        return params;
    }
    // find available segments which are not overshadowed by other segments in DB
    // only those would need to be loaded/dropped
    // anything overshadowed by served segments is dropped automatically by DruidCoordinatorCleanupOvershadowed
    Map<String, VersionedIntervalTimeline<String, DataSegment>> timelines = new HashMap<>();
    for (DataSegment segment : params.getAvailableSegments()) {
        VersionedIntervalTimeline<String, DataSegment> timeline = timelines.get(segment.getDataSource());
        if (timeline == null) {
            timeline = new VersionedIntervalTimeline<>(Comparators.comparable());
            timelines.put(segment.getDataSource(), timeline);
        }
        timeline.add(segment.getInterval(), segment.getVersion(), segment.getShardSpec().createChunk(segment));
    }
    Set<DataSegment> overshadowed = new HashSet<>();
    for (VersionedIntervalTimeline<String, DataSegment> timeline : timelines.values()) {
        for (TimelineObjectHolder<String, DataSegment> holder : timeline.findOvershadowed()) {
            for (DataSegment dataSegment : holder.getObject().payloads()) {
                overshadowed.add(dataSegment);
            }
        }
    }
    Set<DataSegment> nonOvershadowed = new HashSet<>();
    for (DataSegment dataSegment : params.getAvailableSegments()) {
        if (!overshadowed.contains(dataSegment)) {
            nonOvershadowed.add(dataSegment);
        }
    }
    for (String tier : cluster.getTierNames()) {
        replicatorThrottler.updateReplicationState(tier);
    }
    DruidCoordinatorRuntimeParams paramsWithReplicationManager = params.buildFromExistingWithoutAvailableSegments().withReplicationManager(replicatorThrottler).withAvailableSegments(nonOvershadowed).build();
    // Run through all matched rules for available segments
    DateTime now = new DateTime();
    MetadataRuleManager databaseRuleManager = paramsWithReplicationManager.getDatabaseRuleManager();
    final List<String> segmentsWithMissingRules = Lists.newArrayListWithCapacity(MAX_MISSING_RULES);
    int missingRules = 0;
    for (DataSegment segment : paramsWithReplicationManager.getAvailableSegments()) {
        List<Rule> rules = databaseRuleManager.getRulesWithDefault(segment.getDataSource());
        boolean foundMatchingRule = false;
        for (Rule rule : rules) {
            if (rule.appliesTo(segment, now)) {
                stats.accumulate(rule.run(coordinator, paramsWithReplicationManager, segment));
                foundMatchingRule = true;
                break;
            }
        }
        if (!foundMatchingRule) {
            if (segmentsWithMissingRules.size() < MAX_MISSING_RULES) {
                segmentsWithMissingRules.add(segment.getIdentifier());
            }
            missingRules++;
        }
    }
    if (!segmentsWithMissingRules.isEmpty()) {
        log.makeAlert("Unable to find matching rules!").addData("segmentsWithMissingRulesCount", missingRules).addData("segmentsWithMissingRules", segmentsWithMissingRules).emit();
    }
    return paramsWithReplicationManager.buildFromExistingWithoutAvailableSegments().withCoordinatorStats(stats).withAvailableSegments(params.getAvailableSegments()).build();
}

Example 35 with DataSegment

use of io.druid.timeline.DataSegment in project druid by druid-io.

the class DruidCoordinatorVersionConverter method run.

@Override
public DruidCoordinatorRuntimeParams run(DruidCoordinatorRuntimeParams params) {
    DatasourceWhitelist whitelist = whitelistRef.get();
    for (DataSegment dataSegment : params.getAvailableSegments()) {
        if (whitelist == null || whitelist.contains(dataSegment.getDataSource())) {
            final Integer binaryVersion = dataSegment.getBinaryVersion();
            if (binaryVersion == null || binaryVersion < IndexIO.CURRENT_VERSION_ID) {
                log.info("Upgrading version on segment[%s]", dataSegment.getIdentifier());
                indexingServiceClient.upgradeSegment(dataSegment);
            }
        }
    }
    return params;
}