Examples with ImmutableDruidDataSource org.apache.druid.client.ImmutableDruidDataSource used on opensource projects

Example 21 with ImmutableDruidDataSource

use of org.apache.druid.client.ImmutableDruidDataSource in project druid by druid-io.

the class DataSourcesResource method getServedSegment.

@GET
@Path("/{dataSourceName}/segments/{segmentId}")
@Produces(MediaType.APPLICATION_JSON)
@ResourceFilters(DatasourceResourceFilter.class)
public Response getServedSegment(@PathParam("dataSourceName") String dataSourceName, @PathParam("segmentId") String segmentId) {
    ImmutableDruidDataSource dataSource = getDataSource(dataSourceName);
    if (dataSource == null) {
        return logAndCreateDataSourceNotFoundResponse(dataSourceName);
    }
    for (SegmentId possibleSegmentId : SegmentId.iteratePossibleParsingsWithDataSource(dataSourceName, segmentId)) {
        Pair<DataSegment, Set<String>> retVal = getServersWhereSegmentIsServed(possibleSegmentId);
        if (retVal != null) {
            return Response.ok(ImmutableMap.of("metadata", retVal.lhs, "servers", retVal.rhs)).build();
        }
    }
    log.warn("Segment id [%s] is unknown", segmentId);
    return Response.noContent().build();
}

Example 22 with ImmutableDruidDataSource

use of org.apache.druid.client.ImmutableDruidDataSource in project druid by druid-io.

the class DataSourcesResource method getIntervalsWithServedSegmentsOrAllServedSegmentsPerIntervals.

@GET
@Path("/{dataSourceName}/intervals")
@Produces(MediaType.APPLICATION_JSON)
@ResourceFilters(DatasourceResourceFilter.class)
public Response getIntervalsWithServedSegmentsOrAllServedSegmentsPerIntervals(@PathParam("dataSourceName") String dataSourceName, @QueryParam("simple") String simple, @QueryParam("full") String full) {
    if (simple == null && full == null) {
        final ImmutableDruidDataSource dataSource = getDataSource(dataSourceName);
        if (dataSource == null) {
            return logAndCreateDataSourceNotFoundResponse(dataSourceName);
        }
        final Comparator<Interval> comparator = Comparators.intervalsByStartThenEnd().reversed();
        Set<Interval> intervals = new TreeSet<>(comparator);
        dataSource.getSegments().forEach(segment -> intervals.add(segment.getInterval()));
        return Response.ok(intervals).build();
    } else {
        return getServedSegmentsInInterval(dataSourceName, full != null, interval -> true);
    }
}

Example 23 with ImmutableDruidDataSource

use of org.apache.druid.client.ImmutableDruidDataSource in project druid by druid-io.

the class DataSourcesResource method getServedSegmentsInInterval.

@GET
@Path("/{dataSourceName}/intervals/{interval}")
@Produces(MediaType.APPLICATION_JSON)
@ResourceFilters(DatasourceResourceFilter.class)
public Response getServedSegmentsInInterval(@PathParam("dataSourceName") String dataSourceName, @PathParam("interval") String interval, @QueryParam("simple") String simple, @QueryParam("full") String full) {
    final Interval theInterval = Intervals.of(interval.replace('_', '/'));
    if (simple == null && full == null) {
        final ImmutableDruidDataSource dataSource = getDataSource(dataSourceName);
        if (dataSource == null) {
            return logAndCreateDataSourceNotFoundResponse(dataSourceName);
        }
        final Set<SegmentId> segmentIds = new TreeSet<>();
        for (DataSegment dataSegment : dataSource.getSegments()) {
            if (theInterval.contains(dataSegment.getInterval())) {
                segmentIds.add(dataSegment.getId());
            }
        }
        return Response.ok(segmentIds).build();
    }
    return getServedSegmentsInInterval(dataSourceName, full != null, theInterval::contains);
}

Example 24 with ImmutableDruidDataSource

use of org.apache.druid.client.ImmutableDruidDataSource in project druid by druid-io.

the class DruidCoordinator method getLoadStatus.

public Map<String, Double> getLoadStatus() {
    final Map<String, Double> loadStatus = new HashMap<>();
    final Collection<ImmutableDruidDataSource> dataSources = segmentsMetadataManager.getImmutableDataSourcesWithAllUsedSegments();
    for (ImmutableDruidDataSource dataSource : dataSources) {
        final Set<DataSegment> segments = Sets.newHashSet(dataSource.getSegments());
        final int numPublishedSegments = segments.size();
        // remove loaded segments
        for (DruidServer druidServer : serverInventoryView.getInventory()) {
            final DruidDataSource loadedView = druidServer.getDataSource(dataSource.getName());
            if (loadedView != null) {
                // Please see https://github.com/apache/druid/pull/5632 and LoadStatusBenchmark for more info.
                for (DataSegment serverSegment : loadedView.getSegments()) {
                    segments.remove(serverSegment);
                }
            }
        }
        final int numUnavailableSegments = segments.size();
        loadStatus.put(dataSource.getName(), 100 * ((double) (numPublishedSegments - numUnavailableSegments) / (double) numPublishedSegments));
    }
    return loadStatus;
}

Example 25 with ImmutableDruidDataSource

use of org.apache.druid.client.ImmutableDruidDataSource in project druid by druid-io.

the class RunRules method run.

@Override
public DruidCoordinatorRuntimeParams run(DruidCoordinatorRuntimeParams params) {
    replicatorThrottler.updateParams(coordinator.getDynamicConfigs().getReplicationThrottleLimit(), coordinator.getDynamicConfigs().getReplicantLifetime(), false);
    CoordinatorStats stats = new CoordinatorStats();
    DruidCluster cluster = params.getDruidCluster();
    if (cluster.isEmpty()) {
        log.warn("Uh... I have no servers. Not assigning anything...");
        return params;
    }
    // Get used segments which are overshadowed by other used segments. Those would not need to be loaded and
    // eventually will be unloaded from Historical servers. Segments overshadowed by *served* used segments are marked
    // as unused in MarkAsUnusedOvershadowedSegments, and then eventually Coordinator sends commands to Historical nodes
    // to unload such segments in UnloadUnusedSegments.
    Set<SegmentId> overshadowed = params.getDataSourcesSnapshot().getOvershadowedSegments();
    for (String tier : cluster.getTierNames()) {
        replicatorThrottler.updateReplicationState(tier);
    }
    DruidCoordinatorRuntimeParams paramsWithReplicationManager = params.buildFromExistingWithoutSegmentsMetadata().withReplicationManager(replicatorThrottler).build();
    // Run through all matched rules for used segments
    DateTime now = DateTimes.nowUtc();
    MetadataRuleManager databaseRuleManager = paramsWithReplicationManager.getDatabaseRuleManager();
    final List<SegmentId> segmentsWithMissingRules = Lists.newArrayListWithCapacity(MAX_MISSING_RULES);
    int missingRules = 0;
    final Set<String> broadcastDatasources = new HashSet<>();
    for (ImmutableDruidDataSource dataSource : params.getDataSourcesSnapshot().getDataSourcesMap().values()) {
        List<Rule> rules = databaseRuleManager.getRulesWithDefault(dataSource.getName());
        for (Rule rule : rules) {
            // executes before BalanceSegments.
            if (rule instanceof BroadcastDistributionRule) {
                broadcastDatasources.add(dataSource.getName());
                break;
            }
        }
    }
    for (DataSegment segment : params.getUsedSegments()) {
        if (overshadowed.contains(segment.getId())) {
            // Skipping overshadowed segments
            continue;
        }
        List<Rule> rules = databaseRuleManager.getRulesWithDefault(segment.getDataSource());
        boolean foundMatchingRule = false;
        for (Rule rule : rules) {
            if (rule.appliesTo(segment, now)) {
                if (stats.getGlobalStat("totalNonPrimaryReplicantsLoaded") >= paramsWithReplicationManager.getCoordinatorDynamicConfig().getMaxNonPrimaryReplicantsToLoad() && !paramsWithReplicationManager.getReplicationManager().isLoadPrimaryReplicantsOnly()) {
                    log.info("Maximum number of non-primary replicants [%d] have been loaded for the current RunRules execution. Only loading primary replicants from here on for this coordinator run cycle.", paramsWithReplicationManager.getCoordinatorDynamicConfig().getMaxNonPrimaryReplicantsToLoad());
                    paramsWithReplicationManager.getReplicationManager().setLoadPrimaryReplicantsOnly(true);
                }
                stats.accumulate(rule.run(coordinator, paramsWithReplicationManager, segment));
                foundMatchingRule = true;
                break;
            }
        }
        if (!foundMatchingRule) {
            if (segmentsWithMissingRules.size() < MAX_MISSING_RULES) {
                segmentsWithMissingRules.add(segment.getId());
            }
            missingRules++;
        }
    }
    if (!segmentsWithMissingRules.isEmpty()) {
        log.makeAlert("Unable to find matching rules!").addData("segmentsWithMissingRulesCount", missingRules).addData("segmentsWithMissingRules", segmentsWithMissingRules).emit();
    }
    return params.buildFromExisting().withCoordinatorStats(stats).withBroadcastDatasources(broadcastDatasources).build();
}