Examples with BalancerSegmentHolder org.apache.druid.server.coordinator.BalancerSegmentHolder used on opensource projects

Example 1 with BalancerSegmentHolder

use of org.apache.druid.server.coordinator.BalancerSegmentHolder in project druid by druid-io.

the class BalanceSegments method moveSegment.

protected boolean moveSegment(final BalancerSegmentHolder segment, final ImmutableDruidServer toServer, final DruidCoordinatorRuntimeParams params) {
    final LoadQueuePeon toPeon = params.getLoadManagementPeons().get(toServer.getName());
    final ImmutableDruidServer fromServer = segment.getFromServer();
    final DataSegment segmentToMove = segment.getSegment();
    final SegmentId segmentId = segmentToMove.getId();
    if (!toPeon.getSegmentsToLoad().contains(segmentToMove) && (toServer.getSegment(segmentId) == null) && new ServerHolder(toServer, toPeon).getAvailableSize() > segmentToMove.getSize()) {
        log.debug("Moving [%s] from [%s] to [%s]", segmentId, fromServer.getName(), toServer.getName());
        LoadPeonCallback callback = null;
        try {
            ConcurrentMap<SegmentId, BalancerSegmentHolder> movingSegments = currentlyMovingSegments.get(toServer.getTier());
            movingSegments.put(segmentId, segment);
            callback = () -> movingSegments.remove(segmentId);
            coordinator.moveSegment(params, fromServer, toServer, segmentToMove, callback);
            return true;
        } catch (Exception e) {
            log.makeAlert(e, StringUtils.format("[%s] : Moving exception", segmentId)).emit();
            if (callback != null) {
                callback.execute();
            }
        }
    }
    return false;
}

Example 2 with BalancerSegmentHolder

use of org.apache.druid.server.coordinator.BalancerSegmentHolder in project druid by druid-io.

the class BalanceSegments method balanceServers.

private Pair<Integer, Integer> balanceServers(DruidCoordinatorRuntimeParams params, List<ServerHolder> toMoveFrom, List<ServerHolder> toMoveTo, int maxSegmentsToMove) {
    if (maxSegmentsToMove <= 0) {
        log.debug("maxSegmentsToMove is 0; no balancing work can be performed.");
        return new Pair<>(0, 0);
    } else if (toMoveFrom.isEmpty()) {
        log.debug("toMoveFrom is empty; no balancing work can be performed.");
        return new Pair<>(0, 0);
    } else if (toMoveTo.isEmpty()) {
        log.debug("toMoveTo is empty; no balancing work can be peformed.");
        return new Pair<>(0, 0);
    }
    final BalancerStrategy strategy = params.getBalancerStrategy();
    final int maxIterations = 2 * maxSegmentsToMove;
    final int maxToLoad = params.getCoordinatorDynamicConfig().getMaxSegmentsInNodeLoadingQueue();
    int moved = 0, unmoved = 0;
    Iterator<BalancerSegmentHolder> segmentsToMove;
    // The pick method depends on if the operator has enabled batched segment sampling in the Coorinator dynamic config.
    if (params.getCoordinatorDynamicConfig().useBatchedSegmentSampler()) {
        segmentsToMove = strategy.pickSegmentsToMove(toMoveFrom, params.getBroadcastDatasources(), maxSegmentsToMove);
    } else {
        segmentsToMove = strategy.pickSegmentsToMove(toMoveFrom, params.getBroadcastDatasources(), params.getCoordinatorDynamicConfig().getPercentOfSegmentsToConsiderPerMove());
    }
    // noinspection ForLoopThatDoesntUseLoopVariable
    for (int iter = 0; (moved + unmoved) < maxSegmentsToMove; ++iter) {
        if (!segmentsToMove.hasNext()) {
            log.info("All servers to move segments from are empty, ending run.");
            break;
        }
        final BalancerSegmentHolder segmentToMoveHolder = segmentsToMove.next();
        // DruidCoordinatorRuntimeParams.getUsedSegments originate from SegmentsMetadataManager, i. e. that's a set of segments
        // that *should* be loaded. segmentToMoveHolder.getSegment originates from ServerInventoryView,  i. e. that may be
        // any segment that happens to be loaded on some server, even if it is not used. (Coordinator closes such
        // discrepancies eventually via UnloadUnusedSegments). Therefore the picked segmentToMoveHolder's segment may not
        // need to be balanced.
        boolean needToBalancePickedSegment = params.getUsedSegments().contains(segmentToMoveHolder.getSegment());
        if (needToBalancePickedSegment) {
            final DataSegment segmentToMove = segmentToMoveHolder.getSegment();
            final ImmutableDruidServer fromServer = segmentToMoveHolder.getFromServer();
            // we want to leave the server the segment is currently on in the list...
            // but filter out replicas that are already serving the segment, and servers with a full load queue
            final List<ServerHolder> toMoveToWithLoadQueueCapacityAndNotServingSegment = toMoveTo.stream().filter(s -> s.getServer().equals(fromServer) || (!s.isServingSegment(segmentToMove) && (maxToLoad <= 0 || s.getNumberOfSegmentsInQueue() < maxToLoad))).collect(Collectors.toList());
            if (toMoveToWithLoadQueueCapacityAndNotServingSegment.size() > 0) {
                final ServerHolder destinationHolder = strategy.findNewSegmentHomeBalancer(segmentToMove, toMoveToWithLoadQueueCapacityAndNotServingSegment);
                if (destinationHolder != null && !destinationHolder.getServer().equals(fromServer)) {
                    if (moveSegment(segmentToMoveHolder, destinationHolder.getServer(), params)) {
                        moved++;
                    } else {
                        unmoved++;
                    }
                } else {
                    log.debug("Segment [%s] is 'optimally' placed.", segmentToMove.getId());
                    unmoved++;
                }
            } else {
                log.debug("No valid movement destinations for segment [%s].", segmentToMove.getId());
                unmoved++;
            }
        }
        if (iter >= maxIterations) {
            log.info("Unable to select %d remaining candidate segments out of %d total to balance " + "after %d iterations, ending run.", (maxSegmentsToMove - moved - unmoved), maxSegmentsToMove, iter);
            break;
        }
    }
    return new Pair<>(moved, unmoved);
}