Examples with LoadPeonCallback io.druid.server.coordinator.LoadPeonCallback used on opensource projects

Example 1 with LoadPeonCallback

use of io.druid.server.coordinator.LoadPeonCallback in project druid by druid-io.

the class LoadRule method assign.

private CoordinatorStats assign(final ReplicationThrottler replicationManager, final String tier, final int totalReplicantsInCluster, final int expectedReplicantsInTier, final int totalReplicantsInTier, final BalancerStrategy strategy, final List<ServerHolder> serverHolderList, final DataSegment segment) {
    final CoordinatorStats stats = new CoordinatorStats();
    stats.addToTieredStat(assignedCount, tier, 0);
    int currReplicantsInTier = totalReplicantsInTier;
    int currTotalReplicantsInCluster = totalReplicantsInCluster;
    while (currReplicantsInTier < expectedReplicantsInTier) {
        boolean replicate = currTotalReplicantsInCluster > 0;
        if (replicate && !replicationManager.canCreateReplicant(tier)) {
            break;
        }
        final ServerHolder holder = strategy.findNewSegmentHomeReplicator(segment, serverHolderList);
        if (holder == null) {
            log.warn("Not enough [%s] servers or node capacity to assign segment[%s]! Expected Replicants[%d]", tier, segment.getIdentifier(), expectedReplicantsInTier);
            break;
        }
        if (replicate) {
            replicationManager.registerReplicantCreation(tier, segment.getIdentifier(), holder.getServer().getHost());
        }
        holder.getPeon().loadSegment(segment, new LoadPeonCallback() {

            @Override
            public void execute() {
                replicationManager.unregisterReplicantCreation(tier, segment.getIdentifier(), holder.getServer().getHost());
            }
        });
        stats.addToTieredStat(assignedCount, tier, 1);
        ++currReplicantsInTier;
        ++currTotalReplicantsInCluster;
    }
    return stats;
}

Example 2 with LoadPeonCallback

use of io.druid.server.coordinator.LoadPeonCallback in project druid by druid-io.

the class DruidCoordinatorCleanupUnneeded method run.

@Override
public DruidCoordinatorRuntimeParams run(DruidCoordinatorRuntimeParams params) {
    CoordinatorStats stats = new CoordinatorStats();
    Set<DataSegment> availableSegments = params.getAvailableSegments();
    DruidCluster cluster = params.getDruidCluster();
    // cleanup before it finished polling the metadata storage for available segments for the first time.
    if (!availableSegments.isEmpty()) {
        for (MinMaxPriorityQueue<ServerHolder> serverHolders : cluster.getSortedServersByTier()) {
            for (ServerHolder serverHolder : serverHolders) {
                ImmutableDruidServer server = serverHolder.getServer();
                for (ImmutableDruidDataSource dataSource : server.getDataSources()) {
                    for (DataSegment segment : dataSource.getSegments()) {
                        if (!availableSegments.contains(segment)) {
                            LoadQueuePeon queuePeon = params.getLoadManagementPeons().get(server.getName());
                            if (!queuePeon.getSegmentsToDrop().contains(segment)) {
                                queuePeon.dropSegment(segment, new LoadPeonCallback() {

                                    @Override
                                    public void execute() {
                                    }
                                });
                                stats.addToTieredStat("unneededCount", server.getTier(), 1);
                            }
                        }
                    }
                }
            }
        }
    } else {
        log.info("Found 0 availableSegments, skipping the cleanup of segments from historicals. This is done to prevent a race condition in which the coordinator would drop all segments if it started running cleanup before it finished polling the metadata storage for available segments for the first time.");
    }
    return params.buildFromExisting().withCoordinatorStats(stats).build();
}

Example 3 with LoadPeonCallback

use of io.druid.server.coordinator.LoadPeonCallback in project druid by druid-io.

the class DruidCoordinatorBalancer method moveSegment.

protected void moveSegment(final BalancerSegmentHolder segment, final ImmutableDruidServer toServer, final DruidCoordinatorRuntimeParams params) {
    final LoadQueuePeon toPeon = params.getLoadManagementPeons().get(toServer.getName());
    final ImmutableDruidServer fromServer = segment.getFromServer();
    final DataSegment segmentToMove = segment.getSegment();
    final String segmentName = segmentToMove.getIdentifier();
    if (!toPeon.getSegmentsToLoad().contains(segmentToMove) && (toServer.getSegment(segmentName) == null) && new ServerHolder(toServer, toPeon).getAvailableSize() > segmentToMove.getSize()) {
        log.info("Moving [%s] from [%s] to [%s]", segmentName, fromServer.getName(), toServer.getName());
        LoadPeonCallback callback = null;
        try {
            currentlyMovingSegments.get(toServer.getTier()).put(segmentName, segment);
            callback = new LoadPeonCallback() {

                @Override
                public void execute() {
                    Map<String, BalancerSegmentHolder> movingSegments = currentlyMovingSegments.get(toServer.getTier());
                    if (movingSegments != null) {
                        movingSegments.remove(segmentName);
                    }
                }
            };
            coordinator.moveSegment(fromServer, toServer, segmentToMove.getIdentifier(), callback);
        } catch (Exception e) {
            log.makeAlert(e, String.format("[%s] : Moving exception", segmentName)).emit();
            if (callback != null) {
                callback.execute();
            }
        }
    } else {
        currentlyMovingSegments.get(toServer.getTier()).remove(segmentName);
    }
}