Examples with ServerHolder org.apache.druid.server.coordinator.ServerHolder used on opensource projects

Example 6 with ServerHolder

use of org.apache.druid.server.coordinator.ServerHolder in project druid by druid-io.

the class LoadRule method assignReplicasForTier.

/**
 * @param predicate {@link Predicate} used to pre-filter {@link ServerHolder}s retrieved from {@link DruidCluster}.
 */
private int assignReplicasForTier(final String tier, final int targetReplicantsInTier, final int currentReplicantsInTier, final DruidCoordinatorRuntimeParams params, final Predicate<ServerHolder> predicate, final DataSegment segment) {
    final int numToAssign = targetReplicantsInTier - currentReplicantsInTier;
    // if nothing to assign
    if (numToAssign <= 0) {
        return 0;
    }
    String noAvailability = StringUtils.format("No available [%s] servers or node capacity to assign segment[%s]! Expected Replicants[%d]", tier, segment.getId(), targetReplicantsInTier);
    final List<ServerHolder> holders = getFilteredHolders(tier, params.getDruidCluster(), predicate);
    // if no holders available for assignment
    if (holders.isEmpty()) {
        log.warn(noAvailability);
        return 0;
    }
    final ReplicationThrottler throttler = params.getReplicationManager();
    for (int numAssigned = 0; numAssigned < numToAssign; numAssigned++) {
        if (!throttler.canCreateReplicant(tier)) {
            log.info("Throttling replication for segment [%s] in tier [%s]", segment.getId(), tier);
            return numAssigned;
        }
        // Retrieves from cache if available
        ServerHolder holder = strategyCache.remove(tier);
        // Does strategy call if not in cache
        if (holder == null) {
            holder = params.getBalancerStrategy().findNewSegmentHomeReplicator(segment, holders);
        }
        if (holder == null) {
            log.warn(noAvailability);
            return numAssigned;
        }
        holders.remove(holder);
        final SegmentId segmentId = segment.getId();
        final String holderHost = holder.getServer().getHost();
        throttler.registerReplicantCreation(tier, segmentId, holderHost);
        log.info("Assigning 'replica' for segment [%s] to server [%s] in tier [%s]", segment.getId(), holder.getServer().getName(), holder.getServer().getTier());
        holder.getPeon().loadSegment(segment, () -> throttler.unregisterReplicantCreation(tier, segmentId));
    }
    return numToAssign;
}

Example 7 with ServerHolder

use of org.apache.druid.server.coordinator.ServerHolder in project druid by druid-io.

the class LoadRule method assign.

/**
 * @param stats {@link CoordinatorStats} to accumulate assignment statistics.
 */
private void assign(final DruidCoordinatorRuntimeParams params, final DataSegment segment, final CoordinatorStats stats) {
    // if primary replica already exists or is loading
    final int loading = params.getSegmentReplicantLookup().getTotalReplicants(segment.getId());
    if (!currentReplicants.isEmpty() || loading > 0) {
        assignReplicas(params, segment, stats, null);
    } else {
        final ServerHolder primaryHolderToLoad = assignPrimary(params, segment);
        if (primaryHolderToLoad == null) {
            // this implies that no assignment could be done
            return;
        }
        // 1 replica (i.e., primary replica) already assigned
        int numAssigned = 1;
        final String tier = primaryHolderToLoad.getServer().getTier();
        // assign replicas for the rest of the tier
        numAssigned += assignReplicasForTier(tier, targetReplicants.getOrDefault(tier, 0), // note that the currentReplicantsInTier is the just-assigned primary replica.
        numAssigned, params, createLoadQueueSizeLimitingPredicate(params).and(holder -> !holder.equals(primaryHolderToLoad)), segment);
        // numAssigned - 1 because we don't want to count the primary assignment
        stats.addToGlobalStat(NON_PRIMARY_ASSIGNED_COUNT, numAssigned - 1);
        stats.addToTieredStat(ASSIGNED_COUNT, tier, numAssigned);
        // do assign replicas for the other tiers.
        assignReplicas(params, segment, stats, tier);
    }
}

Example 8 with ServerHolder

use of org.apache.druid.server.coordinator.ServerHolder in project druid by druid-io.

the class LoadRuleTest method testDrop.

@Test
public void testDrop() {
    final LoadQueuePeon mockPeon = createEmptyPeon();
    mockPeon.dropSegment(EasyMock.anyObject(), EasyMock.anyObject());
    EasyMock.expectLastCall().atLeastOnce();
    EasyMock.expect(mockBalancerStrategy.pickServersToDrop(EasyMock.anyObject(), EasyMock.anyObject())).andDelegateTo(balancerStrategy).times(4);
    EasyMock.replay(throttler, mockPeon, mockBalancerStrategy);
    LoadRule rule = createLoadRule(ImmutableMap.of("hot", 0, DruidServer.DEFAULT_TIER, 0));
    final DataSegment segment = createDataSegment("foo");
    DruidServer server1 = new DruidServer("serverHot", "hostHot", null, 1000, ServerType.HISTORICAL, "hot", 0);
    server1.addDataSegment(segment);
    DruidServer server2 = new DruidServer("serverNorm", "hostNorm", null, 1000, ServerType.HISTORICAL, DruidServer.DEFAULT_TIER, 0);
    server2.addDataSegment(segment);
    DruidServer server3 = new DruidServer("serverNormNotServing", "hostNorm", null, 10, ServerType.HISTORICAL, DruidServer.DEFAULT_TIER, 0);
    DruidCluster druidCluster = DruidClusterBuilder.newBuilder().addTier("hot", new ServerHolder(server1.toImmutableDruidServer(), mockPeon)).addTier(DruidServer.DEFAULT_TIER, new ServerHolder(server2.toImmutableDruidServer(), mockPeon), new ServerHolder(server3.toImmutableDruidServer(), mockPeon)).build();
    CoordinatorStats stats = rule.run(null, makeCoordinatorRuntimeParams(druidCluster, segment), segment);
    Assert.assertEquals(1L, stats.getTieredStat("droppedCount", "hot"));
    Assert.assertEquals(1L, stats.getTieredStat("droppedCount", DruidServer.DEFAULT_TIER));
    EasyMock.verify(throttler, mockPeon);
}

Example 9 with ServerHolder

use of org.apache.druid.server.coordinator.ServerHolder in project druid by druid-io.

the class LoadRuleTest method testLoadReplicaDuringDecommissioning.

/**
 * 2 tiers, 2 servers each, 1 server of the second tier is decommissioning.
 * Should not load a segment to the server that is decommssioning.
 */
@Test
public void testLoadReplicaDuringDecommissioning() {
    EasyMock.expect(throttler.canCreateReplicant(EasyMock.anyString())).andReturn(true).anyTimes();
    final LoadQueuePeon mockPeon1 = createEmptyPeon();
    final LoadQueuePeon mockPeon2 = createOneCallPeonMock();
    final LoadQueuePeon mockPeon3 = createOneCallPeonMock();
    final LoadQueuePeon mockPeon4 = createOneCallPeonMock();
    LoadRule rule = createLoadRule(ImmutableMap.of("tier1", 2, "tier2", 2));
    final DataSegment segment = createDataSegment("foo");
    throttler.registerReplicantCreation(EasyMock.eq("tier2"), EasyMock.anyObject(), EasyMock.anyObject());
    EasyMock.expectLastCall().times(2);
    ServerHolder holder1 = createServerHolder("tier1", mockPeon1, true);
    ServerHolder holder2 = createServerHolder("tier1", mockPeon2, false);
    ServerHolder holder3 = createServerHolder("tier2", mockPeon3, false);
    ServerHolder holder4 = createServerHolder("tier2", mockPeon4, false);
    EasyMock.expect(mockBalancerStrategy.findNewSegmentHomeReplicator(segment, ImmutableList.of(holder2))).andReturn(holder2);
    EasyMock.expect(mockBalancerStrategy.findNewSegmentHomeReplicator(segment, ImmutableList.of(holder4, holder3))).andReturn(holder3);
    EasyMock.expect(mockBalancerStrategy.findNewSegmentHomeReplicator(segment, ImmutableList.of(holder4))).andReturn(holder4);
    EasyMock.replay(throttler, mockPeon1, mockPeon2, mockPeon3, mockPeon4, mockBalancerStrategy);
    DruidCluster druidCluster = DruidClusterBuilder.newBuilder().addTier("tier1", holder1, holder2).addTier("tier2", holder3, holder4).build();
    CoordinatorStats stats = rule.run(null, makeCoordinatorRuntimeParams(druidCluster, segment), segment);
    Assert.assertEquals(1L, stats.getTieredStat(LoadRule.ASSIGNED_COUNT, "tier1"));
    Assert.assertEquals(2L, stats.getTieredStat(LoadRule.ASSIGNED_COUNT, "tier2"));
    EasyMock.verify(throttler, mockPeon1, mockPeon2, mockPeon3, mockPeon4, mockBalancerStrategy);
}

Example 10 with ServerHolder

use of org.apache.druid.server.coordinator.ServerHolder in project druid by druid-io.

the class LoadRuleTest method testRedundantReplicaDropDuringDecommissioning.

/**
 * 3 servers hosting 3 replicas of the segment.
 * 1 servers is decommissioning.
 * 1 replica is redundant.
 * Should drop from the decommissioning server.
 */
@Test
public void testRedundantReplicaDropDuringDecommissioning() {
    final LoadQueuePeon mockPeon1 = new LoadQueuePeonTester();
    final LoadQueuePeon mockPeon2 = new LoadQueuePeonTester();
    final LoadQueuePeon mockPeon3 = new LoadQueuePeonTester();
    EasyMock.expect(mockBalancerStrategy.pickServersToDrop(EasyMock.anyObject(), EasyMock.anyObject())).andDelegateTo(balancerStrategy).times(4);
    EasyMock.replay(throttler, mockBalancerStrategy);
    LoadRule rule = createLoadRule(ImmutableMap.of("tier1", 2));
    final DataSegment segment1 = createDataSegment("foo1");
    DruidServer server1 = createServer("tier1");
    server1.addDataSegment(segment1);
    DruidServer server2 = createServer("tier1");
    server2.addDataSegment(segment1);
    DruidServer server3 = createServer("tier1");
    server3.addDataSegment(segment1);
    DruidCluster druidCluster = DruidClusterBuilder.newBuilder().addTier("tier1", new ServerHolder(server1.toImmutableDruidServer(), mockPeon1, false), new ServerHolder(server2.toImmutableDruidServer(), mockPeon2, true), new ServerHolder(server3.toImmutableDruidServer(), mockPeon3, false)).build();
    CoordinatorStats stats = rule.run(null, makeCoordinatorRuntimeParams(druidCluster, segment1), segment1);
    Assert.assertEquals(1L, stats.getTieredStat("droppedCount", "tier1"));
    Assert.assertEquals(0, mockPeon1.getSegmentsToDrop().size());
    Assert.assertEquals(1, mockPeon2.getSegmentsToDrop().size());
    Assert.assertEquals(0, mockPeon3.getSegmentsToDrop().size());
    EasyMock.verify(throttler);
}