Search in sources :

Example 1 with LoadQueuePeon

use of org.apache.druid.server.coordinator.LoadQueuePeon in project druid by druid-io.

the class UnloadUnusedSegments method handleUnusedSegmentsForServer.

private void handleUnusedSegmentsForServer(ServerHolder serverHolder, Set<DataSegment> usedSegments, DruidCoordinatorRuntimeParams params, CoordinatorStats stats, boolean dropBroadcastOnly, Map<String, Boolean> broadcastStatusByDatasource) {
    ImmutableDruidServer server = serverHolder.getServer();
    for (ImmutableDruidDataSource dataSource : server.getDataSources()) {
        boolean isBroadcastDatasource = broadcastStatusByDatasource.computeIfAbsent(dataSource.getName(), (dataSourceName) -> {
            List<Rule> rules = params.getDatabaseRuleManager().getRulesWithDefault(dataSource.getName());
            for (Rule rule : rules) {
                // A datasource is considered a broadcast datasource if it has any broadcast rules.
                if (rule instanceof BroadcastDistributionRule) {
                    return true;
                }
            }
            return false;
        });
        // the set of segments that were created by a task/indexer here, and exclude them.
        if (dropBroadcastOnly && !isBroadcastDatasource) {
            continue;
        }
        for (DataSegment segment : dataSource.getSegments()) {
            if (!usedSegments.contains(segment)) {
                LoadQueuePeon queuePeon = params.getLoadManagementPeons().get(server.getName());
                if (!queuePeon.getSegmentsToDrop().contains(segment)) {
                    queuePeon.dropSegment(segment, () -> {
                    });
                    stats.addToTieredStat("unneededCount", server.getTier(), 1);
                    log.info("Dropping uneeded segment [%s] from server [%s] in tier [%s]", segment.getId(), server.getName(), server.getTier());
                }
            }
        }
    }
}
Also used : ImmutableDruidDataSource(org.apache.druid.client.ImmutableDruidDataSource) BroadcastDistributionRule(org.apache.druid.server.coordinator.rules.BroadcastDistributionRule) LoadQueuePeon(org.apache.druid.server.coordinator.LoadQueuePeon) BroadcastDistributionRule(org.apache.druid.server.coordinator.rules.BroadcastDistributionRule) Rule(org.apache.druid.server.coordinator.rules.Rule) DataSegment(org.apache.druid.timeline.DataSegment) ImmutableDruidServer(org.apache.druid.client.ImmutableDruidServer)

Example 2 with LoadQueuePeon

use of org.apache.druid.server.coordinator.LoadQueuePeon in project druid by druid-io.

the class LoadRuleTest method testDrop.

@Test
public void testDrop() {
    final LoadQueuePeon mockPeon = createEmptyPeon();
    mockPeon.dropSegment(EasyMock.anyObject(), EasyMock.anyObject());
    EasyMock.expectLastCall().atLeastOnce();
    EasyMock.expect(mockBalancerStrategy.pickServersToDrop(EasyMock.anyObject(), EasyMock.anyObject())).andDelegateTo(balancerStrategy).times(4);
    EasyMock.replay(throttler, mockPeon, mockBalancerStrategy);
    LoadRule rule = createLoadRule(ImmutableMap.of("hot", 0, DruidServer.DEFAULT_TIER, 0));
    final DataSegment segment = createDataSegment("foo");
    DruidServer server1 = new DruidServer("serverHot", "hostHot", null, 1000, ServerType.HISTORICAL, "hot", 0);
    server1.addDataSegment(segment);
    DruidServer server2 = new DruidServer("serverNorm", "hostNorm", null, 1000, ServerType.HISTORICAL, DruidServer.DEFAULT_TIER, 0);
    server2.addDataSegment(segment);
    DruidServer server3 = new DruidServer("serverNormNotServing", "hostNorm", null, 10, ServerType.HISTORICAL, DruidServer.DEFAULT_TIER, 0);
    DruidCluster druidCluster = DruidClusterBuilder.newBuilder().addTier("hot", new ServerHolder(server1.toImmutableDruidServer(), mockPeon)).addTier(DruidServer.DEFAULT_TIER, new ServerHolder(server2.toImmutableDruidServer(), mockPeon), new ServerHolder(server3.toImmutableDruidServer(), mockPeon)).build();
    CoordinatorStats stats = rule.run(null, makeCoordinatorRuntimeParams(druidCluster, segment), segment);
    Assert.assertEquals(1L, stats.getTieredStat("droppedCount", "hot"));
    Assert.assertEquals(1L, stats.getTieredStat("droppedCount", DruidServer.DEFAULT_TIER));
    EasyMock.verify(throttler, mockPeon);
}
Also used : CoordinatorStats(org.apache.druid.server.coordinator.CoordinatorStats) ServerHolder(org.apache.druid.server.coordinator.ServerHolder) LoadQueuePeon(org.apache.druid.server.coordinator.LoadQueuePeon) DruidServer(org.apache.druid.client.DruidServer) ImmutableDruidServer(org.apache.druid.client.ImmutableDruidServer) DruidCluster(org.apache.druid.server.coordinator.DruidCluster) DataSegment(org.apache.druid.timeline.DataSegment) Test(org.junit.Test)

Example 3 with LoadQueuePeon

use of org.apache.druid.server.coordinator.LoadQueuePeon in project druid by druid-io.

the class LoadRuleTest method testLoadDecommissioning.

/**
 * 2 servers in different tiers, the first is decommissioning.
 * Should not load a segment to the server that is decommissioning
 */
@Test
public void testLoadDecommissioning() {
    final LoadQueuePeon mockPeon1 = createEmptyPeon();
    final LoadQueuePeon mockPeon2 = createOneCallPeonMock();
    LoadRule rule = createLoadRule(ImmutableMap.of("tier1", 1, "tier2", 1));
    final DataSegment segment = createDataSegment("foo");
    EasyMock.expect(mockBalancerStrategy.findNewSegmentHomeReplicator(EasyMock.anyObject(), EasyMock.anyObject())).andDelegateTo(balancerStrategy).times(1);
    EasyMock.replay(mockPeon1, mockPeon2, mockBalancerStrategy);
    DruidCluster druidCluster = DruidClusterBuilder.newBuilder().addTier("tier1", createServerHolder("tier1", mockPeon1, true)).addTier("tier2", createServerHolder("tier2", mockPeon2, false)).build();
    CoordinatorStats stats = rule.run(null, makeCoordinatorRuntimeParams(druidCluster, segment), segment);
    Assert.assertEquals(1L, stats.getTieredStat(LoadRule.ASSIGNED_COUNT, "tier2"));
    EasyMock.verify(mockPeon1, mockPeon2, mockBalancerStrategy);
}
Also used : CoordinatorStats(org.apache.druid.server.coordinator.CoordinatorStats) LoadQueuePeon(org.apache.druid.server.coordinator.LoadQueuePeon) DruidCluster(org.apache.druid.server.coordinator.DruidCluster) DataSegment(org.apache.druid.timeline.DataSegment) Test(org.junit.Test)

Example 4 with LoadQueuePeon

use of org.apache.druid.server.coordinator.LoadQueuePeon in project druid by druid-io.

the class LoadRuleTest method testLoadReplicaDuringDecommissioning.

/**
 * 2 tiers, 2 servers each, 1 server of the second tier is decommissioning.
 * Should not load a segment to the server that is decommssioning.
 */
@Test
public void testLoadReplicaDuringDecommissioning() {
    EasyMock.expect(throttler.canCreateReplicant(EasyMock.anyString())).andReturn(true).anyTimes();
    final LoadQueuePeon mockPeon1 = createEmptyPeon();
    final LoadQueuePeon mockPeon2 = createOneCallPeonMock();
    final LoadQueuePeon mockPeon3 = createOneCallPeonMock();
    final LoadQueuePeon mockPeon4 = createOneCallPeonMock();
    LoadRule rule = createLoadRule(ImmutableMap.of("tier1", 2, "tier2", 2));
    final DataSegment segment = createDataSegment("foo");
    throttler.registerReplicantCreation(EasyMock.eq("tier2"), EasyMock.anyObject(), EasyMock.anyObject());
    EasyMock.expectLastCall().times(2);
    ServerHolder holder1 = createServerHolder("tier1", mockPeon1, true);
    ServerHolder holder2 = createServerHolder("tier1", mockPeon2, false);
    ServerHolder holder3 = createServerHolder("tier2", mockPeon3, false);
    ServerHolder holder4 = createServerHolder("tier2", mockPeon4, false);
    EasyMock.expect(mockBalancerStrategy.findNewSegmentHomeReplicator(segment, ImmutableList.of(holder2))).andReturn(holder2);
    EasyMock.expect(mockBalancerStrategy.findNewSegmentHomeReplicator(segment, ImmutableList.of(holder4, holder3))).andReturn(holder3);
    EasyMock.expect(mockBalancerStrategy.findNewSegmentHomeReplicator(segment, ImmutableList.of(holder4))).andReturn(holder4);
    EasyMock.replay(throttler, mockPeon1, mockPeon2, mockPeon3, mockPeon4, mockBalancerStrategy);
    DruidCluster druidCluster = DruidClusterBuilder.newBuilder().addTier("tier1", holder1, holder2).addTier("tier2", holder3, holder4).build();
    CoordinatorStats stats = rule.run(null, makeCoordinatorRuntimeParams(druidCluster, segment), segment);
    Assert.assertEquals(1L, stats.getTieredStat(LoadRule.ASSIGNED_COUNT, "tier1"));
    Assert.assertEquals(2L, stats.getTieredStat(LoadRule.ASSIGNED_COUNT, "tier2"));
    EasyMock.verify(throttler, mockPeon1, mockPeon2, mockPeon3, mockPeon4, mockBalancerStrategy);
}
Also used : CoordinatorStats(org.apache.druid.server.coordinator.CoordinatorStats) ServerHolder(org.apache.druid.server.coordinator.ServerHolder) LoadQueuePeon(org.apache.druid.server.coordinator.LoadQueuePeon) DruidCluster(org.apache.druid.server.coordinator.DruidCluster) DataSegment(org.apache.druid.timeline.DataSegment) Test(org.junit.Test)

Example 5 with LoadQueuePeon

use of org.apache.druid.server.coordinator.LoadQueuePeon in project druid by druid-io.

the class LoadRuleTest method createLoadingPeon.

private static LoadQueuePeon createLoadingPeon(List<DataSegment> segments, boolean slowLoading) {
    final Set<DataSegment> segs = ImmutableSet.copyOf(segments);
    final long loadingSize = segs.stream().mapToLong(DataSegment::getSize).sum();
    final LoadQueuePeon mockPeon = EasyMock.createMock(LoadQueuePeon.class);
    EasyMock.expect(mockPeon.getSegmentsToLoad()).andReturn(segs).anyTimes();
    EasyMock.expect(mockPeon.getSegmentsMarkedToDrop()).andReturn(new HashSet<>()).anyTimes();
    EasyMock.expect(mockPeon.getLoadQueueSize()).andReturn(loadingSize).anyTimes();
    EasyMock.expect(mockPeon.getNumberOfSegmentsInQueue()).andReturn(segs.size()).anyTimes();
    if (slowLoading) {
        EasyMock.expect(mockPeon.getTimedOutSegments()).andReturn(new HashSet<>(segments)).anyTimes();
    } else {
        EasyMock.expect(mockPeon.getTimedOutSegments()).andReturn(new HashSet<>()).anyTimes();
    }
    return mockPeon;
}
Also used : LoadQueuePeon(org.apache.druid.server.coordinator.LoadQueuePeon) DataSegment(org.apache.druid.timeline.DataSegment) HashSet(java.util.HashSet)

Aggregations

LoadQueuePeon (org.apache.druid.server.coordinator.LoadQueuePeon)19 DataSegment (org.apache.druid.timeline.DataSegment)17 ImmutableDruidServer (org.apache.druid.client.ImmutableDruidServer)14 CoordinatorStats (org.apache.druid.server.coordinator.CoordinatorStats)14 DruidCluster (org.apache.druid.server.coordinator.DruidCluster)14 ServerHolder (org.apache.druid.server.coordinator.ServerHolder)14 Test (org.junit.Test)13 DruidServer (org.apache.druid.client.DruidServer)11 HashSet (java.util.HashSet)2 DruidCoordinatorRuntimeParams (org.apache.druid.server.coordinator.DruidCoordinatorRuntimeParams)2 Object2IntMap (it.unimi.dsi.fastutil.objects.Object2IntMap)1 Object2LongMap (it.unimi.dsi.fastutil.objects.Object2LongMap)1 ImmutableDruidDataSource (org.apache.druid.client.ImmutableDruidDataSource)1 Logger (org.apache.druid.java.util.common.logger.Logger)1 ServiceEmitter (org.apache.druid.java.util.emitter.service.ServiceEmitter)1 ServiceMetricEvent (org.apache.druid.java.util.emitter.service.ServiceMetricEvent)1 DruidMetrics (org.apache.druid.query.DruidMetrics)1 BalancerSegmentHolder (org.apache.druid.server.coordinator.BalancerSegmentHolder)1 DruidCoordinator (org.apache.druid.server.coordinator.DruidCoordinator)1 LoadPeonCallback (org.apache.druid.server.coordinator.LoadPeonCallback)1