use of org.apache.druid.client.ImmutableDruidServer in project druid by druid-io.
the class UnloadUnusedSegments method handleUnusedSegmentsForServer.
private void handleUnusedSegmentsForServer(ServerHolder serverHolder, Set<DataSegment> usedSegments, DruidCoordinatorRuntimeParams params, CoordinatorStats stats, boolean dropBroadcastOnly, Map<String, Boolean> broadcastStatusByDatasource) {
ImmutableDruidServer server = serverHolder.getServer();
for (ImmutableDruidDataSource dataSource : server.getDataSources()) {
boolean isBroadcastDatasource = broadcastStatusByDatasource.computeIfAbsent(dataSource.getName(), (dataSourceName) -> {
List<Rule> rules = params.getDatabaseRuleManager().getRulesWithDefault(dataSource.getName());
for (Rule rule : rules) {
// A datasource is considered a broadcast datasource if it has any broadcast rules.
if (rule instanceof BroadcastDistributionRule) {
return true;
}
}
return false;
});
// the set of segments that were created by a task/indexer here, and exclude them.
if (dropBroadcastOnly && !isBroadcastDatasource) {
continue;
}
for (DataSegment segment : dataSource.getSegments()) {
if (!usedSegments.contains(segment)) {
LoadQueuePeon queuePeon = params.getLoadManagementPeons().get(server.getName());
if (!queuePeon.getSegmentsToDrop().contains(segment)) {
queuePeon.dropSegment(segment, () -> {
});
stats.addToTieredStat("unneededCount", server.getTier(), 1);
log.info("Dropping uneeded segment [%s] from server [%s] in tier [%s]", segment.getId(), server.getName(), server.getTier());
}
}
}
}
}
use of org.apache.druid.client.ImmutableDruidServer in project druid by druid-io.
the class LoadRuleTest method testLoadPrimaryAssignDoesNotOverAssign.
@Test
public void testLoadPrimaryAssignDoesNotOverAssign() {
EasyMock.expect(throttler.canCreateReplicant(EasyMock.anyString())).andReturn(true).anyTimes();
final LoadQueuePeon mockPeon = createEmptyPeon();
mockPeon.loadSegment(EasyMock.anyObject(), EasyMock.anyObject());
EasyMock.expectLastCall().atLeastOnce();
LoadRule rule = createLoadRule(ImmutableMap.of("hot", 1));
final DataSegment segment = createDataSegment("foo");
EasyMock.expect(mockBalancerStrategy.findNewSegmentHomeReplicator(EasyMock.anyObject(), EasyMock.anyObject())).andDelegateTo(balancerStrategy).anyTimes();
EasyMock.replay(throttler, mockPeon, mockBalancerStrategy);
ImmutableDruidServer server1 = new DruidServer("serverHot", "hostHot", null, 1000, ServerType.HISTORICAL, "hot", 1).toImmutableDruidServer();
ImmutableDruidServer server2 = new DruidServer("serverHot2", "hostHot2", null, 1000, ServerType.HISTORICAL, "hot", 1).toImmutableDruidServer();
DruidCluster druidCluster = DruidClusterBuilder.newBuilder().addTier("hot", new ServerHolder(server1, mockPeon), new ServerHolder(server2, mockPeon)).build();
CoordinatorStats stats = rule.run(null, makeCoordinatorRuntimeParams(druidCluster, segment), segment);
Assert.assertEquals(1L, stats.getTieredStat(LoadRule.ASSIGNED_COUNT, "hot"));
// ensure multiple runs don't assign primary segment again if at replication count
final LoadQueuePeon loadingPeon = createLoadingPeon(ImmutableList.of(segment), false);
EasyMock.replay(loadingPeon);
DruidCluster afterLoad = DruidClusterBuilder.newBuilder().addTier("hot", new ServerHolder(server1, loadingPeon), new ServerHolder(server2, mockPeon)).build();
CoordinatorStats statsAfterLoadPrimary = rule.run(null, makeCoordinatorRuntimeParams(afterLoad, segment), segment);
Assert.assertEquals(0, statsAfterLoadPrimary.getTieredStat(LoadRule.ASSIGNED_COUNT, "hot"));
EasyMock.verify(throttler, mockPeon, mockBalancerStrategy);
}
use of org.apache.druid.client.ImmutableDruidServer in project druid by druid-io.
the class LoadRuleTest method testOverAssignForTimedOutSegments.
@Test
public void testOverAssignForTimedOutSegments() {
EasyMock.expect(throttler.canCreateReplicant(EasyMock.anyString())).andReturn(true).anyTimes();
final LoadQueuePeon emptyPeon = createEmptyPeon();
emptyPeon.loadSegment(EasyMock.anyObject(), EasyMock.anyObject());
EasyMock.expectLastCall().atLeastOnce();
LoadRule rule = createLoadRule(ImmutableMap.of("hot", 1));
final DataSegment segment = createDataSegment("foo");
EasyMock.expect(mockBalancerStrategy.findNewSegmentHomeReplicator(EasyMock.anyObject(), EasyMock.anyObject())).andDelegateTo(balancerStrategy).anyTimes();
EasyMock.replay(throttler, emptyPeon, mockBalancerStrategy);
ImmutableDruidServer server1 = new DruidServer("serverHot", "hostHot", null, 1000, ServerType.HISTORICAL, "hot", 1).toImmutableDruidServer();
ImmutableDruidServer server2 = new DruidServer("serverHot2", "hostHot2", null, 1000, ServerType.HISTORICAL, "hot", 1).toImmutableDruidServer();
DruidCluster druidCluster = DruidClusterBuilder.newBuilder().addTier("hot", new ServerHolder(server1, emptyPeon), new ServerHolder(server2, emptyPeon)).build();
CoordinatorStats stats = rule.run(null, makeCoordinatorRuntimeParamsWithLoadReplicationOnTimeout(druidCluster, segment), segment);
// Ensure that the segment is assigned to one of the historicals
Assert.assertEquals(1L, stats.getTieredStat(LoadRule.ASSIGNED_COUNT, "hot"));
// Ensure that the primary segment is assigned again in case the peon timed out on loading the segment
final LoadQueuePeon slowLoadingPeon = createLoadingPeon(ImmutableList.of(segment), true);
EasyMock.replay(slowLoadingPeon);
DruidCluster withLoadTimeout = DruidClusterBuilder.newBuilder().addTier("hot", new ServerHolder(server1, slowLoadingPeon), new ServerHolder(server2, emptyPeon)).build();
CoordinatorStats statsAfterLoadPrimary = rule.run(null, makeCoordinatorRuntimeParamsWithLoadReplicationOnTimeout(withLoadTimeout, segment), segment);
Assert.assertEquals(1L, statsAfterLoadPrimary.getTieredStat(LoadRule.ASSIGNED_COUNT, "hot"));
EasyMock.verify(throttler, emptyPeon, mockBalancerStrategy);
}
use of org.apache.druid.client.ImmutableDruidServer in project druid by druid-io.
the class TestServerInventoryView method getDruidServers.
@Nullable
@Override
public List<ImmutableDruidServer> getDruidServers() {
// do not return broker on purpose to mimic behavior of BrokerServerView
final ImmutableDruidDataSource dataSource = new ImmutableDruidDataSource("DUMMY", Collections.emptyMap(), segments);
final ImmutableDruidServer server = new ImmutableDruidServer(DUMMY_SERVER, 0L, ImmutableMap.of("src", dataSource), 1);
final ImmutableDruidDataSource dataSource2 = new ImmutableDruidDataSource("DUMMY2", Collections.emptyMap(), realtimeSegments);
final ImmutableDruidServer realtimeServer = new ImmutableDruidServer(DUMMY_SERVER_REALTIME, 0L, ImmutableMap.of("src", dataSource2), 1);
return ImmutableList.of(server, realtimeServer);
}
use of org.apache.druid.client.ImmutableDruidServer in project druid by druid-io.
the class DruidSchemaTest method testAvailableSegmentMetadataIsRealtime.
@Test
public void testAvailableSegmentMetadataIsRealtime() {
Map<SegmentId, AvailableSegmentMetadata> segmentsMetadata = schema.getSegmentMetadataSnapshot();
final List<DataSegment> segments = segmentsMetadata.values().stream().map(AvailableSegmentMetadata::getSegment).collect(Collectors.toList());
// find the only realtime segment with datasource "foo3"
final DataSegment existingSegment = segments.stream().filter(segment -> segment.getDataSource().equals("foo3")).findFirst().orElse(null);
Assert.assertNotNull(existingSegment);
final AvailableSegmentMetadata metadata = segmentsMetadata.get(existingSegment.getId());
Assert.assertEquals(1L, metadata.isRealtime());
// get the historical server
final ImmutableDruidServer historicalServer = druidServers.stream().filter(s -> s.getType().equals(ServerType.HISTORICAL)).findAny().orElse(null);
Assert.assertNotNull(historicalServer);
final DruidServerMetadata historicalServerMetadata = historicalServer.getMetadata();
// add existingSegment to historical
schema.addSegment(historicalServerMetadata, existingSegment);
segmentsMetadata = schema.getSegmentMetadataSnapshot();
// get the segment with datasource "foo3"
DataSegment currentSegment = segments.stream().filter(segment -> segment.getDataSource().equals("foo3")).findFirst().orElse(null);
Assert.assertNotNull(currentSegment);
AvailableSegmentMetadata currentMetadata = segmentsMetadata.get(currentSegment.getId());
Assert.assertEquals(0L, currentMetadata.isRealtime());
ImmutableDruidServer realtimeServer = druidServers.stream().filter(s -> s.getType().equals(ServerType.REALTIME)).findAny().orElse(null);
Assert.assertNotNull(realtimeServer);
// drop existingSegment from realtime task
schema.removeServerSegment(realtimeServer.getMetadata(), existingSegment);
segmentsMetadata = schema.getSegmentMetadataSnapshot();
currentSegment = segments.stream().filter(segment -> segment.getDataSource().equals("foo3")).findFirst().orElse(null);
Assert.assertNotNull(currentSegment);
currentMetadata = segmentsMetadata.get(currentSegment.getId());
Assert.assertEquals(0L, currentMetadata.isRealtime());
}
Aggregations