Examples with DruidCluster org.apache.druid.server.coordinator.DruidCluster used on opensource projects

Example 16 with DruidCluster

use of org.apache.druid.server.coordinator.DruidCluster in project druid by druid-io.

the class LoadRuleTest method testMaxLoadingQueueSize.

@Test
public void testMaxLoadingQueueSize() {
    EasyMock.expect(mockBalancerStrategy.findNewSegmentHomeReplicator(EasyMock.anyObject(), EasyMock.anyObject())).andDelegateTo(balancerStrategy).times(2);
    EasyMock.replay(throttler, mockBalancerStrategy);
    final LoadQueuePeonTester peon = new LoadQueuePeonTester();
    LoadRule rule = createLoadRule(ImmutableMap.of("hot", 1));
    DruidCluster druidCluster = DruidClusterBuilder.newBuilder().addTier("hot", new ServerHolder(new DruidServer("serverHot", "hostHot", null, 1000, ServerType.HISTORICAL, "hot", 0).toImmutableDruidServer(), peon)).build();
    DataSegment dataSegment1 = createDataSegment("ds1");
    DataSegment dataSegment2 = createDataSegment("ds2");
    DataSegment dataSegment3 = createDataSegment("ds3");
    DruidCoordinatorRuntimeParams params = CoordinatorRuntimeParamsTestHelpers.newBuilder().withDruidCluster(druidCluster).withSegmentReplicantLookup(SegmentReplicantLookup.make(druidCluster, false)).withReplicationManager(throttler).withBalancerStrategy(mockBalancerStrategy).withUsedSegmentsInTest(dataSegment1, dataSegment2, dataSegment3).withDynamicConfigs(CoordinatorDynamicConfig.builder().withMaxSegmentsInNodeLoadingQueue(2).build()).build();
    CoordinatorStats stats1 = rule.run(null, params, dataSegment1);
    CoordinatorStats stats2 = rule.run(null, params, dataSegment2);
    CoordinatorStats stats3 = rule.run(null, params, dataSegment3);
    Assert.assertEquals(1L, stats1.getTieredStat(LoadRule.ASSIGNED_COUNT, "hot"));
    Assert.assertEquals(1L, stats2.getTieredStat(LoadRule.ASSIGNED_COUNT, "hot"));
    Assert.assertFalse(stats3.getTiers(LoadRule.ASSIGNED_COUNT).contains("hot"));
    EasyMock.verify(throttler, mockBalancerStrategy);
}

Example 17 with DruidCluster

use of org.apache.druid.server.coordinator.DruidCluster in project druid by druid-io.

the class LoadRuleTest method testLoadUsedSegmentsForAllSegmentGranularityAndCachingCostBalancerStrategy.

@Test
public void testLoadUsedSegmentsForAllSegmentGranularityAndCachingCostBalancerStrategy() {
    EasyMock.expect(throttler.canCreateReplicant(EasyMock.anyString())).andReturn(false).anyTimes();
    LoadRule rule = createLoadRule(ImmutableMap.of("tier1", 1));
    DataSegment segment0 = createDataSegmentWithIntervalAndPartition(createDataSegment("foo"), JodaUtils.MIN_INSTANT, JodaUtils.MAX_INSTANT, 0);
    DataSegment segment1 = createDataSegmentWithIntervalAndPartition(createDataSegment("foo"), JodaUtils.MIN_INSTANT, JodaUtils.MAX_INSTANT, 1);
    final LoadQueuePeon loadingPeon = createLoadingPeon(ImmutableList.of(segment0), true);
    loadingPeon.loadSegment(EasyMock.anyObject(), EasyMock.isNull());
    EasyMock.expectLastCall().once();
    EasyMock.expect(mockBalancerStrategy.findNewSegmentHomeReplicator(EasyMock.anyObject(), EasyMock.anyObject())).andDelegateTo(cachingCostBalancerStrategy).anyTimes();
    EasyMock.replay(throttler, loadingPeon, mockBalancerStrategy);
    ImmutableDruidServer server = new DruidServer("serverHot", "hostHot", null, 1000, ServerType.HISTORICAL, "tier1", 1).toImmutableDruidServer();
    DruidCluster druidCluster = DruidClusterBuilder.newBuilder().addTier("tier1", new ServerHolder(server, loadingPeon)).build();
    final CoordinatorStats stats = rule.run(null, makeCoordinatorRuntimeParamsWithLoadReplicationOnTimeout(druidCluster, segment0, segment1), segment1);
    Assert.assertEquals(1L, stats.getTieredStat(LoadRule.ASSIGNED_COUNT, "tier1"));
    EasyMock.verify(throttler, loadingPeon, mockBalancerStrategy);
}

Example 18 with DruidCluster

use of org.apache.druid.server.coordinator.DruidCluster in project druid by druid-io.

the class LoadRuleTest method testSkipReplicationForTimedOutSegments.

@Test
public void testSkipReplicationForTimedOutSegments() {
    EasyMock.expect(throttler.canCreateReplicant(EasyMock.anyString())).andReturn(true).anyTimes();
    final LoadQueuePeon emptyPeon = createEmptyPeon();
    emptyPeon.loadSegment(EasyMock.anyObject(), EasyMock.anyObject());
    EasyMock.expectLastCall().atLeastOnce();
    LoadRule rule = createLoadRule(ImmutableMap.of("hot", 1));
    final DataSegment segment = createDataSegment("foo");
    EasyMock.expect(mockBalancerStrategy.findNewSegmentHomeReplicator(EasyMock.anyObject(), EasyMock.anyObject())).andDelegateTo(balancerStrategy).anyTimes();
    EasyMock.replay(throttler, emptyPeon, mockBalancerStrategy);
    ImmutableDruidServer server1 = new DruidServer("serverHot", "hostHot", null, 1000, ServerType.HISTORICAL, "hot", 1).toImmutableDruidServer();
    ImmutableDruidServer server2 = new DruidServer("serverHot2", "hostHot2", null, 1000, ServerType.HISTORICAL, "hot", 1).toImmutableDruidServer();
    DruidCluster druidCluster = DruidClusterBuilder.newBuilder().addTier("hot", new ServerHolder(server1, emptyPeon), new ServerHolder(server2, emptyPeon)).build();
    CoordinatorStats stats = rule.run(null, makeCoordinatorRuntimeParams(druidCluster, segment), segment);
    // Ensure that the segment is assigned to one of the historicals
    Assert.assertEquals(1L, stats.getTieredStat(LoadRule.ASSIGNED_COUNT, "hot"));
    // Add the segment to the timed out list to simulate peon timeout on loading the segment
    final LoadQueuePeon slowLoadingPeon = createLoadingPeon(ImmutableList.of(segment), true);
    EasyMock.replay(slowLoadingPeon);
    DruidCluster withLoadTimeout = DruidClusterBuilder.newBuilder().addTier("hot", new ServerHolder(server1, slowLoadingPeon), new ServerHolder(server2, emptyPeon)).build();
    // Default behavior is to not replicate the timed out segments on other servers
    CoordinatorStats statsAfterLoadPrimary = rule.run(null, makeCoordinatorRuntimeParams(withLoadTimeout, segment), segment);
    Assert.assertEquals(0L, statsAfterLoadPrimary.getTieredStat(LoadRule.ASSIGNED_COUNT, "hot"));
    EasyMock.verify(throttler, emptyPeon, mockBalancerStrategy);
}

Example 19 with DruidCluster

use of org.apache.druid.server.coordinator.DruidCluster in project druid by druid-io.

the class LoadRuleTest method testDropWithNonExistentTier.

@Test
public void testDropWithNonExistentTier() {
    final LoadQueuePeon mockPeon = createEmptyPeon();
    mockPeon.dropSegment(EasyMock.anyObject(), EasyMock.anyObject());
    EasyMock.expectLastCall().atLeastOnce();
    EasyMock.expect(mockBalancerStrategy.pickServersToDrop(EasyMock.anyObject(), EasyMock.anyObject())).andDelegateTo(balancerStrategy).times(2);
    EasyMock.replay(throttler, mockPeon, mockBalancerStrategy);
    LoadRule rule = createLoadRule(ImmutableMap.of("nonExistentTier", 1, "hot", 1));
    final DataSegment segment = createDataSegment("foo");
    DruidServer server1 = new DruidServer("serverHot", "hostHot", null, 1000, ServerType.HISTORICAL, "hot", 0);
    DruidServer server2 = new DruidServer("serverHot2", "hostHot2", null, 1000, ServerType.HISTORICAL, "hot", 0);
    server1.addDataSegment(segment);
    server2.addDataSegment(segment);
    DruidCluster druidCluster = DruidClusterBuilder.newBuilder().addTier("hot", new ServerHolder(server1.toImmutableDruidServer(), mockPeon), new ServerHolder(server2.toImmutableDruidServer(), mockPeon)).build();
    CoordinatorStats stats = rule.run(null, makeCoordinatorRuntimeParams(druidCluster, segment), segment);
    Assert.assertEquals(1L, stats.getTieredStat("droppedCount", "hot"));
    EasyMock.verify(throttler, mockPeon, mockBalancerStrategy);
}

Example 20 with DruidCluster

use of org.apache.druid.server.coordinator.DruidCluster in project druid by druid-io.

the class LoadRuleTest method testLoadPriority.

@Test
public void testLoadPriority() {
    EasyMock.expect(throttler.canCreateReplicant(EasyMock.anyString())).andReturn(false).anyTimes();
    final LoadQueuePeon mockPeon1 = createEmptyPeon();
    final LoadQueuePeon mockPeon2 = createEmptyPeon();
    mockPeon2.loadSegment(EasyMock.anyObject(), EasyMock.isNull());
    EasyMock.expectLastCall().once();
    EasyMock.expect(mockBalancerStrategy.findNewSegmentHomeReplicator(EasyMock.anyObject(), EasyMock.anyObject())).andDelegateTo(balancerStrategy).times(2);
    EasyMock.replay(throttler, mockPeon1, mockPeon2, mockBalancerStrategy);
    final LoadRule rule = createLoadRule(ImmutableMap.of("tier1", 10, "tier2", 10));
    final DruidCluster druidCluster = DruidClusterBuilder.newBuilder().addTier("tier1", new ServerHolder(new DruidServer("server1", "host1", null, 1000, ServerType.HISTORICAL, "tier1", 0).toImmutableDruidServer(), mockPeon1)).addTier("tier2", new ServerHolder(new DruidServer("server2", "host2", null, 1000, ServerType.HISTORICAL, "tier2", 1).toImmutableDruidServer(), mockPeon2), new ServerHolder(new DruidServer("server3", "host3", null, 1000, ServerType.HISTORICAL, "tier2", 1).toImmutableDruidServer(), mockPeon2)).build();
    final DataSegment segment = createDataSegment("foo");
    final CoordinatorStats stats = rule.run(null, makeCoordinatorRuntimeParams(druidCluster, segment), segment);
    Assert.assertEquals(0L, stats.getTieredStat(LoadRule.ASSIGNED_COUNT, "tier1"));
    Assert.assertEquals(1L, stats.getTieredStat(LoadRule.ASSIGNED_COUNT, "tier2"));
    EasyMock.verify(throttler, mockPeon1, mockPeon2, mockBalancerStrategy);
}