Examples with DruidServer org.apache.druid.client.DruidServer used on opensource projects

Example 66 with DruidServer

use of org.apache.druid.client.DruidServer in project druid by druid-io.

the class RunRulesTest method testOneNodesOneTierOneReplicantCostBalancerStrategyNotEnoughSpace.

/**
 * Tier - __default_tier
 * Nodes - 1
 * Replicants - 1
 * Cost balancer strategy should not assign anything as there is not enough space
 */
@Test
public void testOneNodesOneTierOneReplicantCostBalancerStrategyNotEnoughSpace() {
    mockCoordinator();
    mockEmptyPeon();
    int numReplicants = 1;
    EasyMock.expect(databaseRuleManager.getRulesWithDefault(EasyMock.anyObject())).andReturn(Collections.singletonList(new ForeverLoadRule(ImmutableMap.of(DruidServer.DEFAULT_TIER, numReplicants)))).atLeastOnce();
    EasyMock.replay(databaseRuleManager);
    DataSegment dataSegment = new DataSegment("test", Intervals.utc(0, 1), DateTimes.nowUtc().toString(), new HashMap<>(), new ArrayList<>(), new ArrayList<>(), NoneShardSpec.instance(), IndexIO.CURRENT_VERSION_ID, 11);
    DruidCluster druidCluster = DruidClusterBuilder.newBuilder().addTier(DruidServer.DEFAULT_TIER, new ServerHolder(new DruidServer("server1", "host1", null, 10, ServerType.HISTORICAL, DruidServer.DEFAULT_TIER, 0).toImmutableDruidServer(), mockPeon)).build();
    ListeningExecutorService exec = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(1));
    CostBalancerStrategy balancerStrategy = new CostBalancerStrategy(exec);
    DruidCoordinatorRuntimeParams params = makeCoordinatorRuntimeParams(druidCluster, balancerStrategy, Collections.singletonList(dataSegment)).withDynamicConfigs(CoordinatorDynamicConfig.builder().withMaxSegmentsToMove(5).build()).build();
    DruidCoordinatorRuntimeParams afterParams = ruleRunner.run(params);
    CoordinatorStats stats = afterParams.getCoordinatorStats();
    Assert.assertEquals(dataSegment.getSize() * numReplicants, stats.getTieredStat(LoadRule.REQUIRED_CAPACITY, DruidServer.DEFAULT_TIER));
    // since primary assignment should fail
    Assert.assertTrue(stats.getTiers("assignedCount").isEmpty());
    Assert.assertTrue(stats.getTiers("unassignedCount").isEmpty());
    Assert.assertTrue(stats.getTiers("unassignedSize").isEmpty());
    exec.shutdown();
    EasyMock.verify(mockPeon);
}

Example 67 with DruidServer

use of org.apache.druid.client.DruidServer in project druid by druid-io.

the class RunRulesTest method testDropTooManyInDifferentTiers.

@Test
public void testDropTooManyInDifferentTiers() {
    mockCoordinator();
    mockPeon.loadSegment(EasyMock.anyObject(), EasyMock.anyObject());
    EasyMock.expectLastCall().atLeastOnce();
    mockPeon.dropSegment(EasyMock.anyObject(), EasyMock.anyObject());
    EasyMock.expectLastCall().atLeastOnce();
    mockEmptyPeon();
    EasyMock.expect(databaseRuleManager.getRulesWithDefault(EasyMock.anyObject())).andReturn(Lists.newArrayList(new IntervalLoadRule(Intervals.of("2012-01-01T00:00:00.000Z/2012-01-01T12:00:00.000Z"), ImmutableMap.of("hot", 1)), new IntervalDropRule(Intervals.of("2012-01-01T00:00:00.000Z/2012-01-02T00:00:00.000Z")))).atLeastOnce();
    EasyMock.replay(databaseRuleManager);
    DruidServer server1 = new DruidServer("server1", "host1", null, 1000, ServerType.HISTORICAL, "hot", 0);
    server1.addDataSegment(usedSegments.get(0));
    DruidServer server2 = new DruidServer("serverNorm2", "hostNorm2", null, 1000, ServerType.HISTORICAL, "normal", 0);
    for (DataSegment segment : usedSegments) {
        server2.addDataSegment(segment);
    }
    DruidCluster druidCluster = DruidClusterBuilder.newBuilder().addTier("hot", new ServerHolder(server1.toImmutableDruidServer(), mockPeon)).addTier("normal", new ServerHolder(server2.toImmutableDruidServer(), mockPeon)).build();
    SegmentReplicantLookup segmentReplicantLookup = SegmentReplicantLookup.make(druidCluster, false);
    ListeningExecutorService exec = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(1));
    BalancerStrategy balancerStrategy = new CostBalancerStrategyFactory().createBalancerStrategy(exec);
    DruidCoordinatorRuntimeParams params = createCoordinatorRuntimeParams(druidCluster).withDynamicConfigs(COORDINATOR_CONFIG_WITH_ZERO_LEADING_TIME_BEFORE_CAN_MARK_AS_UNUSED_OVERSHADOWED_SEGMENTS).withSegmentReplicantLookup(segmentReplicantLookup).withBalancerStrategy(balancerStrategy).build();
    DruidCoordinatorRuntimeParams afterParams = ruleRunner.run(params);
    CoordinatorStats stats = afterParams.getCoordinatorStats();
    Assert.assertEquals(1L, stats.getTieredStat("droppedCount", "normal"));
    Assert.assertEquals(12L, stats.getGlobalStat("deletedCount"));
    exec.shutdown();
    EasyMock.verify(mockPeon);
}

Example 68 with DruidServer

use of org.apache.druid.client.DruidServer in project druid by druid-io.

the class RunRulesTest method testReplicantThrottleAcrossTiers.

/**
 * Nodes:
 * hot - nothing loaded
 * _default_tier - 1 segment loaded
 */
@Test
public void testReplicantThrottleAcrossTiers() {
    EasyMock.expect(coordinator.getDynamicConfigs()).andReturn(CoordinatorDynamicConfig.builder().withReplicationThrottleLimit(7).withReplicantLifetime(1).withMaxSegmentsInNodeLoadingQueue(1000).build()).atLeastOnce();
    coordinator.markSegmentAsUnused(EasyMock.anyObject());
    EasyMock.expectLastCall().anyTimes();
    EasyMock.replay(coordinator);
    mockPeon.loadSegment(EasyMock.anyObject(), EasyMock.anyObject());
    EasyMock.expectLastCall().atLeastOnce();
    mockEmptyPeon();
    EasyMock.expect(databaseRuleManager.getRulesWithDefault(EasyMock.anyObject())).andReturn(Collections.singletonList(new IntervalLoadRule(Intervals.of("2012-01-01T00:00:00.000Z/2013-01-01T00:00:00.000Z"), ImmutableMap.of("hot", 1, DruidServer.DEFAULT_TIER, 1)))).atLeastOnce();
    EasyMock.replay(databaseRuleManager);
    DruidCluster druidCluster = DruidClusterBuilder.newBuilder().addTier("hot", new ServerHolder(new DruidServer("serverHot", "hostHot", null, 1000, ServerType.HISTORICAL, "hot", 1).toImmutableDruidServer(), mockPeon)).addTier(DruidServer.DEFAULT_TIER, new ServerHolder(new DruidServer("serverNorm", "hostNorm", null, 1000, ServerType.HISTORICAL, DruidServer.DEFAULT_TIER, 0).toImmutableDruidServer(), mockPeon)).build();
    ListeningExecutorService exec = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(1));
    BalancerStrategy balancerStrategy = new CostBalancerStrategyFactory().createBalancerStrategy(exec);
    DruidCoordinatorRuntimeParams params = makeCoordinatorRuntimeParams(druidCluster, balancerStrategy).build();
    RunRules runner = new RunRules(new ReplicationThrottler(7, 1, false), coordinator);
    DruidCoordinatorRuntimeParams afterParams = runner.run(params);
    CoordinatorStats stats = afterParams.getCoordinatorStats();
    Assert.assertEquals(24L, stats.getTieredStat("assignedCount", "hot"));
    Assert.assertEquals(7L, stats.getTieredStat("assignedCount", DruidServer.DEFAULT_TIER));
    Assert.assertTrue(stats.getTiers("unassignedCount").isEmpty());
    Assert.assertTrue(stats.getTiers("unassignedSize").isEmpty());
    EasyMock.verify(mockPeon);
    exec.shutdown();
}

Example 69 with DruidServer

use of org.apache.druid.client.DruidServer in project druid by druid-io.

the class RunRulesTest method testDropReplicantThrottle.

@Test
public void testDropReplicantThrottle() {
    mockCoordinator();
    mockPeon.dropSegment(EasyMock.anyObject(), EasyMock.anyObject());
    EasyMock.expectLastCall().atLeastOnce();
    mockEmptyPeon();
    EasyMock.expect(databaseRuleManager.getRulesWithDefault(EasyMock.anyObject())).andReturn(Collections.singletonList(new IntervalLoadRule(Intervals.of("2012-01-01T00:00:00.000Z/2013-01-02T00:00:00.000Z"), ImmutableMap.of("normal", 1)))).atLeastOnce();
    EasyMock.replay(databaseRuleManager);
    DataSegment overFlowSegment = new DataSegment("test", Intervals.of("2012-02-01/2012-02-02"), DateTimes.nowUtc().toString(), new HashMap<>(), new ArrayList<>(), new ArrayList<>(), NoneShardSpec.instance(), 1, 0);
    List<DataSegment> longerUsedSegments = Lists.newArrayList(usedSegments);
    longerUsedSegments.add(overFlowSegment);
    DruidServer server1 = new DruidServer("serverNorm1", "hostNorm1", null, 1000, ServerType.HISTORICAL, "normal", 0);
    for (DataSegment segment : longerUsedSegments) {
        server1.addDataSegment(segment);
    }
    DruidServer server2 = new DruidServer("serverNorm2", "hostNorm2", null, 1000, ServerType.HISTORICAL, "normal", 0);
    for (DataSegment segment : longerUsedSegments) {
        server2.addDataSegment(segment);
    }
    DruidCluster druidCluster = DruidClusterBuilder.newBuilder().addTier("normal", new ServerHolder(server1.toImmutableDruidServer(), mockPeon), new ServerHolder(server2.toImmutableDruidServer(), mockPeon)).build();
    SegmentReplicantLookup segmentReplicantLookup = SegmentReplicantLookup.make(druidCluster, false);
    ListeningExecutorService exec = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(1));
    BalancerStrategy balancerStrategy = new CostBalancerStrategyFactory().createBalancerStrategy(exec);
    DruidCoordinatorRuntimeParams params = CoordinatorRuntimeParamsTestHelpers.newBuilder().withDruidCluster(druidCluster).withDynamicConfigs(COORDINATOR_CONFIG_WITH_ZERO_LEADING_TIME_BEFORE_CAN_MARK_AS_UNUSED_OVERSHADOWED_SEGMENTS).withUsedSegmentsInTest(longerUsedSegments).withDatabaseRuleManager(databaseRuleManager).withSegmentReplicantLookup(segmentReplicantLookup).withBalancerStrategy(balancerStrategy).build();
    DruidCoordinatorRuntimeParams afterParams = ruleRunner.run(params);
    CoordinatorStats stats = afterParams.getCoordinatorStats();
    // There is no throttling on drop
    Assert.assertEquals(25L, stats.getTieredStat("droppedCount", "normal"));
    EasyMock.verify(mockPeon);
    exec.shutdown();
}

Example 70 with DruidServer

use of org.apache.druid.client.DruidServer in project druid by druid-io.

the class RunRulesTest method testRunTwoTiersWithExistingSegments.

/**
 * Nodes:
 * hot - 1 replicant
 * normal - 1 replicant
 */
@Test
public void testRunTwoTiersWithExistingSegments() {
    mockCoordinator();
    mockPeon.loadSegment(EasyMock.anyObject(), EasyMock.anyObject());
    EasyMock.expectLastCall().atLeastOnce();
    mockEmptyPeon();
    EasyMock.expect(databaseRuleManager.getRulesWithDefault(EasyMock.anyObject())).andReturn(Lists.newArrayList(new IntervalLoadRule(Intervals.of("2012-01-01T00:00:00.000Z/2012-01-01T12:00:00.000Z"), ImmutableMap.of("hot", 1)), new IntervalLoadRule(Intervals.of("2012-01-01T00:00:00.000Z/2012-01-02T00:00:00.000Z"), ImmutableMap.of("normal", 1)))).atLeastOnce();
    EasyMock.replay(databaseRuleManager);
    DruidServer normServer = new DruidServer("serverNorm", "hostNorm", null, 1000, ServerType.HISTORICAL, "normal", 0);
    for (DataSegment segment : usedSegments) {
        normServer.addDataSegment(segment);
    }
    DruidCluster druidCluster = DruidClusterBuilder.newBuilder().addTier("hot", new ServerHolder(new DruidServer("serverHot", "hostHot", null, 1000, ServerType.HISTORICAL, "hot", 0).toImmutableDruidServer(), mockPeon)).addTier("normal", new ServerHolder(normServer.toImmutableDruidServer(), mockPeon)).build();
    SegmentReplicantLookup segmentReplicantLookup = SegmentReplicantLookup.make(druidCluster, false);
    ListeningExecutorService exec = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(1));
    BalancerStrategy balancerStrategy = new CostBalancerStrategyFactory().createBalancerStrategy(exec);
    DruidCoordinatorRuntimeParams params = createCoordinatorRuntimeParams(druidCluster).withSegmentReplicantLookup(segmentReplicantLookup).withBalancerStrategy(balancerStrategy).build();
    DruidCoordinatorRuntimeParams afterParams = ruleRunner.run(params);
    CoordinatorStats stats = afterParams.getCoordinatorStats();
    Assert.assertEquals(12L, stats.getTieredStat("assignedCount", "hot"));
    Assert.assertEquals(0L, stats.getTieredStat("assignedCount", "normal"));
    Assert.assertTrue(stats.getTiers("unassignedCount").isEmpty());
    Assert.assertTrue(stats.getTiers("unassignedSize").isEmpty());
    exec.shutdown();
    EasyMock.verify(mockPeon);
}