Examples with DruidServer org.apache.druid.client.DruidServer used on opensource projects

Example 11 with DruidServer

use of org.apache.druid.client.DruidServer in project druid by druid-io.

the class CachingCostBalancerStrategyTest method createServerHolder.

private ServerHolder createServerHolder(String name, String host, int maxSize, int numberOfSegments, Random random, DateTime referenceTime) {
    DruidServer druidServer = new DruidServer(name, host, null, maxSize, ServerType.HISTORICAL, "normal", 0);
    createDataSegments(numberOfSegments, random, referenceTime).forEach(druidServer::addDataSegment);
    return new ServerHolder(druidServer.toImmutableDruidServer(), new LoadQueuePeonTester());
}

Example 12 with DruidServer

use of org.apache.druid.client.DruidServer in project druid by druid-io.

the class RunRulesTest method testRulesRunOnNonOvershadowedSegmentsOnly.

@Test
public void testRulesRunOnNonOvershadowedSegmentsOnly() {
    Set<DataSegment> usedSegments = new HashSet<>();
    DataSegment v1 = new DataSegment("test", Intervals.of("2012-01-01/2012-01-02"), "1", new HashMap<>(), new ArrayList<>(), new ArrayList<>(), NoneShardSpec.instance(), IndexIO.CURRENT_VERSION_ID, 1);
    DataSegment v2 = new DataSegment("test", Intervals.of("2012-01-01/2012-01-02"), "2", new HashMap<>(), new ArrayList<>(), new ArrayList<>(), NoneShardSpec.instance(), IndexIO.CURRENT_VERSION_ID, 1);
    usedSegments.add(v1);
    usedSegments.add(v2);
    mockCoordinator();
    mockPeon.loadSegment(EasyMock.eq(v2), EasyMock.anyObject());
    EasyMock.expectLastCall().once();
    mockEmptyPeon();
    EasyMock.expect(databaseRuleManager.getRulesWithDefault(EasyMock.anyObject())).andReturn(Collections.singletonList(new ForeverLoadRule(ImmutableMap.of(DruidServer.DEFAULT_TIER, 1)))).atLeastOnce();
    EasyMock.replay(databaseRuleManager);
    DruidCluster druidCluster = DruidClusterBuilder.newBuilder().addTier(DruidServer.DEFAULT_TIER, new ServerHolder(new DruidServer("serverHot", "hostHot", null, 1000, ServerType.HISTORICAL, DruidServer.DEFAULT_TIER, 0).toImmutableDruidServer(), mockPeon)).build();
    ListeningExecutorService exec = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(1));
    BalancerStrategy balancerStrategy = new CostBalancerStrategyFactory().createBalancerStrategy(exec);
    DruidCoordinatorRuntimeParams params = CoordinatorRuntimeParamsTestHelpers.newBuilder().withDruidCluster(druidCluster).withUsedSegmentsInTest(usedSegments).withDatabaseRuleManager(databaseRuleManager).withSegmentReplicantLookup(SegmentReplicantLookup.make(new DruidCluster(), false)).withBalancerStrategy(balancerStrategy).withDynamicConfigs(CoordinatorDynamicConfig.builder().withMaxSegmentsToMove(5).build()).build();
    DruidCoordinatorRuntimeParams afterParams = ruleRunner.run(params);
    CoordinatorStats stats = afterParams.getCoordinatorStats();
    Assert.assertEquals(1, stats.getTiers("assignedCount").size());
    Assert.assertEquals(1, stats.getTieredStat("assignedCount", "_default_tier"));
    Assert.assertTrue(stats.getTiers("unassignedCount").isEmpty());
    Assert.assertTrue(stats.getTiers("unassignedSize").isEmpty());
    Assert.assertEquals(2, usedSegments.size());
    Assert.assertEquals(usedSegments, params.getUsedSegments());
    Assert.assertEquals(usedSegments, afterParams.getUsedSegments());
    EasyMock.verify(mockPeon);
    exec.shutdown();
}

Example 13 with DruidServer

use of org.apache.druid.client.DruidServer in project druid by druid-io.

the class RunRulesTest method testTwoTiersTwoReplicantsWithStrictReplicantLimit.

/**
 * Nodes:
 * normal - 2 replicants
 * hot - 2 replicants
 * maxNonPrimaryReplicantsToLoad - 48
 * Expect only 72 segments to be loaded despite there being 96 primary + non-primary replicants to load!
 */
@Test
public void testTwoTiersTwoReplicantsWithStrictReplicantLimit() {
    mockCoordinator();
    mockPeon.loadSegment(EasyMock.anyObject(), EasyMock.anyObject());
    EasyMock.expectLastCall().atLeastOnce();
    mockEmptyPeon();
    EasyMock.expect(databaseRuleManager.getRulesWithDefault(EasyMock.anyObject())).andReturn(Collections.singletonList(new IntervalLoadRule(Intervals.of("2012-01-01T00:00:00.000Z/2012-01-02T00:00:00.000Z"), ImmutableMap.of("hot", 2, "normal", 2)))).atLeastOnce();
    EasyMock.replay(databaseRuleManager);
    DruidCluster druidCluster = DruidClusterBuilder.newBuilder().addTier("hot", new ServerHolder(new DruidServer("serverHot", "hostHot", null, 1000, ServerType.HISTORICAL, "hot", 0).toImmutableDruidServer(), mockPeon), new ServerHolder(new DruidServer("serverHot2", "hostHot2", null, 1000, ServerType.HISTORICAL, "hot", 0).toImmutableDruidServer(), mockPeon)).addTier("normal", new ServerHolder(new DruidServer("serverNorm", "hostNorm", null, 1000, ServerType.HISTORICAL, "normal", 0).toImmutableDruidServer(), mockPeon), new ServerHolder(new DruidServer("serverNorm2", "hostNorm2", null, 1000, ServerType.HISTORICAL, "normal", 0).toImmutableDruidServer(), mockPeon)).build();
    ListeningExecutorService exec = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(1));
    BalancerStrategy balancerStrategy = new CostBalancerStrategyFactory().createBalancerStrategy(exec);
    DruidCoordinatorRuntimeParams params = makeCoordinatorRuntimeParams(druidCluster, balancerStrategy).withDynamicConfigs(CoordinatorDynamicConfig.builder().withMaxSegmentsToMove(5).withMaxNonPrimaryReplicantsToLoad(48).build()).build();
    DruidCoordinatorRuntimeParams afterParams = ruleRunner.run(params);
    CoordinatorStats stats = afterParams.getCoordinatorStats();
    Assert.assertEquals(72L, stats.getTieredStat("assignedCount", "hot") + stats.getTieredStat("assignedCount", "normal"));
    Assert.assertEquals(48L, stats.getGlobalStat("totalNonPrimaryReplicantsLoaded"));
    exec.shutdown();
    EasyMock.verify(mockPeon);
}

Example 14 with DruidServer

use of org.apache.druid.client.DruidServer in project druid by druid-io.

the class RunRulesTest method testTwoNodesOneTierThreeReplicantsRandomStrategyNotEnoughNodes.

/**
 * Tier - __default_tier
 * Nodes - 2
 * Replicants - 3
 * Random balancer strategy should not assign anything and not get into loop as there are not enough nodes for replication
 */
@Test(timeout = 5000L)
public void testTwoNodesOneTierThreeReplicantsRandomStrategyNotEnoughNodes() {
    mockCoordinator();
    mockEmptyPeon();
    EasyMock.expect(databaseRuleManager.getRulesWithDefault(EasyMock.anyObject())).andReturn(Collections.singletonList(new ForeverLoadRule(ImmutableMap.of(DruidServer.DEFAULT_TIER, 3)))).atLeastOnce();
    EasyMock.replay(databaseRuleManager);
    DataSegment dataSegment = new DataSegment("test", Intervals.utc(0, 1), DateTimes.nowUtc().toString(), new HashMap<>(), new ArrayList<>(), new ArrayList<>(), NoneShardSpec.instance(), IndexIO.CURRENT_VERSION_ID, 1);
    DruidCluster druidCluster = DruidClusterBuilder.newBuilder().addTier(DruidServer.DEFAULT_TIER, new ServerHolder(new DruidServer("server1", "host1", null, 1000, ServerType.HISTORICAL, DruidServer.DEFAULT_TIER, 0).addDataSegment(dataSegment).toImmutableDruidServer(), mockPeon), new ServerHolder(new DruidServer("server2", "host2", null, 1000, ServerType.HISTORICAL, DruidServer.DEFAULT_TIER, 0).addDataSegment(dataSegment).toImmutableDruidServer(), mockPeon)).build();
    RandomBalancerStrategy balancerStrategy = new RandomBalancerStrategy();
    DruidCoordinatorRuntimeParams params = makeCoordinatorRuntimeParams(druidCluster, balancerStrategy, Collections.singletonList(dataSegment)).withDynamicConfigs(CoordinatorDynamicConfig.builder().withMaxSegmentsToMove(5).build()).build();
    DruidCoordinatorRuntimeParams afterParams = ruleRunner.run(params);
    CoordinatorStats stats = afterParams.getCoordinatorStats();
    Assert.assertEquals(0L, stats.getTieredStat("assignedCount", DruidServer.DEFAULT_TIER));
    Assert.assertTrue(stats.getTiers("unassignedCount").isEmpty());
    Assert.assertTrue(stats.getTiers("unassignedSize").isEmpty());
    EasyMock.verify(mockPeon);
}

Example 15 with DruidServer

use of org.apache.druid.client.DruidServer in project druid by druid-io.

the class RunRulesTest method testDropServerActuallyServesSegment.

@Test
public void testDropServerActuallyServesSegment() {
    mockCoordinator();
    EasyMock.expect(databaseRuleManager.getRulesWithDefault(EasyMock.anyObject())).andReturn(Collections.singletonList(new IntervalLoadRule(Intervals.of("2012-01-01T00:00:00.000Z/2012-01-01T01:00:00.000Z"), ImmutableMap.of("normal", 0)))).atLeastOnce();
    EasyMock.replay(databaseRuleManager);
    DruidServer server1 = new DruidServer("server1", "host1", null, 1000, ServerType.HISTORICAL, "normal", 0);
    server1.addDataSegment(usedSegments.get(0));
    DruidServer server2 = new DruidServer("serverNorm2", "hostNorm2", null, 1000, ServerType.HISTORICAL, "normal", 0);
    server2.addDataSegment(usedSegments.get(1));
    DruidServer server3 = new DruidServer("serverNorm3", "hostNorm3", null, 1000, ServerType.HISTORICAL, "normal", 0);
    server3.addDataSegment(usedSegments.get(1));
    server3.addDataSegment(usedSegments.get(2));
    mockPeon.dropSegment(EasyMock.anyObject(), EasyMock.anyObject());
    EasyMock.expectLastCall().atLeastOnce();
    mockEmptyPeon();
    LoadQueuePeon anotherMockPeon = EasyMock.createMock(LoadQueuePeon.class);
    EasyMock.expect(anotherMockPeon.getLoadQueueSize()).andReturn(10L).atLeastOnce();
    EasyMock.expect(anotherMockPeon.getSegmentsToLoad()).andReturn(new HashSet<>()).anyTimes();
    EasyMock.replay(anotherMockPeon);
    DruidCluster druidCluster = DruidClusterBuilder.newBuilder().addTier("normal", new ServerHolder(server1.toImmutableDruidServer(), mockPeon, false), new ServerHolder(server2.toImmutableDruidServer(), anotherMockPeon, false), new ServerHolder(server3.toImmutableDruidServer(), anotherMockPeon, false)).build();
    SegmentReplicantLookup segmentReplicantLookup = SegmentReplicantLookup.make(druidCluster, false);
    ListeningExecutorService exec = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(1));
    BalancerStrategy balancerStrategy = new CostBalancerStrategyFactory().createBalancerStrategy(exec);
    DruidCoordinatorRuntimeParams params = CoordinatorRuntimeParamsTestHelpers.newBuilder().withDruidCluster(druidCluster).withDynamicConfigs(COORDINATOR_CONFIG_WITH_ZERO_LEADING_TIME_BEFORE_CAN_MARK_AS_UNUSED_OVERSHADOWED_SEGMENTS).withUsedSegmentsInTest(usedSegments).withDatabaseRuleManager(databaseRuleManager).withSegmentReplicantLookup(segmentReplicantLookup).withBalancerStrategy(balancerStrategy).build();
    DruidCoordinatorRuntimeParams afterParams = ruleRunner.run(params);
    CoordinatorStats stats = afterParams.getCoordinatorStats();
    Assert.assertEquals(1L, stats.getTieredStat("droppedCount", "normal"));
    exec.shutdown();
    EasyMock.verify(mockPeon);
    EasyMock.verify(anotherMockPeon);
}