Examples with RunRules org.apache.druid.server.coordinator.duty.RunRules used on opensource projects

Example 1 with RunRules

use of org.apache.druid.server.coordinator.duty.RunRules in project druid by druid-io.

the class BalanceSegmentsProfiler method bigProfiler.

public void bigProfiler() {
    Stopwatch watch = Stopwatch.createUnstarted();
    int numSegments = 55000;
    int numServers = 50;
    EasyMock.expect(manager.getAllRules()).andReturn(ImmutableMap.of("test", rules)).anyTimes();
    EasyMock.expect(manager.getRules(EasyMock.anyObject())).andReturn(rules).anyTimes();
    EasyMock.expect(manager.getRulesWithDefault(EasyMock.anyObject())).andReturn(rules).anyTimes();
    EasyMock.replay(manager);
    coordinator.moveSegment(EasyMock.anyObject(), EasyMock.anyObject(), EasyMock.anyObject(), EasyMock.anyObject(), EasyMock.anyObject());
    EasyMock.expectLastCall().anyTimes();
    EasyMock.replay(coordinator);
    Map<String, LoadQueuePeon> peonMap = new HashMap<>();
    List<ServerHolder> serverHolderList = new ArrayList<>();
    List<DataSegment> segments = new ArrayList<>();
    for (int i = 0; i < numSegments; i++) {
        segments.add(new DataSegment("datasource" + i, new Interval(DateTimes.of("2012-01-01"), (DateTimes.of("2012-01-01")).plusHours(1)), (DateTimes.of("2012-03-01")).toString(), new HashMap<>(), new ArrayList<>(), new ArrayList<>(), NoneShardSpec.instance(), 0, 4L));
    }
    for (int i = 0; i < numServers; i++) {
        ImmutableDruidServer server = EasyMock.createMock(ImmutableDruidServer.class);
        EasyMock.expect(server.getMetadata()).andReturn(null).anyTimes();
        EasyMock.expect(server.getCurrSize()).andReturn(30L).atLeastOnce();
        EasyMock.expect(server.getMaxSize()).andReturn(100L).atLeastOnce();
        EasyMock.expect(server.getTier()).andReturn("normal").anyTimes();
        EasyMock.expect(server.getName()).andReturn(Integer.toString(i)).atLeastOnce();
        EasyMock.expect(server.getHost()).andReturn(Integer.toString(i)).anyTimes();
        if (i == 0) {
            ImmutableDruidServerTests.expectSegments(server, segments);
        } else {
            ImmutableDruidServerTests.expectSegments(server, Collections.emptyList());
        }
        EasyMock.expect(server.getSegment(EasyMock.anyObject())).andReturn(null).anyTimes();
        EasyMock.replay(server);
        LoadQueuePeon peon = new LoadQueuePeonTester();
        peonMap.put(Integer.toString(i), peon);
        serverHolderList.add(new ServerHolder(server, peon));
    }
    DruidCluster druidCluster = DruidClusterBuilder.newBuilder().addTier("normal", serverHolderList.toArray(new ServerHolder[0])).build();
    DruidCoordinatorRuntimeParams params = CoordinatorRuntimeParamsTestHelpers.newBuilder(druidCluster).withLoadManagementPeons(peonMap).withUsedSegmentsInTest(segments).withDynamicConfigs(CoordinatorDynamicConfig.builder().withMaxSegmentsToMove(MAX_SEGMENTS_TO_MOVE).withReplicantLifetime(500).withReplicationThrottleLimit(5).build()).withEmitter(emitter).withDatabaseRuleManager(manager).withReplicationManager(new ReplicationThrottler(2, 500, false)).build();
    BalanceSegmentsTester tester = new BalanceSegmentsTester(coordinator);
    RunRules runner = new RunRules(coordinator);
    watch.start();
    DruidCoordinatorRuntimeParams balanceParams = tester.run(params);
    DruidCoordinatorRuntimeParams assignParams = runner.run(params);
    System.out.println(watch.stop());
}

Example 2 with RunRules

use of org.apache.druid.server.coordinator.duty.RunRules in project druid by druid-io.

the class RunRulesTest method testReplicantThrottleAcrossTiers.

/**
 * Nodes:
 * hot - nothing loaded
 * _default_tier - 1 segment loaded
 */
@Test
public void testReplicantThrottleAcrossTiers() {
    EasyMock.expect(coordinator.getDynamicConfigs()).andReturn(CoordinatorDynamicConfig.builder().withReplicationThrottleLimit(7).withReplicantLifetime(1).withMaxSegmentsInNodeLoadingQueue(1000).build()).atLeastOnce();
    coordinator.markSegmentAsUnused(EasyMock.anyObject());
    EasyMock.expectLastCall().anyTimes();
    EasyMock.replay(coordinator);
    mockPeon.loadSegment(EasyMock.anyObject(), EasyMock.anyObject());
    EasyMock.expectLastCall().atLeastOnce();
    mockEmptyPeon();
    EasyMock.expect(databaseRuleManager.getRulesWithDefault(EasyMock.anyObject())).andReturn(Collections.singletonList(new IntervalLoadRule(Intervals.of("2012-01-01T00:00:00.000Z/2013-01-01T00:00:00.000Z"), ImmutableMap.of("hot", 1, DruidServer.DEFAULT_TIER, 1)))).atLeastOnce();
    EasyMock.replay(databaseRuleManager);
    DruidCluster druidCluster = DruidClusterBuilder.newBuilder().addTier("hot", new ServerHolder(new DruidServer("serverHot", "hostHot", null, 1000, ServerType.HISTORICAL, "hot", 1).toImmutableDruidServer(), mockPeon)).addTier(DruidServer.DEFAULT_TIER, new ServerHolder(new DruidServer("serverNorm", "hostNorm", null, 1000, ServerType.HISTORICAL, DruidServer.DEFAULT_TIER, 0).toImmutableDruidServer(), mockPeon)).build();
    ListeningExecutorService exec = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(1));
    BalancerStrategy balancerStrategy = new CostBalancerStrategyFactory().createBalancerStrategy(exec);
    DruidCoordinatorRuntimeParams params = makeCoordinatorRuntimeParams(druidCluster, balancerStrategy).build();
    RunRules runner = new RunRules(new ReplicationThrottler(7, 1, false), coordinator);
    DruidCoordinatorRuntimeParams afterParams = runner.run(params);
    CoordinatorStats stats = afterParams.getCoordinatorStats();
    Assert.assertEquals(24L, stats.getTieredStat("assignedCount", "hot"));
    Assert.assertEquals(7L, stats.getTieredStat("assignedCount", DruidServer.DEFAULT_TIER));
    Assert.assertTrue(stats.getTiers("unassignedCount").isEmpty());
    Assert.assertTrue(stats.getTiers("unassignedSize").isEmpty());
    EasyMock.verify(mockPeon);
    exec.shutdown();
}

Example 3 with RunRules

use of org.apache.druid.server.coordinator.duty.RunRules in project druid by druid-io.

the class RunRulesTest method setUp.

@Before
public void setUp() {
    coordinator = EasyMock.createMock(DruidCoordinator.class);
    mockPeon = EasyMock.createMock(LoadQueuePeon.class);
    emitter = EasyMock.createMock(ServiceEmitter.class);
    EmittingLogger.registerEmitter(emitter);
    databaseRuleManager = EasyMock.createMock(MetadataRuleManager.class);
    segmentsMetadataManager = EasyMock.createNiceMock(SegmentsMetadataManager.class);
    DateTime start = DateTimes.of("2012-01-01");
    usedSegments = new ArrayList<>();
    for (int i = 0; i < 24; i++) {
        usedSegments.add(new DataSegment("test", new Interval(start, start.plusHours(1)), DateTimes.nowUtc().toString(), new HashMap<>(), new ArrayList<>(), new ArrayList<>(), NoneShardSpec.instance(), IndexIO.CURRENT_VERSION_ID, 1));
        start = start.plusHours(1);
    }
    ruleRunner = new RunRules(new ReplicationThrottler(24, 1, false), coordinator);
}