Examples with IntervalLoadRule org.apache.druid.server.coordinator.rules.IntervalLoadRule used on opensource projects

Example 1 with IntervalLoadRule

use of org.apache.druid.server.coordinator.rules.IntervalLoadRule in project druid by druid-io.

the class SQLMetadataRuleManagerTest method testRuleInsert.

@Test
public void testRuleInsert() {
    List<Rule> rules = Collections.singletonList(new IntervalLoadRule(Intervals.of("2015-01-01/2015-02-01"), ImmutableMap.of(DruidServer.DEFAULT_TIER, DruidServer.DEFAULT_NUM_REPLICANTS)));
    AuditInfo auditInfo = new AuditInfo("test_author", "test_comment", "127.0.0.1");
    ruleManager.overrideRule("test_dataSource", rules, auditInfo);
    // New rule should be be reflected in the in memory rules map immediately after being set by user
    Map<String, List<Rule>> allRules = ruleManager.getAllRules();
    Assert.assertEquals(1, allRules.size());
    Assert.assertEquals(1, allRules.get("test_dataSource").size());
    Assert.assertEquals(rules.get(0), allRules.get("test_dataSource").get(0));
}

Example 2 with IntervalLoadRule

use of org.apache.druid.server.coordinator.rules.IntervalLoadRule in project druid by druid-io.

the class SQLMetadataRuleManagerTest method testFetchAuditEntriesForAllDataSources.

@Test
public void testFetchAuditEntriesForAllDataSources() throws Exception {
    List<Rule> rules = Collections.singletonList(new IntervalLoadRule(Intervals.of("2015-01-01/2015-02-01"), ImmutableMap.of(DruidServer.DEFAULT_TIER, DruidServer.DEFAULT_NUM_REPLICANTS)));
    AuditInfo auditInfo = new AuditInfo("test_author", "test_comment", "127.0.0.1");
    ruleManager.overrideRule("test_dataSource", rules, auditInfo);
    ruleManager.overrideRule("test_dataSource2", rules, auditInfo);
    // fetch rules from metadata storage
    ruleManager.poll();
    Assert.assertEquals(rules, ruleManager.getRules("test_dataSource"));
    Assert.assertEquals(rules, ruleManager.getRules("test_dataSource2"));
    // test fetch audit entries
    List<AuditEntry> auditEntries = auditManager.fetchAuditHistory("rules", null);
    Assert.assertEquals(2, auditEntries.size());
    for (AuditEntry entry : auditEntries) {
        Assert.assertEquals(rules, mapper.readValue(entry.getPayload(), new TypeReference<List<Rule>>() {
        }));
        Assert.assertEquals(auditInfo, entry.getAuditInfo());
    }
}

Example 3 with IntervalLoadRule

use of org.apache.druid.server.coordinator.rules.IntervalLoadRule in project druid by druid-io.

the class SQLMetadataRuleManagerTest method testRemoveRulesOlderThanWithNonExistenceDatasourceAndOlderThanTimestampShouldDelete.

@Test
public void testRemoveRulesOlderThanWithNonExistenceDatasourceAndOlderThanTimestampShouldDelete() {
    List<Rule> rules = ImmutableList.of(new IntervalLoadRule(Intervals.of("2015-01-01/2015-02-01"), ImmutableMap.of(DruidServer.DEFAULT_TIER, DruidServer.DEFAULT_NUM_REPLICANTS)));
    AuditInfo auditInfo = new AuditInfo("test_author", "test_comment", "127.0.0.1");
    ruleManager.overrideRule("test_dataSource", rules, auditInfo);
    // Verify that rule was added
    ruleManager.poll();
    Map<String, List<Rule>> allRules = ruleManager.getAllRules();
    Assert.assertEquals(1, allRules.size());
    Assert.assertEquals(1, allRules.get("test_dataSource").size());
    // Now delete rules
    ruleManager.removeRulesForEmptyDatasourcesOlderThan(System.currentTimeMillis());
    // Verify that rule was deleted
    ruleManager.poll();
    allRules = ruleManager.getAllRules();
    Assert.assertEquals(0, allRules.size());
}

Example 4 with IntervalLoadRule

use of org.apache.druid.server.coordinator.rules.IntervalLoadRule in project druid by druid-io.

the class RunRulesTest method testTwoTiersTwoReplicantsWithStrictReplicantLimit.

/**
 * Nodes:
 * normal - 2 replicants
 * hot - 2 replicants
 * maxNonPrimaryReplicantsToLoad - 48
 * Expect only 72 segments to be loaded despite there being 96 primary + non-primary replicants to load!
 */
@Test
public void testTwoTiersTwoReplicantsWithStrictReplicantLimit() {
    mockCoordinator();
    mockPeon.loadSegment(EasyMock.anyObject(), EasyMock.anyObject());
    EasyMock.expectLastCall().atLeastOnce();
    mockEmptyPeon();
    EasyMock.expect(databaseRuleManager.getRulesWithDefault(EasyMock.anyObject())).andReturn(Collections.singletonList(new IntervalLoadRule(Intervals.of("2012-01-01T00:00:00.000Z/2012-01-02T00:00:00.000Z"), ImmutableMap.of("hot", 2, "normal", 2)))).atLeastOnce();
    EasyMock.replay(databaseRuleManager);
    DruidCluster druidCluster = DruidClusterBuilder.newBuilder().addTier("hot", new ServerHolder(new DruidServer("serverHot", "hostHot", null, 1000, ServerType.HISTORICAL, "hot", 0).toImmutableDruidServer(), mockPeon), new ServerHolder(new DruidServer("serverHot2", "hostHot2", null, 1000, ServerType.HISTORICAL, "hot", 0).toImmutableDruidServer(), mockPeon)).addTier("normal", new ServerHolder(new DruidServer("serverNorm", "hostNorm", null, 1000, ServerType.HISTORICAL, "normal", 0).toImmutableDruidServer(), mockPeon), new ServerHolder(new DruidServer("serverNorm2", "hostNorm2", null, 1000, ServerType.HISTORICAL, "normal", 0).toImmutableDruidServer(), mockPeon)).build();
    ListeningExecutorService exec = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(1));
    BalancerStrategy balancerStrategy = new CostBalancerStrategyFactory().createBalancerStrategy(exec);
    DruidCoordinatorRuntimeParams params = makeCoordinatorRuntimeParams(druidCluster, balancerStrategy).withDynamicConfigs(CoordinatorDynamicConfig.builder().withMaxSegmentsToMove(5).withMaxNonPrimaryReplicantsToLoad(48).build()).build();
    DruidCoordinatorRuntimeParams afterParams = ruleRunner.run(params);
    CoordinatorStats stats = afterParams.getCoordinatorStats();
    Assert.assertEquals(72L, stats.getTieredStat("assignedCount", "hot") + stats.getTieredStat("assignedCount", "normal"));
    Assert.assertEquals(48L, stats.getGlobalStat("totalNonPrimaryReplicantsLoaded"));
    exec.shutdown();
    EasyMock.verify(mockPeon);
}

Example 5 with IntervalLoadRule

use of org.apache.druid.server.coordinator.rules.IntervalLoadRule in project druid by druid-io.

the class RunRulesTest method testDropServerActuallyServesSegment.

@Test
public void testDropServerActuallyServesSegment() {
    mockCoordinator();
    EasyMock.expect(databaseRuleManager.getRulesWithDefault(EasyMock.anyObject())).andReturn(Collections.singletonList(new IntervalLoadRule(Intervals.of("2012-01-01T00:00:00.000Z/2012-01-01T01:00:00.000Z"), ImmutableMap.of("normal", 0)))).atLeastOnce();
    EasyMock.replay(databaseRuleManager);
    DruidServer server1 = new DruidServer("server1", "host1", null, 1000, ServerType.HISTORICAL, "normal", 0);
    server1.addDataSegment(usedSegments.get(0));
    DruidServer server2 = new DruidServer("serverNorm2", "hostNorm2", null, 1000, ServerType.HISTORICAL, "normal", 0);
    server2.addDataSegment(usedSegments.get(1));
    DruidServer server3 = new DruidServer("serverNorm3", "hostNorm3", null, 1000, ServerType.HISTORICAL, "normal", 0);
    server3.addDataSegment(usedSegments.get(1));
    server3.addDataSegment(usedSegments.get(2));
    mockPeon.dropSegment(EasyMock.anyObject(), EasyMock.anyObject());
    EasyMock.expectLastCall().atLeastOnce();
    mockEmptyPeon();
    LoadQueuePeon anotherMockPeon = EasyMock.createMock(LoadQueuePeon.class);
    EasyMock.expect(anotherMockPeon.getLoadQueueSize()).andReturn(10L).atLeastOnce();
    EasyMock.expect(anotherMockPeon.getSegmentsToLoad()).andReturn(new HashSet<>()).anyTimes();
    EasyMock.replay(anotherMockPeon);
    DruidCluster druidCluster = DruidClusterBuilder.newBuilder().addTier("normal", new ServerHolder(server1.toImmutableDruidServer(), mockPeon, false), new ServerHolder(server2.toImmutableDruidServer(), anotherMockPeon, false), new ServerHolder(server3.toImmutableDruidServer(), anotherMockPeon, false)).build();
    SegmentReplicantLookup segmentReplicantLookup = SegmentReplicantLookup.make(druidCluster, false);
    ListeningExecutorService exec = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(1));
    BalancerStrategy balancerStrategy = new CostBalancerStrategyFactory().createBalancerStrategy(exec);
    DruidCoordinatorRuntimeParams params = CoordinatorRuntimeParamsTestHelpers.newBuilder().withDruidCluster(druidCluster).withDynamicConfigs(COORDINATOR_CONFIG_WITH_ZERO_LEADING_TIME_BEFORE_CAN_MARK_AS_UNUSED_OVERSHADOWED_SEGMENTS).withUsedSegmentsInTest(usedSegments).withDatabaseRuleManager(databaseRuleManager).withSegmentReplicantLookup(segmentReplicantLookup).withBalancerStrategy(balancerStrategy).build();
    DruidCoordinatorRuntimeParams afterParams = ruleRunner.run(params);
    CoordinatorStats stats = afterParams.getCoordinatorStats();
    Assert.assertEquals(1L, stats.getTieredStat("droppedCount", "normal"));
    exec.shutdown();
    EasyMock.verify(mockPeon);
    EasyMock.verify(anotherMockPeon);
}