Search in sources :

Example 11 with IntervalLoadRule

use of org.apache.druid.server.coordinator.rules.IntervalLoadRule in project druid by druid-io.

the class RunRulesTest method testRunTwoTiersTierDoesNotExist.

@Test
public void testRunTwoTiersTierDoesNotExist() {
    mockCoordinator();
    mockPeon.loadSegment(EasyMock.anyObject(), EasyMock.anyObject());
    EasyMock.expectLastCall().atLeastOnce();
    mockEmptyPeon();
    emitter.emit(EasyMock.<ServiceEventBuilder>anyObject());
    EasyMock.expectLastCall().times(12);
    EasyMock.replay(emitter);
    EasyMock.expect(databaseRuleManager.getRulesWithDefault(EasyMock.anyObject())).andReturn(Lists.newArrayList(new IntervalLoadRule(Intervals.of("2012-01-01T00:00:00.000Z/2012-01-01T12:00:00.000Z"), ImmutableMap.of("hot", 1)), new IntervalLoadRule(Intervals.of("2012-01-01T00:00:00.000Z/2012-01-02T00:00:00.000Z"), ImmutableMap.of("normal", 1)))).atLeastOnce();
    EasyMock.replay(databaseRuleManager);
    DruidCluster druidCluster = DruidClusterBuilder.newBuilder().addTier("normal", new ServerHolder(new DruidServer("serverNorm", "hostNorm", null, 1000, ServerType.HISTORICAL, "normal", 0).toImmutableDruidServer(), mockPeon)).build();
    ListeningExecutorService exec = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(1));
    BalancerStrategy balancerStrategy = new CostBalancerStrategyFactory().createBalancerStrategy(exec);
    DruidCoordinatorRuntimeParams params = makeCoordinatorRuntimeParams(druidCluster, balancerStrategy).withEmitter(emitter).build();
    ruleRunner.run(params);
    exec.shutdown();
    EasyMock.verify(emitter);
    EasyMock.verify(mockPeon);
}
Also used : IntervalLoadRule(org.apache.druid.server.coordinator.rules.IntervalLoadRule) DruidServer(org.apache.druid.client.DruidServer) ListeningExecutorService(com.google.common.util.concurrent.ListeningExecutorService) Test(org.junit.Test)

Example 12 with IntervalLoadRule

use of org.apache.druid.server.coordinator.rules.IntervalLoadRule in project druid by druid-io.

the class SQLMetadataRuleManagerTest method testRemoveRulesOlderThanWithNonExistenceDatasourceAndNewerThanTimestampShouldNotDelete.

@Test
public void testRemoveRulesOlderThanWithNonExistenceDatasourceAndNewerThanTimestampShouldNotDelete() {
    List<Rule> rules = ImmutableList.of(new IntervalLoadRule(Intervals.of("2015-01-01/2015-02-01"), ImmutableMap.of(DruidServer.DEFAULT_TIER, DruidServer.DEFAULT_NUM_REPLICANTS)));
    AuditInfo auditInfo = new AuditInfo("test_author", "test_comment", "127.0.0.1");
    ruleManager.overrideRule("test_dataSource", rules, auditInfo);
    // Verify that rule was added
    ruleManager.poll();
    Map<String, List<Rule>> allRules = ruleManager.getAllRules();
    Assert.assertEquals(1, allRules.size());
    Assert.assertEquals(1, allRules.get("test_dataSource").size());
    // This will not delete the rule as the rule was created just now so it will have the created timestamp later than
    // the timestamp 2012-01-01T00:00:00Z
    ruleManager.removeRulesForEmptyDatasourcesOlderThan(DateTimes.of("2012-01-01T00:00:00Z").getMillis());
    // Verify that rule was not deleted
    ruleManager.poll();
    allRules = ruleManager.getAllRules();
    Assert.assertEquals(1, allRules.size());
    Assert.assertEquals(1, allRules.get("test_dataSource").size());
}
Also used : AuditInfo(org.apache.druid.audit.AuditInfo) IntervalLoadRule(org.apache.druid.server.coordinator.rules.IntervalLoadRule) ImmutableList(com.google.common.collect.ImmutableList) List(java.util.List) IntervalLoadRule(org.apache.druid.server.coordinator.rules.IntervalLoadRule) Rule(org.apache.druid.server.coordinator.rules.Rule) Test(org.junit.Test)

Example 13 with IntervalLoadRule

use of org.apache.druid.server.coordinator.rules.IntervalLoadRule in project druid by druid-io.

the class SQLMetadataRuleManagerTest method testAuditEntryCreated.

@Test
public void testAuditEntryCreated() throws Exception {
    List<Rule> rules = Collections.singletonList(new IntervalLoadRule(Intervals.of("2015-01-01/2015-02-01"), ImmutableMap.of(DruidServer.DEFAULT_TIER, DruidServer.DEFAULT_NUM_REPLICANTS)));
    AuditInfo auditInfo = new AuditInfo("test_author", "test_comment", "127.0.0.1");
    ruleManager.overrideRule("test_dataSource", rules, auditInfo);
    // fetch rules from metadata storage
    ruleManager.poll();
    Assert.assertEquals(rules, ruleManager.getRules("test_dataSource"));
    // verify audit entry is created
    List<AuditEntry> auditEntries = auditManager.fetchAuditHistory("test_dataSource", "rules", null);
    Assert.assertEquals(1, auditEntries.size());
    AuditEntry entry = auditEntries.get(0);
    Assert.assertEquals(rules, mapper.readValue(entry.getPayload(), new TypeReference<List<Rule>>() {
    }));
    Assert.assertEquals(auditInfo, entry.getAuditInfo());
    Assert.assertEquals("test_dataSource", entry.getKey());
}
Also used : AuditInfo(org.apache.druid.audit.AuditInfo) IntervalLoadRule(org.apache.druid.server.coordinator.rules.IntervalLoadRule) AuditEntry(org.apache.druid.audit.AuditEntry) IntervalLoadRule(org.apache.druid.server.coordinator.rules.IntervalLoadRule) Rule(org.apache.druid.server.coordinator.rules.Rule) TypeReference(com.fasterxml.jackson.core.type.TypeReference) Test(org.junit.Test)

Example 14 with IntervalLoadRule

use of org.apache.druid.server.coordinator.rules.IntervalLoadRule in project druid by druid-io.

the class SQLMetadataRuleManagerTest method testRemoveRulesOlderThanWithActiveDatasourceShouldNotDelete.

@Test
public void testRemoveRulesOlderThanWithActiveDatasourceShouldNotDelete() throws Exception {
    List<Rule> rules = ImmutableList.of(new IntervalLoadRule(Intervals.of("2015-01-01/2015-02-01"), ImmutableMap.of(DruidServer.DEFAULT_TIER, DruidServer.DEFAULT_NUM_REPLICANTS)));
    AuditInfo auditInfo = new AuditInfo("test_author", "test_comment", "127.0.0.1");
    ruleManager.overrideRule("test_dataSource", rules, auditInfo);
    // Verify that rule was added
    ruleManager.poll();
    Map<String, List<Rule>> allRules = ruleManager.getAllRules();
    Assert.assertEquals(1, allRules.size());
    Assert.assertEquals(1, allRules.get("test_dataSource").size());
    // Add segment metadata to segment table so that the datasource is considered active
    DataSegment dataSegment = new DataSegment("test_dataSource", Intervals.of("2015-01-01/2015-02-01"), "1", ImmutableMap.of("type", "s3_zip", "bucket", "test", "key", "test_dataSource/xxx"), ImmutableList.of("dim1", "dim2", "dim3"), ImmutableList.of("count", "value"), NoneShardSpec.instance(), 1, 1234L);
    publisher.publishSegment(dataSegment);
    // This will not delete the rule as the datasource has segment in the segment metadata table
    ruleManager.removeRulesForEmptyDatasourcesOlderThan(System.currentTimeMillis());
    // Verify that rule was not deleted
    ruleManager.poll();
    allRules = ruleManager.getAllRules();
    Assert.assertEquals(1, allRules.size());
    Assert.assertEquals(1, allRules.get("test_dataSource").size());
}
Also used : AuditInfo(org.apache.druid.audit.AuditInfo) IntervalLoadRule(org.apache.druid.server.coordinator.rules.IntervalLoadRule) ImmutableList(com.google.common.collect.ImmutableList) List(java.util.List) IntervalLoadRule(org.apache.druid.server.coordinator.rules.IntervalLoadRule) Rule(org.apache.druid.server.coordinator.rules.Rule) DataSegment(org.apache.druid.timeline.DataSegment) Test(org.junit.Test)

Example 15 with IntervalLoadRule

use of org.apache.druid.server.coordinator.rules.IntervalLoadRule in project druid by druid-io.

the class DataSourcesResourceTest method testIsHandOffComplete.

@Test
public void testIsHandOffComplete() {
    MetadataRuleManager databaseRuleManager = EasyMock.createMock(MetadataRuleManager.class);
    Rule loadRule = new IntervalLoadRule(Intervals.of("2013-01-02T00:00:00Z/2013-01-03T00:00:00Z"), null);
    Rule dropRule = new IntervalDropRule(Intervals.of("2013-01-01T00:00:00Z/2013-01-02T00:00:00Z"));
    DataSourcesResource dataSourcesResource = new DataSourcesResource(inventoryView, null, databaseRuleManager, null, null, null);
    // test dropped
    EasyMock.expect(databaseRuleManager.getRulesWithDefault("dataSource1")).andReturn(ImmutableList.of(loadRule, dropRule)).once();
    EasyMock.replay(databaseRuleManager);
    String interval1 = "2013-01-01T01:00:00Z/2013-01-01T02:00:00Z";
    Response response1 = dataSourcesResource.isHandOffComplete("dataSource1", interval1, 1, "v1");
    Assert.assertTrue((boolean) response1.getEntity());
    EasyMock.verify(databaseRuleManager);
    // test isn't dropped and no timeline found
    EasyMock.reset(databaseRuleManager);
    EasyMock.expect(databaseRuleManager.getRulesWithDefault("dataSource1")).andReturn(ImmutableList.of(loadRule, dropRule)).once();
    EasyMock.expect(inventoryView.getTimeline(new TableDataSource("dataSource1"))).andReturn(null).once();
    EasyMock.replay(inventoryView, databaseRuleManager);
    String interval2 = "2013-01-02T01:00:00Z/2013-01-02T02:00:00Z";
    Response response2 = dataSourcesResource.isHandOffComplete("dataSource1", interval2, 1, "v1");
    Assert.assertFalse((boolean) response2.getEntity());
    EasyMock.verify(inventoryView, databaseRuleManager);
    // test isn't dropped and timeline exist
    String interval3 = "2013-01-02T02:00:00Z/2013-01-02T03:00:00Z";
    SegmentLoadInfo segmentLoadInfo = new SegmentLoadInfo(createSegment(Intervals.of(interval3), "v1", 1));
    segmentLoadInfo.addServer(createHistoricalServerMetadata("test"));
    VersionedIntervalTimeline<String, SegmentLoadInfo> timeline = new VersionedIntervalTimeline<String, SegmentLoadInfo>(null) {

        @Override
        public List<TimelineObjectHolder<String, SegmentLoadInfo>> lookupWithIncompletePartitions(Interval interval) {
            PartitionHolder<SegmentLoadInfo> partitionHolder = new PartitionHolder<>(new NumberedPartitionChunk<>(1, 1, segmentLoadInfo));
            List<TimelineObjectHolder<String, SegmentLoadInfo>> ret = new ArrayList<>();
            ret.add(new TimelineObjectHolder<>(Intervals.of(interval3), "v1", partitionHolder));
            return ret;
        }
    };
    EasyMock.reset(inventoryView, databaseRuleManager);
    EasyMock.expect(databaseRuleManager.getRulesWithDefault("dataSource1")).andReturn(ImmutableList.of(loadRule, dropRule)).once();
    EasyMock.expect(inventoryView.getTimeline(new TableDataSource("dataSource1"))).andReturn(timeline).once();
    EasyMock.replay(inventoryView, databaseRuleManager);
    Response response3 = dataSourcesResource.isHandOffComplete("dataSource1", interval3, 1, "v1");
    Assert.assertTrue((boolean) response3.getEntity());
    EasyMock.verify(inventoryView, databaseRuleManager);
}
Also used : PartitionHolder(org.apache.druid.timeline.partition.PartitionHolder) MetadataRuleManager(org.apache.druid.metadata.MetadataRuleManager) IntervalLoadRule(org.apache.druid.server.coordinator.rules.IntervalLoadRule) SegmentLoadInfo(org.apache.druid.client.SegmentLoadInfo) ImmutableSegmentLoadInfo(org.apache.druid.client.ImmutableSegmentLoadInfo) ArrayList(java.util.ArrayList) IntervalDropRule(org.apache.druid.server.coordinator.rules.IntervalDropRule) Response(javax.ws.rs.core.Response) TimelineObjectHolder(org.apache.druid.timeline.TimelineObjectHolder) TableDataSource(org.apache.druid.query.TableDataSource) VersionedIntervalTimeline(org.apache.druid.timeline.VersionedIntervalTimeline) IntervalLoadRule(org.apache.druid.server.coordinator.rules.IntervalLoadRule) Rule(org.apache.druid.server.coordinator.rules.Rule) IntervalDropRule(org.apache.druid.server.coordinator.rules.IntervalDropRule) Interval(org.joda.time.Interval) Test(org.junit.Test)

Aggregations

IntervalLoadRule (org.apache.druid.server.coordinator.rules.IntervalLoadRule)23 Test (org.junit.Test)23 DruidServer (org.apache.druid.client.DruidServer)16 ListeningExecutorService (com.google.common.util.concurrent.ListeningExecutorService)14 DataSegment (org.apache.druid.timeline.DataSegment)9 Rule (org.apache.druid.server.coordinator.rules.Rule)8 AuditInfo (org.apache.druid.audit.AuditInfo)6 IntervalDropRule (org.apache.druid.server.coordinator.rules.IntervalDropRule)5 ImmutableList (com.google.common.collect.ImmutableList)4 List (java.util.List)4 TypeReference (com.fasterxml.jackson.core.type.TypeReference)2 AuditEntry (org.apache.druid.audit.AuditEntry)2 Object2LongMap (it.unimi.dsi.fastutil.objects.Object2LongMap)1 ArrayList (java.util.ArrayList)1 HashSet (java.util.HashSet)1 CountDownLatch (java.util.concurrent.CountDownLatch)1 Response (javax.ws.rs.core.Response)1 PathChildrenCache (org.apache.curator.framework.recipes.cache.PathChildrenCache)1 DruidDataSource (org.apache.druid.client.DruidDataSource)1 ImmutableDruidDataSource (org.apache.druid.client.ImmutableDruidDataSource)1