Search in sources :

Example 36 with DruidServer

use of io.druid.client.DruidServer in project druid by druid-io.

the class DruidCoordinatorRuleRunnerTest method testRunThreeTiersOneReplicant.

/**
   * Nodes:
   * hot - 1 replicant
   * normal - 1 replicant
   * cold - 1 replicant
   *
   * @throws Exception
   */
@Test
public void testRunThreeTiersOneReplicant() throws Exception {
    mockCoordinator();
    mockPeon.loadSegment(EasyMock.<DataSegment>anyObject(), EasyMock.<LoadPeonCallback>anyObject());
    EasyMock.expectLastCall().atLeastOnce();
    EasyMock.expect(mockPeon.getSegmentsToLoad()).andReturn(Sets.<DataSegment>newHashSet()).atLeastOnce();
    EasyMock.expect(mockPeon.getLoadQueueSize()).andReturn(0L).atLeastOnce();
    EasyMock.replay(mockPeon);
    EasyMock.expect(databaseRuleManager.getRulesWithDefault(EasyMock.<String>anyObject())).andReturn(Lists.<Rule>newArrayList(new IntervalLoadRule(new Interval("2012-01-01T00:00:00.000Z/2012-01-01T06:00:00.000Z"), ImmutableMap.<String, Integer>of("hot", 1)), new IntervalLoadRule(new Interval("2012-01-01T00:00:00.000Z/2012-01-01T12:00:00.000Z"), ImmutableMap.<String, Integer>of("normal", 1)), new IntervalLoadRule(new Interval("2012-01-01T00:00:00.000Z/2012-01-02T00:00:00.000Z"), ImmutableMap.<String, Integer>of("cold", 1)))).atLeastOnce();
    EasyMock.replay(databaseRuleManager);
    DruidCluster druidCluster = new DruidCluster(ImmutableMap.of("hot", MinMaxPriorityQueue.orderedBy(Ordering.natural().reverse()).create(Arrays.asList(new ServerHolder(new DruidServer("serverHot", "hostHot", 1000, "historical", "hot", 0).toImmutableDruidServer(), mockPeon))), "normal", MinMaxPriorityQueue.orderedBy(Ordering.natural().reverse()).create(Arrays.asList(new ServerHolder(new DruidServer("serverNorm", "hostNorm", 1000, "historical", "normal", 0).toImmutableDruidServer(), mockPeon))), "cold", MinMaxPriorityQueue.orderedBy(Ordering.natural().reverse()).create(Arrays.asList(new ServerHolder(new DruidServer("serverCold", "hostCold", 1000, "historical", "cold", 0).toImmutableDruidServer(), mockPeon)))));
    ListeningExecutorService exec = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(1));
    BalancerStrategy balancerStrategy = new CostBalancerStrategyFactory().createBalancerStrategy(exec);
    DruidCoordinatorRuntimeParams params = new DruidCoordinatorRuntimeParams.Builder().withDruidCluster(druidCluster).withAvailableSegments(availableSegments).withDatabaseRuleManager(databaseRuleManager).withSegmentReplicantLookup(SegmentReplicantLookup.make(new DruidCluster())).withBalancerStrategy(balancerStrategy).withBalancerReferenceTimestamp(new DateTime("2013-01-01")).withDynamicConfigs(new CoordinatorDynamicConfig.Builder().withMaxSegmentsToMove(5).build()).build();
    DruidCoordinatorRuntimeParams afterParams = ruleRunner.run(params);
    CoordinatorStats stats = afterParams.getCoordinatorStats();
    Assert.assertTrue(stats.getPerTierStats().get("assignedCount").get("hot").get() == 6);
    Assert.assertTrue(stats.getPerTierStats().get("assignedCount").get("normal").get() == 6);
    Assert.assertTrue(stats.getPerTierStats().get("assignedCount").get("cold").get() == 12);
    Assert.assertTrue(stats.getPerTierStats().get("unassignedCount") == null);
    Assert.assertTrue(stats.getPerTierStats().get("unassignedSize") == null);
    exec.shutdown();
    EasyMock.verify(mockPeon);
}
Also used : IntervalLoadRule(io.druid.server.coordinator.rules.IntervalLoadRule) ServiceEventBuilder(com.metamx.emitter.service.ServiceEventBuilder) DruidServer(io.druid.client.DruidServer) DataSegment(io.druid.timeline.DataSegment) DateTime(org.joda.time.DateTime) ListeningExecutorService(com.google.common.util.concurrent.ListeningExecutorService) IntervalLoadRule(io.druid.server.coordinator.rules.IntervalLoadRule) ForeverLoadRule(io.druid.server.coordinator.rules.ForeverLoadRule) Rule(io.druid.server.coordinator.rules.Rule) IntervalDropRule(io.druid.server.coordinator.rules.IntervalDropRule) Interval(org.joda.time.Interval) Test(org.junit.Test)

Example 37 with DruidServer

use of io.druid.client.DruidServer in project druid by druid-io.

the class DruidCoordinatorRuleRunnerTest method testDropRemove.

@Test
public void testDropRemove() throws Exception {
    mockPeon.dropSegment(EasyMock.<DataSegment>anyObject(), EasyMock.<LoadPeonCallback>anyObject());
    EasyMock.expectLastCall().atLeastOnce();
    EasyMock.expect(mockPeon.getSegmentsToLoad()).andReturn(Sets.<DataSegment>newHashSet()).atLeastOnce();
    EasyMock.expect(mockPeon.getLoadQueueSize()).andReturn(0L).atLeastOnce();
    EasyMock.replay(mockPeon);
    EasyMock.expect(coordinator.getDynamicConfigs()).andReturn(new CoordinatorDynamicConfig(0, 0, 0, 0, 1, 24, 0, false, null, false)).anyTimes();
    coordinator.removeSegment(EasyMock.<DataSegment>anyObject());
    EasyMock.expectLastCall().atLeastOnce();
    EasyMock.replay(coordinator);
    EasyMock.expect(databaseRuleManager.getRulesWithDefault(EasyMock.<String>anyObject())).andReturn(Lists.<Rule>newArrayList(new IntervalLoadRule(new Interval("2012-01-01T00:00:00.000Z/2012-01-01T12:00:00.000Z"), ImmutableMap.<String, Integer>of("normal", 1)), new IntervalDropRule(new Interval("2012-01-01T00:00:00.000Z/2012-01-02T00:00:00.000Z")))).atLeastOnce();
    EasyMock.replay(databaseRuleManager);
    DruidServer server = new DruidServer("serverNorm", "hostNorm", 1000, "historical", "normal", 0);
    for (DataSegment segment : availableSegments) {
        server.addDataSegment(segment.getIdentifier(), segment);
    }
    DruidCluster druidCluster = new DruidCluster(ImmutableMap.of("normal", MinMaxPriorityQueue.orderedBy(Ordering.natural().reverse()).create(Arrays.asList(new ServerHolder(server.toImmutableDruidServer(), mockPeon)))));
    SegmentReplicantLookup segmentReplicantLookup = SegmentReplicantLookup.make(druidCluster);
    ListeningExecutorService exec = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(1));
    BalancerStrategy balancerStrategy = new CostBalancerStrategyFactory().createBalancerStrategy(exec);
    DruidCoordinatorRuntimeParams params = new DruidCoordinatorRuntimeParams.Builder().withDruidCluster(druidCluster).withDynamicConfigs(new CoordinatorDynamicConfig.Builder().withMillisToWaitBeforeDeleting(0L).build()).withAvailableSegments(availableSegments).withDatabaseRuleManager(databaseRuleManager).withSegmentReplicantLookup(segmentReplicantLookup).withBalancerStrategy(balancerStrategy).withBalancerReferenceTimestamp(new DateTime("2013-01-01")).build();
    DruidCoordinatorRuntimeParams afterParams = ruleRunner.run(params);
    CoordinatorStats stats = afterParams.getCoordinatorStats();
    Assert.assertTrue(stats.getGlobalStats().get("deletedCount").get() == 12);
    exec.shutdown();
    EasyMock.verify(coordinator);
}
Also used : IntervalLoadRule(io.druid.server.coordinator.rules.IntervalLoadRule) ServiceEventBuilder(com.metamx.emitter.service.ServiceEventBuilder) DruidServer(io.druid.client.DruidServer) IntervalDropRule(io.druid.server.coordinator.rules.IntervalDropRule) DataSegment(io.druid.timeline.DataSegment) DateTime(org.joda.time.DateTime) ListeningExecutorService(com.google.common.util.concurrent.ListeningExecutorService) IntervalLoadRule(io.druid.server.coordinator.rules.IntervalLoadRule) ForeverLoadRule(io.druid.server.coordinator.rules.ForeverLoadRule) Rule(io.druid.server.coordinator.rules.Rule) IntervalDropRule(io.druid.server.coordinator.rules.IntervalDropRule) Interval(org.joda.time.Interval) Test(org.junit.Test)

Example 38 with DruidServer

use of io.druid.client.DruidServer in project druid by druid-io.

the class DruidCoordinatorRuleRunnerTest method testDontDropInDifferentTiers.

@Test
public void testDontDropInDifferentTiers() throws Exception {
    mockCoordinator();
    mockPeon.loadSegment(EasyMock.<DataSegment>anyObject(), EasyMock.<LoadPeonCallback>anyObject());
    EasyMock.expectLastCall().atLeastOnce();
    EasyMock.expect(mockPeon.getSegmentsToLoad()).andReturn(Sets.<DataSegment>newHashSet()).atLeastOnce();
    EasyMock.expect(mockPeon.getLoadQueueSize()).andReturn(0L).atLeastOnce();
    EasyMock.replay(mockPeon);
    EasyMock.expect(databaseRuleManager.getRulesWithDefault(EasyMock.<String>anyObject())).andReturn(Lists.<Rule>newArrayList(new IntervalLoadRule(new Interval("2012-01-01T00:00:00.000Z/2012-01-01T12:00:00.000Z"), ImmutableMap.<String, Integer>of("hot", 1)), new IntervalDropRule(new Interval("2012-01-01T00:00:00.000Z/2012-01-02T00:00:00.000Z")))).atLeastOnce();
    EasyMock.replay(databaseRuleManager);
    DruidServer server1 = new DruidServer("server1", "host1", 1000, "historical", "hot", 0);
    DruidServer server2 = new DruidServer("serverNorm2", "hostNorm2", 1000, "historical", "normal", 0);
    for (DataSegment segment : availableSegments) {
        server2.addDataSegment(segment.getIdentifier(), segment);
    }
    DruidCluster druidCluster = new DruidCluster(ImmutableMap.of("hot", MinMaxPriorityQueue.orderedBy(Ordering.natural().reverse()).create(Arrays.asList(new ServerHolder(server1.toImmutableDruidServer(), mockPeon))), "normal", MinMaxPriorityQueue.orderedBy(Ordering.natural().reverse()).create(Arrays.asList(new ServerHolder(server2.toImmutableDruidServer(), mockPeon)))));
    SegmentReplicantLookup segmentReplicantLookup = SegmentReplicantLookup.make(druidCluster);
    ListeningExecutorService exec = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(1));
    BalancerStrategy balancerStrategy = new CostBalancerStrategyFactory().createBalancerStrategy(exec);
    DruidCoordinatorRuntimeParams params = new DruidCoordinatorRuntimeParams.Builder().withDruidCluster(druidCluster).withDynamicConfigs(new CoordinatorDynamicConfig.Builder().withMillisToWaitBeforeDeleting(0L).build()).withAvailableSegments(availableSegments).withDatabaseRuleManager(databaseRuleManager).withSegmentReplicantLookup(segmentReplicantLookup).withBalancerStrategy(balancerStrategy).withBalancerReferenceTimestamp(new DateTime("2013-01-01")).build();
    DruidCoordinatorRuntimeParams afterParams = ruleRunner.run(params);
    CoordinatorStats stats = afterParams.getCoordinatorStats();
    Assert.assertTrue(stats.getPerTierStats().get("droppedCount") == null);
    Assert.assertTrue(stats.getGlobalStats().get("deletedCount").get() == 12);
    exec.shutdown();
    EasyMock.verify(mockPeon);
}
Also used : IntervalLoadRule(io.druid.server.coordinator.rules.IntervalLoadRule) ServiceEventBuilder(com.metamx.emitter.service.ServiceEventBuilder) DruidServer(io.druid.client.DruidServer) IntervalDropRule(io.druid.server.coordinator.rules.IntervalDropRule) DataSegment(io.druid.timeline.DataSegment) DateTime(org.joda.time.DateTime) ListeningExecutorService(com.google.common.util.concurrent.ListeningExecutorService) IntervalLoadRule(io.druid.server.coordinator.rules.IntervalLoadRule) ForeverLoadRule(io.druid.server.coordinator.rules.ForeverLoadRule) Rule(io.druid.server.coordinator.rules.Rule) IntervalDropRule(io.druid.server.coordinator.rules.IntervalDropRule) Interval(org.joda.time.Interval) Test(org.junit.Test)

Example 39 with DruidServer

use of io.druid.client.DruidServer in project druid by druid-io.

the class DruidCoordinatorRuleRunnerTest method testDropTooManyInSameTier.

@Test
public void testDropTooManyInSameTier() throws Exception {
    mockCoordinator();
    mockPeon.dropSegment(EasyMock.<DataSegment>anyObject(), EasyMock.<LoadPeonCallback>anyObject());
    EasyMock.expectLastCall().atLeastOnce();
    EasyMock.expect(mockPeon.getSegmentsToLoad()).andReturn(Sets.<DataSegment>newHashSet()).atLeastOnce();
    EasyMock.expect(mockPeon.getLoadQueueSize()).andReturn(0L).atLeastOnce();
    EasyMock.replay(mockPeon);
    EasyMock.expect(databaseRuleManager.getRulesWithDefault(EasyMock.<String>anyObject())).andReturn(Lists.<Rule>newArrayList(new IntervalLoadRule(new Interval("2012-01-01T00:00:00.000Z/2012-01-01T12:00:00.000Z"), ImmutableMap.<String, Integer>of("normal", 1)), new IntervalDropRule(new Interval("2012-01-01T00:00:00.000Z/2012-01-02T00:00:00.000Z")))).atLeastOnce();
    EasyMock.replay(databaseRuleManager);
    DruidServer server1 = new DruidServer("serverNorm", "hostNorm", 1000, "historical", "normal", 0);
    server1.addDataSegment(availableSegments.get(0).getIdentifier(), availableSegments.get(0));
    DruidServer server2 = new DruidServer("serverNorm2", "hostNorm2", 1000, "historical", "normal", 0);
    for (DataSegment segment : availableSegments) {
        server2.addDataSegment(segment.getIdentifier(), segment);
    }
    DruidCluster druidCluster = new DruidCluster(ImmutableMap.of("normal", MinMaxPriorityQueue.orderedBy(Ordering.natural().reverse()).create(Arrays.asList(new ServerHolder(server1.toImmutableDruidServer(), mockPeon), new ServerHolder(server2.toImmutableDruidServer(), mockPeon)))));
    SegmentReplicantLookup segmentReplicantLookup = SegmentReplicantLookup.make(druidCluster);
    ListeningExecutorService exec = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(1));
    BalancerStrategy balancerStrategy = new CostBalancerStrategyFactory().createBalancerStrategy(exec);
    DruidCoordinatorRuntimeParams params = new DruidCoordinatorRuntimeParams.Builder().withDruidCluster(druidCluster).withDynamicConfigs(new CoordinatorDynamicConfig.Builder().withMillisToWaitBeforeDeleting(0L).build()).withAvailableSegments(availableSegments).withDatabaseRuleManager(databaseRuleManager).withSegmentReplicantLookup(segmentReplicantLookup).withBalancerStrategy(balancerStrategy).withBalancerReferenceTimestamp(new DateTime("2013-01-01")).build();
    DruidCoordinatorRuntimeParams afterParams = ruleRunner.run(params);
    CoordinatorStats stats = afterParams.getCoordinatorStats();
    Assert.assertTrue(stats.getPerTierStats().get("droppedCount").get("normal").get() == 1);
    Assert.assertTrue(stats.getGlobalStats().get("deletedCount").get() == 12);
    exec.shutdown();
    EasyMock.verify(mockPeon);
}
Also used : IntervalLoadRule(io.druid.server.coordinator.rules.IntervalLoadRule) ServiceEventBuilder(com.metamx.emitter.service.ServiceEventBuilder) DruidServer(io.druid.client.DruidServer) IntervalDropRule(io.druid.server.coordinator.rules.IntervalDropRule) DataSegment(io.druid.timeline.DataSegment) DateTime(org.joda.time.DateTime) ListeningExecutorService(com.google.common.util.concurrent.ListeningExecutorService) IntervalLoadRule(io.druid.server.coordinator.rules.IntervalLoadRule) ForeverLoadRule(io.druid.server.coordinator.rules.ForeverLoadRule) Rule(io.druid.server.coordinator.rules.Rule) IntervalDropRule(io.druid.server.coordinator.rules.IntervalDropRule) Interval(org.joda.time.Interval) Test(org.junit.Test)

Example 40 with DruidServer

use of io.druid.client.DruidServer in project druid by druid-io.

the class DruidCoordinatorRuleRunnerTest method testRulesRunOnNonOvershadowedSegmentsOnly.

@Test
public void testRulesRunOnNonOvershadowedSegmentsOnly() throws Exception {
    Set<DataSegment> availableSegments = new HashSet<>();
    DataSegment v1 = new DataSegment("test", new Interval("2012-01-01/2012-01-02"), "1", Maps.<String, Object>newHashMap(), Lists.<String>newArrayList(), Lists.<String>newArrayList(), NoneShardSpec.instance(), IndexIO.CURRENT_VERSION_ID, 1);
    DataSegment v2 = new DataSegment("test", new Interval("2012-01-01/2012-01-02"), "2", Maps.<String, Object>newHashMap(), Lists.<String>newArrayList(), Lists.<String>newArrayList(), NoneShardSpec.instance(), IndexIO.CURRENT_VERSION_ID, 1);
    availableSegments.add(v1);
    availableSegments.add(v2);
    mockCoordinator();
    mockPeon.loadSegment(EasyMock.eq(v2), EasyMock.<LoadPeonCallback>anyObject());
    EasyMock.expectLastCall().once();
    EasyMock.expect(mockPeon.getSegmentsToLoad()).andReturn(Sets.<DataSegment>newHashSet()).atLeastOnce();
    EasyMock.expect(mockPeon.getLoadQueueSize()).andReturn(0L).atLeastOnce();
    EasyMock.replay(mockPeon);
    EasyMock.expect(databaseRuleManager.getRulesWithDefault(EasyMock.<String>anyObject())).andReturn(Lists.<Rule>newArrayList(new ForeverLoadRule(ImmutableMap.of(DruidServer.DEFAULT_TIER, 1)))).atLeastOnce();
    EasyMock.replay(databaseRuleManager);
    DruidCluster druidCluster = new DruidCluster(ImmutableMap.of(DruidServer.DEFAULT_TIER, MinMaxPriorityQueue.orderedBy(Ordering.natural().reverse()).create(Arrays.asList(new ServerHolder(new DruidServer("serverHot", "hostHot", 1000, "historical", DruidServer.DEFAULT_TIER, 0).toImmutableDruidServer(), mockPeon)))));
    ListeningExecutorService exec = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(1));
    BalancerStrategy balancerStrategy = new CostBalancerStrategyFactory().createBalancerStrategy(exec);
    DruidCoordinatorRuntimeParams params = new DruidCoordinatorRuntimeParams.Builder().withDruidCluster(druidCluster).withAvailableSegments(availableSegments).withDatabaseRuleManager(databaseRuleManager).withSegmentReplicantLookup(SegmentReplicantLookup.make(new DruidCluster())).withBalancerStrategy(balancerStrategy).withBalancerReferenceTimestamp(new DateTime("2013-01-01")).withDynamicConfigs(new CoordinatorDynamicConfig.Builder().withMaxSegmentsToMove(5).build()).build();
    DruidCoordinatorRuntimeParams afterParams = ruleRunner.run(params);
    CoordinatorStats stats = afterParams.getCoordinatorStats();
    Assert.assertEquals(1, stats.getPerTierStats().get("assignedCount").size());
    Assert.assertEquals(1, stats.getPerTierStats().get("assignedCount").get("_default_tier").get());
    Assert.assertNull(stats.getPerTierStats().get("unassignedCount"));
    Assert.assertNull(stats.getPerTierStats().get("unassignedSize"));
    Assert.assertEquals(2, availableSegments.size());
    Assert.assertEquals(availableSegments, params.getAvailableSegments());
    Assert.assertEquals(availableSegments, afterParams.getAvailableSegments());
    EasyMock.verify(mockPeon);
    exec.shutdown();
}
Also used : ServiceEventBuilder(com.metamx.emitter.service.ServiceEventBuilder) DruidServer(io.druid.client.DruidServer) DataSegment(io.druid.timeline.DataSegment) DateTime(org.joda.time.DateTime) ForeverLoadRule(io.druid.server.coordinator.rules.ForeverLoadRule) ListeningExecutorService(com.google.common.util.concurrent.ListeningExecutorService) IntervalLoadRule(io.druid.server.coordinator.rules.IntervalLoadRule) ForeverLoadRule(io.druid.server.coordinator.rules.ForeverLoadRule) Rule(io.druid.server.coordinator.rules.Rule) IntervalDropRule(io.druid.server.coordinator.rules.IntervalDropRule) HashSet(java.util.HashSet) Interval(org.joda.time.Interval) Test(org.junit.Test)

Aggregations

DruidServer (io.druid.client.DruidServer)47 DataSegment (io.druid.timeline.DataSegment)36 Test (org.junit.Test)30 Interval (org.joda.time.Interval)25 DateTime (org.joda.time.DateTime)20 ListeningExecutorService (com.google.common.util.concurrent.ListeningExecutorService)18 ForeverLoadRule (io.druid.server.coordinator.rules.ForeverLoadRule)15 Rule (io.druid.server.coordinator.rules.Rule)15 IntervalDropRule (io.druid.server.coordinator.rules.IntervalDropRule)14 IntervalLoadRule (io.druid.server.coordinator.rules.IntervalLoadRule)14 ServiceEventBuilder (com.metamx.emitter.service.ServiceEventBuilder)11 Map (java.util.Map)11 DruidDataSource (io.druid.client.DruidDataSource)7 ImmutableMap (com.google.common.collect.ImmutableMap)6 Response (javax.ws.rs.core.Response)6 ImmutableDruidServer (io.druid.client.ImmutableDruidServer)5 DruidServerMetadata (io.druid.server.coordination.DruidServerMetadata)5 GET (javax.ws.rs.GET)5 Produces (javax.ws.rs.Produces)5 BalancerStrategy (io.druid.server.coordinator.BalancerStrategy)4