Examples with DruidServer io.druid.client.DruidServer used on opensource projects

Example 16 with DruidServer

use of io.druid.client.DruidServer in project druid by druid-io.

the class DruidCoordinatorTest method testMoveSegment.

@Test
public void testMoveSegment() throws Exception {
    loadQueuePeon = EasyMock.createNiceMock(LoadQueuePeon.class);
    EasyMock.expect(loadQueuePeon.getLoadQueueSize()).andReturn(new Long(1));
    EasyMock.replay(loadQueuePeon);
    segment = EasyMock.createNiceMock(DataSegment.class);
    EasyMock.replay(segment);
    scheduledExecutorFactory = EasyMock.createNiceMock(ScheduledExecutorFactory.class);
    EasyMock.replay(scheduledExecutorFactory);
    EasyMock.replay(metadataRuleManager);
    EasyMock.expect(druidServer.toImmutableDruidServer()).andReturn(new ImmutableDruidServer(new DruidServerMetadata("from", null, 5L, null, null, 0), 1L, null, ImmutableMap.of("dummySegment", segment))).atLeastOnce();
    EasyMock.replay(druidServer);
    druidServer2 = EasyMock.createMock(DruidServer.class);
    EasyMock.expect(druidServer2.toImmutableDruidServer()).andReturn(new ImmutableDruidServer(new DruidServerMetadata("to", null, 5L, null, null, 0), 1L, null, ImmutableMap.of("dummySegment2", segment))).atLeastOnce();
    EasyMock.replay(druidServer2);
    loadManagementPeons.put("from", loadQueuePeon);
    loadManagementPeons.put("to", loadQueuePeon);
    EasyMock.expect(serverInventoryView.getInventoryManagerConfig()).andReturn(new InventoryManagerConfig() {

        @Override
        public String getContainerPath() {
            return "";
        }

        @Override
        public String getInventoryPath() {
            return "";
        }
    });
    EasyMock.replay(serverInventoryView);
    coordinator.moveSegment(druidServer.toImmutableDruidServer(), druidServer2.toImmutableDruidServer(), "dummySegment", null);
    EasyMock.verify(druidServer);
    EasyMock.verify(druidServer2);
    EasyMock.verify(loadQueuePeon);
    EasyMock.verify(serverInventoryView);
    EasyMock.verify(metadataRuleManager);
}

Example 17 with DruidServer

use of io.druid.client.DruidServer in project druid by druid-io.

the class DruidCoordinatorTest method testCoordinatorRun.

@Test(timeout = 60_000L)
public void testCoordinatorRun() throws Exception {
    String dataSource = "dataSource1";
    String tier = "hot";
    // Setup MetadataRuleManager
    Rule foreverLoadRule = new ForeverLoadRule(ImmutableMap.of(tier, 2));
    EasyMock.expect(metadataRuleManager.getRulesWithDefault(EasyMock.anyString())).andReturn(ImmutableList.of(foreverLoadRule)).atLeastOnce();
    metadataRuleManager.stop();
    EasyMock.expectLastCall().once();
    EasyMock.replay(metadataRuleManager);
    // Setup MetadataSegmentManager
    DruidDataSource[] druidDataSources = { new DruidDataSource(dataSource, Collections.<String, String>emptyMap()) };
    final DataSegment dataSegment = new DataSegment(dataSource, new Interval("2010-01-01/P1D"), "v1", null, null, null, null, 0x9, 0);
    druidDataSources[0].addSegment("0", dataSegment);
    EasyMock.expect(databaseSegmentManager.isStarted()).andReturn(true).anyTimes();
    EasyMock.expect(databaseSegmentManager.getInventory()).andReturn(ImmutableList.of(druidDataSources[0])).atLeastOnce();
    EasyMock.replay(databaseSegmentManager);
    ImmutableDruidDataSource immutableDruidDataSource = EasyMock.createNiceMock(ImmutableDruidDataSource.class);
    EasyMock.expect(immutableDruidDataSource.getSegments()).andReturn(ImmutableSet.of(dataSegment)).atLeastOnce();
    EasyMock.replay(immutableDruidDataSource);
    // Setup ServerInventoryView
    druidServer = new DruidServer("server1", "localhost", 5L, "historical", tier, 0);
    loadManagementPeons.put("server1", loadQueuePeon);
    EasyMock.expect(serverInventoryView.getInventory()).andReturn(ImmutableList.of(druidServer)).atLeastOnce();
    EasyMock.expect(serverInventoryView.isStarted()).andReturn(true).anyTimes();
    EasyMock.replay(serverInventoryView);
    coordinator.start();
    // Wait for this coordinator to become leader
    leaderAnnouncerLatch.await();
    // This coordinator should be leader by now
    Assert.assertTrue(coordinator.isLeader());
    Assert.assertEquals(druidNode.getHostAndPort(), coordinator.getCurrentLeader());
    final CountDownLatch assignSegmentLatch = new CountDownLatch(1);
    pathChildrenCache.getListenable().addListener(new PathChildrenCacheListener() {

        @Override
        public void childEvent(CuratorFramework curatorFramework, PathChildrenCacheEvent pathChildrenCacheEvent) throws Exception {
            if (pathChildrenCacheEvent.getType().equals(PathChildrenCacheEvent.Type.CHILD_ADDED)) {
                //Coordinator should try to assign segment to druidServer historical
                //Simulate historical loading segment
                druidServer.addDataSegment(dataSegment.getIdentifier(), dataSegment);
                assignSegmentLatch.countDown();
            }
        }
    });
    pathChildrenCache.start();
    assignSegmentLatch.await();
    Assert.assertEquals(ImmutableMap.of(dataSource, 100.0), coordinator.getLoadStatus());
    curator.delete().guaranteed().forPath(ZKPaths.makePath(LOADPATH, dataSegment.getIdentifier()));
    // Wait for coordinator thread to run so that replication status is updated
    while (coordinator.getSegmentAvailability().snapshot().get(dataSource) != 0) {
        Thread.sleep(50);
    }
    Map segmentAvailability = coordinator.getSegmentAvailability().snapshot();
    Assert.assertEquals(1, segmentAvailability.size());
    Assert.assertEquals(0L, segmentAvailability.get(dataSource));
    while (coordinator.getLoadPendingDatasources().get(dataSource).get() > 0) {
        Thread.sleep(50);
    }
    // wait historical data to be updated
    long startMillis = System.currentTimeMillis();
    long coordinatorRunPeriodMillis = druidCoordinatorConfig.getCoordinatorPeriod().getMillis();
    while (System.currentTimeMillis() - startMillis < coordinatorRunPeriodMillis) {
        Thread.sleep(100);
    }
    Map<String, CountingMap<String>> replicationStatus = coordinator.getReplicationStatus();
    Assert.assertNotNull(replicationStatus);
    Assert.assertEquals(1, replicationStatus.entrySet().size());
    CountingMap<String> dataSourceMap = replicationStatus.get(tier);
    Assert.assertNotNull(dataSourceMap);
    Assert.assertEquals(1, dataSourceMap.size());
    Assert.assertNotNull(dataSourceMap.get(dataSource));
    // The load rules asks for 2 replicas, therefore 1 replica should still be pending
    while (dataSourceMap.get(dataSource).get() != 1L) {
        Thread.sleep(50);
    }
    coordinator.stop();
    leaderUnannouncerLatch.await();
    Assert.assertFalse(coordinator.isLeader());
    Assert.assertNull(coordinator.getCurrentLeader());
    EasyMock.verify(serverInventoryView);
    EasyMock.verify(metadataRuleManager);
}

Example 18 with DruidServer

use of io.druid.client.DruidServer in project druid by druid-io.

the class DatasourcesResourceTest method testGetSegmentDataSourceSpecificInterval.

@Test
public void testGetSegmentDataSourceSpecificInterval() {
    server = new DruidServer("who", "host", 1234, "historical", "tier1", 0);
    server.addDataSegment(dataSegmentList.get(0).getIdentifier(), dataSegmentList.get(0));
    server.addDataSegment(dataSegmentList.get(1).getIdentifier(), dataSegmentList.get(1));
    server.addDataSegment(dataSegmentList.get(2).getIdentifier(), dataSegmentList.get(2));
    EasyMock.expect(inventoryView.getInventory()).andReturn(ImmutableList.of(server)).atLeastOnce();
    EasyMock.replay(inventoryView);
    DatasourcesResource datasourcesResource = new DatasourcesResource(inventoryView, null, null, new AuthConfig());
    Response response = datasourcesResource.getSegmentDataSourceSpecificInterval("invalidDataSource", "2010-01-01/P1D", null, null);
    Assert.assertEquals(null, response.getEntity());
    response = datasourcesResource.getSegmentDataSourceSpecificInterval("datasource1", "2010-03-01/P1D", null, null);
    // interval not present in the datasource
    Assert.assertEquals(ImmutableSet.of(), response.getEntity());
    response = datasourcesResource.getSegmentDataSourceSpecificInterval("datasource1", "2010-01-01/P1D", null, null);
    Assert.assertEquals(ImmutableSet.of(dataSegmentList.get(0).getIdentifier()), response.getEntity());
    response = datasourcesResource.getSegmentDataSourceSpecificInterval("datasource1", "2010-01-01/P1M", null, null);
    Assert.assertEquals(ImmutableSet.of(dataSegmentList.get(1).getIdentifier(), dataSegmentList.get(0).getIdentifier()), response.getEntity());
    response = datasourcesResource.getSegmentDataSourceSpecificInterval("datasource1", "2010-01-01/P1M", "simple", null);
    HashMap<Interval, Map<String, Object>> results = ((HashMap<Interval, Map<String, Object>>) response.getEntity());
    Assert.assertEquals(2, results.size());
    int i;
    for (i = 0; i < 2; i++) {
        Assert.assertTrue(results.containsKey(dataSegmentList.get(i).getInterval()));
        Assert.assertEquals(1, (results.get(dataSegmentList.get(i).getInterval())).get("count"));
    }
    response = datasourcesResource.getSegmentDataSourceSpecificInterval("datasource1", "2010-01-01/P1M", null, "full");
    TreeMap<Interval, Map<String, Object>> results1 = ((TreeMap<Interval, Map<String, Object>>) response.getEntity());
    i = 1;
    for (Map.Entry<Interval, Map<String, Object>> entry : results1.entrySet()) {
        Assert.assertEquals(dataSegmentList.get(i).getInterval(), entry.getKey());
        Assert.assertEquals(dataSegmentList.get(i), ((Map<String, Object>) entry.getValue().get(dataSegmentList.get(i).getIdentifier())).get("metadata"));
        i--;
    }
    EasyMock.verify(inventoryView);
}

Example 19 with DruidServer

use of io.druid.client.DruidServer in project druid by druid-io.

the class DatasourcesResourceTest method testSimpleGetTheDataSourceManyTiers.

@Test
public void testSimpleGetTheDataSourceManyTiers() throws Exception {
    EasyMock.expect(server.getDataSource("datasource1")).andReturn(listDataSources.get(0)).atLeastOnce();
    EasyMock.expect(server.getTier()).andReturn("cold").atLeastOnce();
    DruidServer server2 = EasyMock.createStrictMock(DruidServer.class);
    EasyMock.expect(server2.getDataSource("datasource1")).andReturn(listDataSources.get(1)).atLeastOnce();
    EasyMock.expect(server2.getTier()).andReturn("hot").atLeastOnce();
    DruidServer server3 = EasyMock.createStrictMock(DruidServer.class);
    EasyMock.expect(server3.getDataSource("datasource1")).andReturn(listDataSources.get(1)).atLeastOnce();
    EasyMock.expect(server3.getTier()).andReturn("cold").atLeastOnce();
    EasyMock.expect(inventoryView.getInventory()).andReturn(ImmutableList.of(server, server2, server3)).atLeastOnce();
    EasyMock.replay(inventoryView, server, server2, server3);
    DatasourcesResource datasourcesResource = new DatasourcesResource(inventoryView, null, null, new AuthConfig());
    Response response = datasourcesResource.getTheDataSource("datasource1", null);
    Assert.assertEquals(200, response.getStatus());
    Map<String, Map<String, Object>> result = (Map<String, Map<String, Object>>) response.getEntity();
    Assert.assertEquals(2, ((Map) (result.get("tiers").get("cold"))).get("segmentCount"));
    Assert.assertEquals(30L, ((Map) (result.get("tiers").get("cold"))).get("size"));
    Assert.assertEquals(1, ((Map) (result.get("tiers").get("hot"))).get("segmentCount"));
    Assert.assertEquals(20L, ((Map) (result.get("tiers").get("hot"))).get("size"));
    Assert.assertNotNull(result.get("segments"));
    Assert.assertEquals("2010-01-01T00:00:00.000Z", result.get("segments").get("minTime").toString());
    Assert.assertEquals("2010-01-23T00:00:00.000Z", result.get("segments").get("maxTime").toString());
    Assert.assertEquals(2, result.get("segments").get("count"));
    Assert.assertEquals(30L, result.get("segments").get("size"));
    EasyMock.verify(inventoryView, server, server2, server3);
}

Example 20 with DruidServer

use of io.druid.client.DruidServer in project druid by druid-io.

the class DatasourcesResourceTest method testGetSegmentDataSourceIntervals.

@Test
public void testGetSegmentDataSourceIntervals() {
    server = new DruidServer("who", "host", 1234, "historical", "tier1", 0);
    server.addDataSegment(dataSegmentList.get(0).getIdentifier(), dataSegmentList.get(0));
    server.addDataSegment(dataSegmentList.get(1).getIdentifier(), dataSegmentList.get(1));
    server.addDataSegment(dataSegmentList.get(2).getIdentifier(), dataSegmentList.get(2));
    EasyMock.expect(inventoryView.getInventory()).andReturn(ImmutableList.of(server)).atLeastOnce();
    EasyMock.replay(inventoryView);
    List<Interval> expectedIntervals = new ArrayList<>();
    expectedIntervals.add(new Interval("2010-01-22T00:00:00.000Z/2010-01-23T00:00:00.000Z"));
    expectedIntervals.add(new Interval("2010-01-01T00:00:00.000Z/2010-01-02T00:00:00.000Z"));
    DatasourcesResource datasourcesResource = new DatasourcesResource(inventoryView, null, null, new AuthConfig());
    Response response = datasourcesResource.getSegmentDataSourceIntervals("invalidDataSource", null, null);
    Assert.assertEquals(response.getEntity(), null);
    response = datasourcesResource.getSegmentDataSourceIntervals("datasource1", null, null);
    TreeSet<Interval> actualIntervals = (TreeSet) response.getEntity();
    Assert.assertEquals(2, actualIntervals.size());
    Assert.assertEquals(expectedIntervals.get(0), actualIntervals.first());
    Assert.assertEquals(expectedIntervals.get(1), actualIntervals.last());
    response = datasourcesResource.getSegmentDataSourceIntervals("datasource1", "simple", null);
    TreeMap<Interval, Map<String, Object>> results = (TreeMap) response.getEntity();
    Assert.assertEquals(2, results.size());
    Assert.assertEquals(expectedIntervals.get(0), results.firstKey());
    Assert.assertEquals(expectedIntervals.get(1), results.lastKey());
    Assert.assertEquals(1, results.firstEntry().getValue().get("count"));
    Assert.assertEquals(1, results.lastEntry().getValue().get("count"));
    response = datasourcesResource.getSegmentDataSourceIntervals("datasource1", null, "full");
    results = ((TreeMap<Interval, Map<String, Object>>) response.getEntity());
    int i = 1;
    for (Map.Entry<Interval, Map<String, Object>> entry : results.entrySet()) {
        Assert.assertEquals(dataSegmentList.get(i).getInterval(), entry.getKey());
        Assert.assertEquals(dataSegmentList.get(i), ((Map<String, Object>) entry.getValue().get(dataSegmentList.get(i).getIdentifier())).get("metadata"));
        i--;
    }
    EasyMock.verify(inventoryView);
}