Examples with DruidDataSource org.apache.druid.client.DruidDataSource used on opensource projects

Example 6 with DruidDataSource

use of org.apache.druid.client.DruidDataSource in project druid by druid-io.

the class DataSourcesResourceTest method testGetFullQueryableDataSources.

@Test
public void testGetFullQueryableDataSources() {
    // first request
    EasyMock.expect(server.getDataSources()).andReturn(ImmutableList.of(listDataSources.get(0), listDataSources.get(1))).once();
    EasyMock.expect(inventoryView.getInventory()).andReturn(ImmutableList.of(server)).once();
    EasyMock.expect(request.getAttribute(AuthConfig.DRUID_ALLOW_UNSECURED_PATH)).andReturn(null).once();
    EasyMock.expect(request.getAttribute(AuthConfig.DRUID_AUTHORIZATION_CHECKED)).andReturn(null).once();
    EasyMock.expect(request.getAttribute(AuthConfig.DRUID_AUTHENTICATION_RESULT)).andReturn(new AuthenticationResult("druid", "druid", null, null)).atLeastOnce();
    request.setAttribute(AuthConfig.DRUID_AUTHORIZATION_CHECKED, true);
    EasyMock.expectLastCall().times(1);
    // second request
    EasyMock.expect(server.getDataSources()).andReturn(ImmutableList.of(listDataSources.get(0), listDataSources.get(1))).once();
    EasyMock.expect(inventoryView.getInventory()).andReturn(ImmutableList.of(server)).once();
    EasyMock.expect(request.getAttribute(AuthConfig.DRUID_ALLOW_UNSECURED_PATH)).andReturn(null).once();
    EasyMock.expect(request.getAttribute(AuthConfig.DRUID_AUTHORIZATION_CHECKED)).andReturn(null).once();
    EasyMock.expect(request.getAttribute(AuthConfig.DRUID_AUTHENTICATION_RESULT)).andReturn(new AuthenticationResult("druid", "druid", null, null)).once();
    request.setAttribute(AuthConfig.DRUID_AUTHORIZATION_CHECKED, true);
    EasyMock.expectLastCall().times(1);
    EasyMock.replay(inventoryView, server, request);
    DataSourcesResource dataSourcesResource = new DataSourcesResource(inventoryView, null, null, null, AuthTestUtils.TEST_AUTHORIZER_MAPPER, null);
    Response response = dataSourcesResource.getQueryableDataSources("full", null, request);
    Set<ImmutableDruidDataSource> result = (Set<ImmutableDruidDataSource>) response.getEntity();
    Assert.assertEquals(200, response.getStatus());
    Assert.assertEquals(2, result.size());
    ImmutableDruidDataSourceTestUtils.assertEquals(listDataSources.stream().map(DruidDataSource::toImmutableDruidDataSource).collect(Collectors.toList()), new ArrayList<>(result));
    response = dataSourcesResource.getQueryableDataSources(null, null, request);
    List<String> result1 = (List<String>) response.getEntity();
    Assert.assertEquals(200, response.getStatus());
    Assert.assertEquals(2, result1.size());
    Assert.assertTrue(result1.contains("datasource1"));
    Assert.assertTrue(result1.contains("datasource2"));
    EasyMock.verify(inventoryView, server);
}

Example 7 with DruidDataSource

use of org.apache.druid.client.DruidDataSource in project druid by druid-io.

the class DataSourcesResourceTest method testMarkAsUnusedSegmentsInInterval.

@Test
public void testMarkAsUnusedSegmentsInInterval() {
    final Interval theInterval = Intervals.of("2010-01-01/P1D");
    final DruidDataSource dataSource1 = new DruidDataSource("datasource1", new HashMap<>());
    EasyMock.expect(inventoryView.getInventory()).andReturn(ImmutableList.of(server)).once();
    EasyMock.expect(server.getDataSource("datasource1")).andReturn(dataSource1).once();
    EasyMock.expect(segmentsMetadataManager.markAsUnusedSegmentsInInterval("datasource1", theInterval)).andReturn(1).once();
    EasyMock.replay(segmentsMetadataManager, inventoryView, server);
    final DataSourcesResource.MarkDataSourceSegmentsPayload payload = new DataSourcesResource.MarkDataSourceSegmentsPayload(theInterval, null);
    DataSourcesResource dataSourcesResource = new DataSourcesResource(inventoryView, segmentsMetadataManager, null, null, null, null);
    Response response = dataSourcesResource.markSegmentsAsUnused("datasource1", payload);
    Assert.assertEquals(200, response.getStatus());
    Assert.assertEquals(ImmutableMap.of("numChangedSegments", 1), response.getEntity());
    EasyMock.verify(segmentsMetadataManager, inventoryView, server);
    EasyMock.verify(segmentsMetadataManager, inventoryView, server);
}

Example 8 with DruidDataSource

use of org.apache.druid.client.DruidDataSource in project druid by druid-io.

the class DataSourcesResourceTest method testFullGetTheDataSource.

@Test
public void testFullGetTheDataSource() {
    DruidDataSource dataSource1 = new DruidDataSource("datasource1", new HashMap<>());
    EasyMock.expect(server.getDataSource("datasource1")).andReturn(dataSource1).atLeastOnce();
    EasyMock.expect(inventoryView.getInventory()).andReturn(ImmutableList.of(server)).atLeastOnce();
    EasyMock.replay(inventoryView, server);
    DataSourcesResource dataSourcesResource = new DataSourcesResource(inventoryView, null, null, null, null, null);
    Response response = dataSourcesResource.getDataSource("datasource1", "full");
    ImmutableDruidDataSource result = (ImmutableDruidDataSource) response.getEntity();
    Assert.assertEquals(200, response.getStatus());
    ImmutableDruidDataSourceTestUtils.assertEquals(dataSource1.toImmutableDruidDataSource(), result);
    EasyMock.verify(inventoryView, server);
}

Example 9 with DruidDataSource

use of org.apache.druid.client.DruidDataSource in project druid by druid-io.

the class DataSourcesResourceTest method testMarkSegmentsAsUnusedNoChanges.

@Test
public void testMarkSegmentsAsUnusedNoChanges() {
    final DruidDataSource dataSource1 = new DruidDataSource("datasource1", new HashMap<>());
    final Set<SegmentId> segmentIds = dataSegmentList.stream().filter(segment -> segment.getDataSource().equals(dataSource1.getName())).map(DataSegment::getId).collect(Collectors.toSet());
    EasyMock.expect(inventoryView.getInventory()).andReturn(ImmutableList.of(server)).once();
    EasyMock.expect(server.getDataSource("datasource1")).andReturn(dataSource1).once();
    EasyMock.expect(segmentsMetadataManager.markSegmentsAsUnused(segmentIds)).andReturn(0).once();
    EasyMock.replay(segmentsMetadataManager, inventoryView, server);
    final DataSourcesResource.MarkDataSourceSegmentsPayload payload = new DataSourcesResource.MarkDataSourceSegmentsPayload(null, segmentIds.stream().map(SegmentId::toString).collect(Collectors.toSet()));
    DataSourcesResource dataSourcesResource = new DataSourcesResource(inventoryView, segmentsMetadataManager, null, null, null, null);
    Response response = dataSourcesResource.markSegmentsAsUnused("datasource1", payload);
    Assert.assertEquals(200, response.getStatus());
    Assert.assertEquals(ImmutableMap.of("numChangedSegments", 0), response.getEntity());
    EasyMock.verify(segmentsMetadataManager, inventoryView, server);
}

Example 10 with DruidDataSource

use of org.apache.druid.client.DruidDataSource in project druid by druid-io.

the class DruidCoordinatorTest method testComputeUnderReplicationCountsPerDataSourcePerTierForSegmentsWithBroadcastRule.

@Test(timeout = 60_000L)
public void testComputeUnderReplicationCountsPerDataSourcePerTierForSegmentsWithBroadcastRule() throws Exception {
    final String dataSource = "dataSource";
    final String hotTierName = "hot";
    final String coldTierName = "cold";
    final String tierName1 = "tier1";
    final String tierName2 = "tier2";
    final Rule broadcastDistributionRule = new ForeverBroadcastDistributionRule();
    final String loadPathCold = "/druid/loadqueue/cold:1234";
    final String loadPathBroker1 = "/druid/loadqueue/broker1:1234";
    final String loadPathBroker2 = "/druid/loadqueue/broker2:1234";
    final String loadPathPeon = "/druid/loadqueue/peon:1234";
    final DruidServer hotServer = new DruidServer("hot", "hot", null, 5L, ServerType.HISTORICAL, hotTierName, 0);
    final DruidServer coldServer = new DruidServer("cold", "cold", null, 5L, ServerType.HISTORICAL, coldTierName, 0);
    final DruidServer brokerServer1 = new DruidServer("broker1", "broker1", null, 5L, ServerType.BROKER, tierName1, 0);
    final DruidServer brokerServer2 = new DruidServer("broker2", "broker2", null, 5L, ServerType.BROKER, tierName2, 0);
    final DruidServer peonServer = new DruidServer("peon", "peon", null, 5L, ServerType.INDEXER_EXECUTOR, tierName2, 0);
    final Map<String, DataSegment> dataSegments = ImmutableMap.of("2018-01-02T00:00:00.000Z_2018-01-03T00:00:00.000Z", new DataSegment(dataSource, Intervals.of("2018-01-02/P1D"), "v1", null, null, null, null, 0x9, 0), "2018-01-03T00:00:00.000Z_2018-01-04T00:00:00.000Z", new DataSegment(dataSource, Intervals.of("2018-01-03/P1D"), "v1", null, null, null, null, 0x9, 0), "2017-01-01T00:00:00.000Z_2017-01-02T00:00:00.000Z", new DataSegment(dataSource, Intervals.of("2017-01-01/P1D"), "v1", null, null, null, null, 0x9, 0));
    final LoadQueuePeon loadQueuePeonCold = new CuratorLoadQueuePeon(curator, loadPathCold, objectMapper, Execs.scheduledSingleThreaded("coordinator_test_load_queue_peon_cold_scheduled-%d"), Execs.singleThreaded("coordinator_test_load_queue_peon_cold-%d"), druidCoordinatorConfig);
    final LoadQueuePeon loadQueuePeonBroker1 = new CuratorLoadQueuePeon(curator, loadPathBroker1, objectMapper, Execs.scheduledSingleThreaded("coordinator_test_load_queue_peon_broker1_scheduled-%d"), Execs.singleThreaded("coordinator_test_load_queue_peon_broker1-%d"), druidCoordinatorConfig);
    final LoadQueuePeon loadQueuePeonBroker2 = new CuratorLoadQueuePeon(curator, loadPathBroker2, objectMapper, Execs.scheduledSingleThreaded("coordinator_test_load_queue_peon_broker2_scheduled-%d"), Execs.singleThreaded("coordinator_test_load_queue_peon_broker2-%d"), druidCoordinatorConfig);
    final LoadQueuePeon loadQueuePeonPoenServer = new CuratorLoadQueuePeon(curator, loadPathPeon, objectMapper, Execs.scheduledSingleThreaded("coordinator_test_load_queue_peon_peon_scheduled-%d"), Execs.singleThreaded("coordinator_test_load_queue_peon_peon-%d"), druidCoordinatorConfig);
    final PathChildrenCache pathChildrenCacheCold = new PathChildrenCache(curator, loadPathCold, true, true, Execs.singleThreaded("coordinator_test_path_children_cache_cold-%d"));
    final PathChildrenCache pathChildrenCacheBroker1 = new PathChildrenCache(curator, loadPathBroker1, true, true, Execs.singleThreaded("coordinator_test_path_children_cache_broker1-%d"));
    final PathChildrenCache pathChildrenCacheBroker2 = new PathChildrenCache(curator, loadPathBroker2, true, true, Execs.singleThreaded("coordinator_test_path_children_cache_broker2-%d"));
    final PathChildrenCache pathChildrenCachePeon = new PathChildrenCache(curator, loadPathPeon, true, true, Execs.singleThreaded("coordinator_test_path_children_cache_peon-%d"));
    loadManagementPeons.putAll(ImmutableMap.of("hot", loadQueuePeon, "cold", loadQueuePeonCold, "broker1", loadQueuePeonBroker1, "broker2", loadQueuePeonBroker2, "peon", loadQueuePeonPoenServer));
    loadQueuePeonCold.start();
    loadQueuePeonBroker1.start();
    loadQueuePeonBroker2.start();
    loadQueuePeonPoenServer.start();
    pathChildrenCache.start();
    pathChildrenCacheCold.start();
    pathChildrenCacheBroker1.start();
    pathChildrenCacheBroker2.start();
    pathChildrenCachePeon.start();
    DruidDataSource[] druidDataSources = { new DruidDataSource(dataSource, Collections.emptyMap()) };
    dataSegments.values().forEach(druidDataSources[0]::addSegment);
    setupSegmentsMetadataMock(druidDataSources[0]);
    EasyMock.expect(metadataRuleManager.getRulesWithDefault(EasyMock.anyString())).andReturn(ImmutableList.of(broadcastDistributionRule)).atLeastOnce();
    EasyMock.expect(metadataRuleManager.getAllRules()).andReturn(ImmutableMap.of(dataSource, ImmutableList.of(broadcastDistributionRule))).atLeastOnce();
    EasyMock.expect(serverInventoryView.getInventory()).andReturn(ImmutableList.of(hotServer, coldServer, brokerServer1, brokerServer2, peonServer)).atLeastOnce();
    EasyMock.expect(serverInventoryView.isStarted()).andReturn(true).anyTimes();
    EasyMock.replay(metadataRuleManager, serverInventoryView);
    coordinator.start();
    // Wait for this coordinator to become leader
    leaderAnnouncerLatch.await();
    final CountDownLatch assignSegmentLatchHot = createCountDownLatchAndSetPathChildrenCacheListenerWithLatch(3, pathChildrenCache, dataSegments, hotServer);
    final CountDownLatch assignSegmentLatchCold = createCountDownLatchAndSetPathChildrenCacheListenerWithLatch(3, pathChildrenCacheCold, dataSegments, coldServer);
    final CountDownLatch assignSegmentLatchBroker1 = createCountDownLatchAndSetPathChildrenCacheListenerWithLatch(3, pathChildrenCacheBroker1, dataSegments, brokerServer1);
    final CountDownLatch assignSegmentLatchBroker2 = createCountDownLatchAndSetPathChildrenCacheListenerWithLatch(3, pathChildrenCacheBroker2, dataSegments, brokerServer2);
    final CountDownLatch assignSegmentLatchPeon = createCountDownLatchAndSetPathChildrenCacheListenerWithLatch(3, pathChildrenCachePeon, dataSegments, peonServer);
    assignSegmentLatchHot.await();
    assignSegmentLatchCold.await();
    assignSegmentLatchBroker1.await();
    assignSegmentLatchBroker2.await();
    assignSegmentLatchPeon.await();
    final CountDownLatch coordinatorRunLatch = new CountDownLatch(2);
    serviceEmitter.latch = coordinatorRunLatch;
    coordinatorRunLatch.await();
    Assert.assertEquals(ImmutableMap.of(dataSource, 100.0), coordinator.getLoadStatus());
    Map<String, Object2LongMap<String>> underReplicationCountsPerDataSourcePerTier = coordinator.computeUnderReplicationCountsPerDataSourcePerTier();
    Assert.assertEquals(4, underReplicationCountsPerDataSourcePerTier.size());
    Assert.assertEquals(0L, underReplicationCountsPerDataSourcePerTier.get(hotTierName).getLong(dataSource));
    Assert.assertEquals(0L, underReplicationCountsPerDataSourcePerTier.get(coldTierName).getLong(dataSource));
    Assert.assertEquals(0L, underReplicationCountsPerDataSourcePerTier.get(tierName1).getLong(dataSource));
    Assert.assertEquals(0L, underReplicationCountsPerDataSourcePerTier.get(tierName2).getLong(dataSource));
    Map<String, Object2LongMap<String>> underReplicationCountsPerDataSourcePerTierUsingClusterView = coordinator.computeUnderReplicationCountsPerDataSourcePerTierUsingClusterView();
    Assert.assertEquals(4, underReplicationCountsPerDataSourcePerTierUsingClusterView.size());
    Assert.assertEquals(0L, underReplicationCountsPerDataSourcePerTierUsingClusterView.get(hotTierName).getLong(dataSource));
    Assert.assertEquals(0L, underReplicationCountsPerDataSourcePerTierUsingClusterView.get(coldTierName).getLong(dataSource));
    Assert.assertEquals(0L, underReplicationCountsPerDataSourcePerTierUsingClusterView.get(tierName1).getLong(dataSource));
    Assert.assertEquals(0L, underReplicationCountsPerDataSourcePerTierUsingClusterView.get(tierName2).getLong(dataSource));
    coordinator.stop();
    leaderUnannouncerLatch.await();
    EasyMock.verify(serverInventoryView);
    EasyMock.verify(segmentsMetadataManager);
    EasyMock.verify(metadataRuleManager);
}