Search in sources :

Example 6 with DataSourcesSnapshot

use of org.apache.druid.client.DataSourcesSnapshot in project druid by apache.

the class SqlSegmentsMetadataManagerTest method testPollPeriodicallyAndOnDemandInterleave.

@Test(timeout = 60_000)
public void testPollPeriodicallyAndOnDemandInterleave() throws Exception {
    DataSourcesSnapshot dataSourcesSnapshot = sqlSegmentsMetadataManager.getDataSourcesSnapshot();
    Assert.assertNull(dataSourcesSnapshot);
    sqlSegmentsMetadataManager.startPollingDatabasePeriodically();
    Assert.assertTrue(sqlSegmentsMetadataManager.isPollingDatabasePeriodically());
    // This call make sure that the first poll is completed
    sqlSegmentsMetadataManager.useLatestSnapshotIfWithinDelay();
    Assert.assertTrue(sqlSegmentsMetadataManager.getLatestDatabasePoll() instanceof SqlSegmentsMetadataManager.PeriodicDatabasePoll);
    dataSourcesSnapshot = sqlSegmentsMetadataManager.getDataSourcesSnapshot();
    Assert.assertEquals(ImmutableList.of("wikipedia"), dataSourcesSnapshot.getDataSourcesWithAllUsedSegments().stream().map(ImmutableDruidDataSource::getName).collect(Collectors.toList()));
    final String newDataSource2 = "wikipedia2";
    final DataSegment newSegment2 = createNewSegment1(newDataSource2);
    publisher.publishSegment(newSegment2);
    // This call will force on demand poll
    sqlSegmentsMetadataManager.forceOrWaitOngoingDatabasePoll();
    Assert.assertTrue(sqlSegmentsMetadataManager.isPollingDatabasePeriodically());
    Assert.assertTrue(sqlSegmentsMetadataManager.getLatestDatabasePoll() instanceof SqlSegmentsMetadataManager.OnDemandDatabasePoll);
    // New datasource should now be in the snapshot since we just force on demand poll.
    dataSourcesSnapshot = sqlSegmentsMetadataManager.getDataSourcesSnapshot();
    Assert.assertEquals(ImmutableList.of("wikipedia2", "wikipedia"), dataSourcesSnapshot.getDataSourcesWithAllUsedSegments().stream().map(ImmutableDruidDataSource::getName).collect(Collectors.toList()));
    final String newDataSource3 = "wikipedia3";
    final DataSegment newSegment3 = createNewSegment1(newDataSource3);
    publisher.publishSegment(newSegment3);
    // This time wait for periodic poll (not doing on demand poll so we have to wait a bit...)
    while (sqlSegmentsMetadataManager.getDataSourcesSnapshot().getDataSource(newDataSource3) == null) {
        Thread.sleep(1000);
    }
    Assert.assertTrue(sqlSegmentsMetadataManager.isPollingDatabasePeriodically());
    Assert.assertTrue(sqlSegmentsMetadataManager.getLatestDatabasePoll() instanceof SqlSegmentsMetadataManager.PeriodicDatabasePoll);
    dataSourcesSnapshot = sqlSegmentsMetadataManager.getDataSourcesSnapshot();
    Assert.assertEquals(ImmutableList.of("wikipedia2", "wikipedia3", "wikipedia"), dataSourcesSnapshot.getDataSourcesWithAllUsedSegments().stream().map(ImmutableDruidDataSource::getName).collect(Collectors.toList()));
}
Also used : ImmutableDruidDataSource(org.apache.druid.client.ImmutableDruidDataSource) DataSourcesSnapshot(org.apache.druid.client.DataSourcesSnapshot) DataSegment(org.apache.druid.timeline.DataSegment) Test(org.junit.Test)

Example 7 with DataSourcesSnapshot

use of org.apache.druid.client.DataSourcesSnapshot in project druid by apache.

the class CuratorDruidCoordinatorTest method setUp.

@Before
public void setUp() throws Exception {
    segmentsMetadataManager = EasyMock.createNiceMock(SegmentsMetadataManager.class);
    dataSourcesSnapshot = EasyMock.createNiceMock(DataSourcesSnapshot.class);
    coordinatorRuntimeParams = EasyMock.createNiceMock(DruidCoordinatorRuntimeParams.class);
    metadataRuleManager = EasyMock.createNiceMock(MetadataRuleManager.class);
    configManager = EasyMock.createNiceMock(JacksonConfigManager.class);
    EasyMock.expect(configManager.watch(EasyMock.eq(CoordinatorDynamicConfig.CONFIG_KEY), EasyMock.anyObject(Class.class), EasyMock.anyObject())).andReturn(new AtomicReference<>(CoordinatorDynamicConfig.builder().build())).anyTimes();
    EasyMock.expect(configManager.watch(EasyMock.eq(CoordinatorCompactionConfig.CONFIG_KEY), EasyMock.anyObject(Class.class), EasyMock.anyObject())).andReturn(new AtomicReference(CoordinatorCompactionConfig.empty())).anyTimes();
    EasyMock.replay(configManager);
    setupServerAndCurator();
    curator.start();
    curator.blockUntilConnected();
    curator.create().creatingParentsIfNeeded().forPath(SEGPATH);
    curator.create().creatingParentsIfNeeded().forPath(SOURCE_LOAD_PATH);
    curator.create().creatingParentsIfNeeded().forPath(DESTINATION_LOAD_PATH);
    objectMapper = new DefaultObjectMapper();
    druidCoordinatorConfig = new TestDruidCoordinatorConfig(new Duration(COORDINATOR_START_DELAY), new Duration(COORDINATOR_PERIOD), null, null, null, new Duration(COORDINATOR_PERIOD), null, null, null, null, null, null, null, null, null, null, 10, new Duration("PT0s"));
    sourceLoadQueueChildrenCache = new PathChildrenCache(curator, SOURCE_LOAD_PATH, true, true, Execs.singleThreaded("coordinator_test_path_children_cache_src-%d"));
    destinationLoadQueueChildrenCache = new PathChildrenCache(curator, DESTINATION_LOAD_PATH, true, true, Execs.singleThreaded("coordinator_test_path_children_cache_dest-%d"));
    sourceLoadQueuePeon = new CuratorLoadQueuePeon(curator, SOURCE_LOAD_PATH, objectMapper, peonExec, callbackExec, druidCoordinatorConfig);
    destinationLoadQueuePeon = new CuratorLoadQueuePeon(curator, DESTINATION_LOAD_PATH, objectMapper, peonExec, callbackExec, druidCoordinatorConfig);
    druidNode = new DruidNode("hey", "what", false, 1234, null, true, false);
    loadManagementPeons = new ConcurrentHashMap<>();
    scheduledExecutorFactory = (corePoolSize, nameFormat) -> Executors.newSingleThreadScheduledExecutor();
    leaderAnnouncerLatch = new CountDownLatch(1);
    leaderUnannouncerLatch = new CountDownLatch(1);
    coordinator = new DruidCoordinator(druidCoordinatorConfig, new ZkPathsConfig() {

        @Override
        public String getBase() {
            return "druid";
        }
    }, configManager, segmentsMetadataManager, baseView, metadataRuleManager, () -> curator, new NoopServiceEmitter(), scheduledExecutorFactory, null, null, new NoopServiceAnnouncer() {

        @Override
        public void announce(DruidNode node) {
            // count down when this coordinator becomes the leader
            leaderAnnouncerLatch.countDown();
        }

        @Override
        public void unannounce(DruidNode node) {
            leaderUnannouncerLatch.countDown();
        }
    }, druidNode, loadManagementPeons, null, null, new CoordinatorCustomDutyGroups(ImmutableSet.of()), new CostBalancerStrategyFactory(), EasyMock.createNiceMock(LookupCoordinatorManager.class), new TestDruidLeaderSelector(), null, ZkEnablementConfig.ENABLED);
}
Also used : SegmentsMetadataManager(org.apache.druid.metadata.SegmentsMetadataManager) MetadataRuleManager(org.apache.druid.metadata.MetadataRuleManager) JacksonConfigManager(org.apache.druid.common.config.JacksonConfigManager) AtomicReference(java.util.concurrent.atomic.AtomicReference) Duration(org.joda.time.Duration) NoopServiceEmitter(org.apache.druid.server.metrics.NoopServiceEmitter) CountDownLatch(java.util.concurrent.CountDownLatch) PathChildrenCache(org.apache.curator.framework.recipes.cache.PathChildrenCache) ZkPathsConfig(org.apache.druid.server.initialization.ZkPathsConfig) DefaultObjectMapper(org.apache.druid.jackson.DefaultObjectMapper) DruidNode(org.apache.druid.server.DruidNode) DataSourcesSnapshot(org.apache.druid.client.DataSourcesSnapshot) NoopServiceAnnouncer(org.apache.druid.curator.discovery.NoopServiceAnnouncer) CoordinatorCustomDutyGroups(org.apache.druid.server.coordinator.duty.CoordinatorCustomDutyGroups) Before(org.junit.Before)

Example 8 with DataSourcesSnapshot

use of org.apache.druid.client.DataSourcesSnapshot in project druid by druid-io.

the class DataSourcesSnapshotBenchmark method setUp.

@Setup
public void setUp() {
    long start = System.currentTimeMillis();
    Map<String, ImmutableDruidDataSource> dataSources = new HashMap<>();
    for (int i = 0; i < numDataSources; i++) {
        String dataSource = StringUtils.format("ds-%d", i);
        List<DataSegment> segments = new ArrayList<>();
        for (int j = 0; j < numSegmentPerDataSource; j++) {
            segments.add(new DataSegment(dataSource, TEST_SEGMENT_INTERVAL, String.valueOf(j), Collections.emptyMap(), Collections.emptyList(), Collections.emptyList(), NoneShardSpec.instance(), 0, 10L));
        }
        dataSources.put(dataSource, new ImmutableDruidDataSource(dataSource, Collections.emptyMap(), segments));
    }
    snapshot = new DataSourcesSnapshot(dataSources);
    System.out.println("Setup Time " + (System.currentTimeMillis() - start) + " ms");
}
Also used : ImmutableDruidDataSource(org.apache.druid.client.ImmutableDruidDataSource) HashMap(java.util.HashMap) ArrayList(java.util.ArrayList) DataSegment(org.apache.druid.timeline.DataSegment) DataSourcesSnapshot(org.apache.druid.client.DataSourcesSnapshot) Setup(org.openjdk.jmh.annotations.Setup)

Example 9 with DataSourcesSnapshot

use of org.apache.druid.client.DataSourcesSnapshot in project druid by druid-io.

the class MetadataResource method getAllUsedSegmentsWithOvershadowedStatus.

private Response getAllUsedSegmentsWithOvershadowedStatus(HttpServletRequest req, @Nullable Set<String> dataSources) {
    DataSourcesSnapshot dataSourcesSnapshot = segmentsMetadataManager.getSnapshotOfDataSourcesWithAllUsedSegments();
    Collection<ImmutableDruidDataSource> dataSourcesWithUsedSegments = dataSourcesSnapshot.getDataSourcesWithAllUsedSegments();
    if (dataSources != null && !dataSources.isEmpty()) {
        dataSourcesWithUsedSegments = dataSourcesWithUsedSegments.stream().filter(dataSourceWithUsedSegments -> dataSources.contains(dataSourceWithUsedSegments.getName())).collect(Collectors.toList());
    }
    final Stream<DataSegment> usedSegments = dataSourcesWithUsedSegments.stream().flatMap(t -> t.getSegments().stream());
    final Set<SegmentId> overshadowedSegments = dataSourcesSnapshot.getOvershadowedSegments();
    final Stream<SegmentWithOvershadowedStatus> usedSegmentsWithOvershadowedStatus = usedSegments.map(segment -> new SegmentWithOvershadowedStatus(segment, overshadowedSegments.contains(segment.getId())));
    final Function<SegmentWithOvershadowedStatus, Iterable<ResourceAction>> raGenerator = segment -> Collections.singletonList(AuthorizationUtils.DATASOURCE_READ_RA_GENERATOR.apply(segment.getDataSegment().getDataSource()));
    final Iterable<SegmentWithOvershadowedStatus> authorizedSegments = AuthorizationUtils.filterAuthorizedResources(req, usedSegmentsWithOvershadowedStatus::iterator, raGenerator, authorizerMapper);
    Response.ResponseBuilder builder = Response.status(Response.Status.OK);
    return builder.entity(authorizedSegments).build();
}
Also used : Iterables(com.google.common.collect.Iterables) PathParam(javax.ws.rs.PathParam) Produces(javax.ws.rs.Produces) GET(javax.ws.rs.GET) AuthorizerMapper(org.apache.druid.server.security.AuthorizerMapper) Inject(com.google.inject.Inject) SegmentWithOvershadowedStatus(org.apache.druid.timeline.SegmentWithOvershadowedStatus) Path(javax.ws.rs.Path) Collections2(com.google.common.collect.Collections2) ResourceFilters(com.sun.jersey.spi.container.ResourceFilters) DataSourcesSnapshot(org.apache.druid.client.DataSourcesSnapshot) TreeSet(java.util.TreeSet) Interval(org.joda.time.Interval) HttpServletRequest(javax.servlet.http.HttpServletRequest) MediaType(javax.ws.rs.core.MediaType) QueryParam(javax.ws.rs.QueryParam) ImmutableDruidDataSource(org.apache.druid.client.ImmutableDruidDataSource) Nullable(javax.annotation.Nullable) JettyUtils(org.apache.druid.server.JettyUtils) Function(com.google.common.base.Function) POST(javax.ws.rs.POST) Context(javax.ws.rs.core.Context) SegmentsMetadataManager(org.apache.druid.metadata.SegmentsMetadataManager) Collection(java.util.Collection) ObjectMapper(com.fasterxml.jackson.databind.ObjectMapper) Segments(org.apache.druid.indexing.overlord.Segments) Set(java.util.Set) Json(org.apache.druid.guice.annotations.Json) AuthorizationUtils(org.apache.druid.server.security.AuthorizationUtils) Collectors(java.util.stream.Collectors) DatasourceResourceFilter(org.apache.druid.server.http.security.DatasourceResourceFilter) List(java.util.List) Stream(java.util.stream.Stream) Response(javax.ws.rs.core.Response) ResourceAction(org.apache.druid.server.security.ResourceAction) IndexerMetadataStorageCoordinator(org.apache.druid.indexing.overlord.IndexerMetadataStorageCoordinator) DataSegment(org.apache.druid.timeline.DataSegment) UriInfo(javax.ws.rs.core.UriInfo) SegmentId(org.apache.druid.timeline.SegmentId) Collections(java.util.Collections) ImmutableDruidDataSource(org.apache.druid.client.ImmutableDruidDataSource) SegmentId(org.apache.druid.timeline.SegmentId) DataSegment(org.apache.druid.timeline.DataSegment) Response(javax.ws.rs.core.Response) SegmentWithOvershadowedStatus(org.apache.druid.timeline.SegmentWithOvershadowedStatus) DataSourcesSnapshot(org.apache.druid.client.DataSourcesSnapshot)

Example 10 with DataSourcesSnapshot

use of org.apache.druid.client.DataSourcesSnapshot in project druid by druid-io.

the class SqlSegmentsMetadataManagerTest method testPollOnDemand.

@Test
public void testPollOnDemand() {
    DataSourcesSnapshot dataSourcesSnapshot = sqlSegmentsMetadataManager.getDataSourcesSnapshot();
    Assert.assertNull(dataSourcesSnapshot);
    // This should return false and not wait/poll anything as we did not schedule periodic poll
    Assert.assertFalse(sqlSegmentsMetadataManager.useLatestSnapshotIfWithinDelay());
    Assert.assertNull(dataSourcesSnapshot);
    // This call will force on demand poll
    sqlSegmentsMetadataManager.forceOrWaitOngoingDatabasePoll();
    Assert.assertFalse(sqlSegmentsMetadataManager.isPollingDatabasePeriodically());
    Assert.assertTrue(sqlSegmentsMetadataManager.getLatestDatabasePoll() instanceof SqlSegmentsMetadataManager.OnDemandDatabasePoll);
    dataSourcesSnapshot = sqlSegmentsMetadataManager.getDataSourcesSnapshot();
    Assert.assertEquals(ImmutableSet.of("wikipedia"), sqlSegmentsMetadataManager.retrieveAllDataSourceNames());
    Assert.assertEquals(ImmutableList.of("wikipedia"), dataSourcesSnapshot.getDataSourcesWithAllUsedSegments().stream().map(ImmutableDruidDataSource::getName).collect(Collectors.toList()));
    Assert.assertEquals(ImmutableSet.of(segment1, segment2), ImmutableSet.copyOf(dataSourcesSnapshot.getDataSource("wikipedia").getSegments()));
    Assert.assertEquals(ImmutableSet.of(segment1, segment2), ImmutableSet.copyOf(dataSourcesSnapshot.iterateAllUsedSegmentsInSnapshot()));
}
Also used : ImmutableDruidDataSource(org.apache.druid.client.ImmutableDruidDataSource) DataSourcesSnapshot(org.apache.druid.client.DataSourcesSnapshot) Test(org.junit.Test)

Aggregations

DataSourcesSnapshot (org.apache.druid.client.DataSourcesSnapshot)22 ImmutableDruidDataSource (org.apache.druid.client.ImmutableDruidDataSource)14 DataSegment (org.apache.druid.timeline.DataSegment)12 Test (org.junit.Test)8 CountDownLatch (java.util.concurrent.CountDownLatch)6 AtomicReference (java.util.concurrent.atomic.AtomicReference)6 JacksonConfigManager (org.apache.druid.common.config.JacksonConfigManager)6 NoopServiceAnnouncer (org.apache.druid.curator.discovery.NoopServiceAnnouncer)6 SegmentsMetadataManager (org.apache.druid.metadata.SegmentsMetadataManager)6 DruidNode (org.apache.druid.server.DruidNode)6 CoordinatorCustomDutyGroups (org.apache.druid.server.coordinator.duty.CoordinatorCustomDutyGroups)6 ZkPathsConfig (org.apache.druid.server.initialization.ZkPathsConfig)6 HashSet (java.util.HashSet)4 Nullable (javax.annotation.Nullable)4 PathChildrenCache (org.apache.curator.framework.recipes.cache.PathChildrenCache)4 DefaultObjectMapper (org.apache.druid.jackson.DefaultObjectMapper)4 MetadataRuleManager (org.apache.druid.metadata.MetadataRuleManager)4 Duration (org.joda.time.Duration)4 Before (org.junit.Before)4 ObjectMapper (com.fasterxml.jackson.databind.ObjectMapper)2