Examples with TimelineLookup org.apache.druid.timeline.TimelineLookup used on opensource projects

Example 1 with TimelineLookup

use of org.apache.druid.timeline.TimelineLookup in project druid by druid-io.

the class CoordinatorServerViewTest method testMultipleServerAddedRemovedSegment.

@Test
public void testMultipleServerAddedRemovedSegment() throws Exception {
    segmentViewInitLatch = new CountDownLatch(1);
    segmentAddedLatch = new CountDownLatch(5);
    // temporarily set latch count to 1
    segmentRemovedLatch = new CountDownLatch(1);
    setupViews();
    final List<DruidServer> druidServers = Lists.transform(ImmutableList.of("localhost:0", "localhost:1", "localhost:2", "localhost:3", "localhost:4"), new Function<String, DruidServer>() {

        @Override
        public DruidServer apply(String input) {
            return new DruidServer(input, input, null, 10000000L, ServerType.HISTORICAL, "default_tier", 0);
        }
    });
    for (DruidServer druidServer : druidServers) {
        setupZNodeForServer(druidServer, zkPathsConfig, jsonMapper);
    }
    final List<DataSegment> segments = Lists.transform(ImmutableList.of(Pair.of("2011-04-01/2011-04-03", "v1"), Pair.of("2011-04-03/2011-04-06", "v1"), Pair.of("2011-04-01/2011-04-09", "v2"), Pair.of("2011-04-06/2011-04-09", "v3"), Pair.of("2011-04-01/2011-04-02", "v3")), new Function<Pair<String, String>, DataSegment>() {

        @Override
        public DataSegment apply(Pair<String, String> input) {
            return dataSegmentWithIntervalAndVersion(input.lhs, input.rhs);
        }
    });
    for (int i = 0; i < 5; ++i) {
        announceSegmentForServer(druidServers.get(i), segments.get(i), zkPathsConfig, jsonMapper);
    }
    Assert.assertTrue(timing.forWaiting().awaitLatch(segmentViewInitLatch));
    Assert.assertTrue(timing.forWaiting().awaitLatch(segmentAddedLatch));
    TimelineLookup timeline = overlordServerView.getTimeline(new TableDataSource("test_overlord_server_view"));
    assertValues(Arrays.asList(createExpected("2011-04-01/2011-04-02", "v3", druidServers.get(4), segments.get(4)), createExpected("2011-04-02/2011-04-06", "v2", druidServers.get(2), segments.get(2)), createExpected("2011-04-06/2011-04-09", "v3", druidServers.get(3), segments.get(3))), (List<TimelineObjectHolder>) timeline.lookup(Intervals.of("2011-04-01/2011-04-09")));
    // unannounce the segment created by dataSegmentWithIntervalAndVersion("2011-04-01/2011-04-09", "v2")
    unannounceSegmentForServer(druidServers.get(2), segments.get(2));
    Assert.assertTrue(timing.forWaiting().awaitLatch(segmentRemovedLatch));
    // renew segmentRemovedLatch since we still have 4 segments to unannounce
    segmentRemovedLatch = new CountDownLatch(4);
    timeline = overlordServerView.getTimeline(new TableDataSource("test_overlord_server_view"));
    assertValues(Arrays.asList(createExpected("2011-04-01/2011-04-02", "v3", druidServers.get(4), segments.get(4)), createExpected("2011-04-02/2011-04-03", "v1", druidServers.get(0), segments.get(0)), createExpected("2011-04-03/2011-04-06", "v1", druidServers.get(1), segments.get(1)), createExpected("2011-04-06/2011-04-09", "v3", druidServers.get(3), segments.get(3))), (List<TimelineObjectHolder>) timeline.lookup(Intervals.of("2011-04-01/2011-04-09")));
    // unannounce all the segments
    for (int i = 0; i < 5; ++i) {
        // skip the one that was previously unannounced
        if (i != 2) {
            unannounceSegmentForServer(druidServers.get(i), segments.get(i));
        }
    }
    Assert.assertTrue(timing.forWaiting().awaitLatch(segmentRemovedLatch));
    Assert.assertEquals(0, ((List<TimelineObjectHolder>) timeline.lookup(Intervals.of("2011-04-01/2011-04-09"))).size());
}

Example 2 with TimelineLookup

use of org.apache.druid.timeline.TimelineLookup in project druid by druid-io.

the class BrokerServerViewTest method testMultipleServerAddedRemovedSegment.

@Test
public void testMultipleServerAddedRemovedSegment() throws Exception {
    segmentViewInitLatch = new CountDownLatch(1);
    segmentAddedLatch = new CountDownLatch(5);
    // temporarily set latch count to 1
    segmentRemovedLatch = new CountDownLatch(1);
    setupViews();
    final List<DruidServer> druidServers = Lists.transform(ImmutableList.of("locahost:0", "localhost:1", "localhost:2", "localhost:3", "localhost:4"), hostname -> setupHistoricalServer("default_tier", hostname, 0));
    final List<DataSegment> segments = Lists.transform(ImmutableList.of(Pair.of("2011-04-01/2011-04-03", "v1"), Pair.of("2011-04-03/2011-04-06", "v1"), Pair.of("2011-04-01/2011-04-09", "v2"), Pair.of("2011-04-06/2011-04-09", "v3"), Pair.of("2011-04-01/2011-04-02", "v3")), input -> dataSegmentWithIntervalAndVersion(input.lhs, input.rhs));
    for (int i = 0; i < 5; ++i) {
        announceSegmentForServer(druidServers.get(i), segments.get(i), zkPathsConfig, jsonMapper);
    }
    Assert.assertTrue(timing.forWaiting().awaitLatch(segmentViewInitLatch));
    Assert.assertTrue(timing.forWaiting().awaitLatch(segmentAddedLatch));
    TimelineLookup timeline = brokerServerView.getTimeline(DataSourceAnalysis.forDataSource(new TableDataSource("test_broker_server_view"))).get();
    assertValues(Arrays.asList(createExpected("2011-04-01/2011-04-02", "v3", druidServers.get(4), segments.get(4)), createExpected("2011-04-02/2011-04-06", "v2", druidServers.get(2), segments.get(2)), createExpected("2011-04-06/2011-04-09", "v3", druidServers.get(3), segments.get(3))), (List<TimelineObjectHolder>) timeline.lookup(Intervals.of("2011-04-01/2011-04-09")));
    // unannounce the segment created by dataSegmentWithIntervalAndVersion("2011-04-01/2011-04-09", "v2")
    unannounceSegmentForServer(druidServers.get(2), segments.get(2), zkPathsConfig);
    Assert.assertTrue(timing.forWaiting().awaitLatch(segmentRemovedLatch));
    // renew segmentRemovedLatch since we still have 4 segments to unannounce
    segmentRemovedLatch = new CountDownLatch(4);
    timeline = brokerServerView.getTimeline(DataSourceAnalysis.forDataSource(new TableDataSource("test_broker_server_view"))).get();
    assertValues(Arrays.asList(createExpected("2011-04-01/2011-04-02", "v3", druidServers.get(4), segments.get(4)), createExpected("2011-04-02/2011-04-03", "v1", druidServers.get(0), segments.get(0)), createExpected("2011-04-03/2011-04-06", "v1", druidServers.get(1), segments.get(1)), createExpected("2011-04-06/2011-04-09", "v3", druidServers.get(3), segments.get(3))), (List<TimelineObjectHolder>) timeline.lookup(Intervals.of("2011-04-01/2011-04-09")));
    // unannounce all the segments
    for (int i = 0; i < 5; ++i) {
        // skip the one that was previously unannounced
        if (i != 2) {
            unannounceSegmentForServer(druidServers.get(i), segments.get(i), zkPathsConfig);
        }
    }
    Assert.assertTrue(timing.forWaiting().awaitLatch(segmentRemovedLatch));
    Assert.assertEquals(0, ((List<TimelineObjectHolder>) timeline.lookup(Intervals.of("2011-04-01/2011-04-09"))).size());
}

Example 3 with TimelineLookup

use of org.apache.druid.timeline.TimelineLookup in project druid by druid-io.

the class BrokerServerViewTest method testMultipleServerAndBroker.

@Test
public void testMultipleServerAndBroker() throws Exception {
    segmentViewInitLatch = new CountDownLatch(1);
    segmentAddedLatch = new CountDownLatch(6);
    // temporarily set latch count to 1
    segmentRemovedLatch = new CountDownLatch(1);
    setupViews();
    final DruidServer druidBroker = new DruidServer("localhost:5", "localhost:5", null, 10000000L, ServerType.BROKER, "default_tier", 0);
    final List<DruidServer> druidServers = Lists.transform(ImmutableList.of("locahost:0", "localhost:1", "localhost:2", "localhost:3", "localhost:4"), hostname -> setupHistoricalServer("default_tier", hostname, 0));
    setupZNodeForServer(druidBroker, zkPathsConfig, jsonMapper);
    final List<DataSegment> segments = Lists.transform(ImmutableList.of(Pair.of("2011-04-01/2011-04-03", "v1"), Pair.of("2011-04-03/2011-04-06", "v1"), Pair.of("2011-04-01/2011-04-09", "v2"), Pair.of("2011-04-06/2011-04-09", "v3"), Pair.of("2011-04-01/2011-04-02", "v3")), input -> dataSegmentWithIntervalAndVersion(input.lhs, input.rhs));
    DataSegment brokerSegment = dataSegmentWithIntervalAndVersion("2011-04-01/2011-04-11", "v4");
    announceSegmentForServer(druidBroker, brokerSegment, zkPathsConfig, jsonMapper);
    for (int i = 0; i < 5; ++i) {
        announceSegmentForServer(druidServers.get(i), segments.get(i), zkPathsConfig, jsonMapper);
    }
    Assert.assertTrue(timing.forWaiting().awaitLatch(segmentViewInitLatch));
    Assert.assertTrue(timing.forWaiting().awaitLatch(segmentAddedLatch));
    TimelineLookup timeline = brokerServerView.getTimeline(DataSourceAnalysis.forDataSource(new TableDataSource("test_broker_server_view"))).get();
    assertValues(Arrays.asList(createExpected("2011-04-01/2011-04-02", "v3", druidServers.get(4), segments.get(4)), createExpected("2011-04-02/2011-04-06", "v2", druidServers.get(2), segments.get(2)), createExpected("2011-04-06/2011-04-09", "v3", druidServers.get(3), segments.get(3))), (List<TimelineObjectHolder>) timeline.lookup(Intervals.of("2011-04-01/2011-04-09")));
    // unannounce the broker segment should do nothing to announcements
    unannounceSegmentForServer(druidBroker, brokerSegment, zkPathsConfig);
    Assert.assertTrue(timing.forWaiting().awaitLatch(segmentRemovedLatch));
    // renew segmentRemovedLatch since we still have 5 segments to unannounce
    segmentRemovedLatch = new CountDownLatch(5);
    timeline = brokerServerView.getTimeline(DataSourceAnalysis.forDataSource(new TableDataSource("test_broker_server_view"))).get();
    // expect same set of segments as before
    assertValues(Arrays.asList(createExpected("2011-04-01/2011-04-02", "v3", druidServers.get(4), segments.get(4)), createExpected("2011-04-02/2011-04-06", "v2", druidServers.get(2), segments.get(2)), createExpected("2011-04-06/2011-04-09", "v3", druidServers.get(3), segments.get(3))), (List<TimelineObjectHolder>) timeline.lookup(Intervals.of("2011-04-01/2011-04-09")));
    // unannounce all the segments
    for (int i = 0; i < 5; ++i) {
        unannounceSegmentForServer(druidServers.get(i), segments.get(i), zkPathsConfig);
    }
    Assert.assertTrue(timing.forWaiting().awaitLatch(segmentRemovedLatch));
}

Example 4 with TimelineLookup

use of org.apache.druid.timeline.TimelineLookup in project druid by druid-io.

the class MovingAverageQueryTest method testQuery.

/**
 * Validate that the specified query behaves correctly.
 */
@SuppressWarnings({ "unchecked", "rawtypes" })
@Test
public void testQuery() throws IOException {
    Query<?> query = jsonMapper.readValue(getQueryString(), Query.class);
    Assert.assertThat(query, IsInstanceOf.instanceOf(getExpectedQueryType()));
    List<MapBasedRow> expectedResults = jsonMapper.readValue(getExpectedResultString(), getExpectedResultType());
    Assert.assertNotNull(expectedResults);
    Assert.assertThat(expectedResults, IsInstanceOf.instanceOf(List.class));
    CachingClusteredClient baseClient = new CachingClusteredClient(warehouse, new TimelineServerView() {

        @Override
        public Optional<? extends TimelineLookup<String, ServerSelector>> getTimeline(DataSourceAnalysis analysis) {
            return Optional.empty();
        }

        @Override
        public List<ImmutableDruidServer> getDruidServers() {
            return null;
        }

        @Override
        public <T> QueryRunner<T> getQueryRunner(DruidServer server) {
            return null;
        }

        @Override
        public void registerTimelineCallback(Executor exec, TimelineCallback callback) {
        }

        @Override
        public void registerSegmentCallback(Executor exec, SegmentCallback callback) {
        }

        @Override
        public void registerServerRemovedCallback(Executor exec, ServerRemovedCallback callback) {
        }
    }, MapCache.create(100000), jsonMapper, new ForegroundCachePopulator(jsonMapper, new CachePopulatorStats(), -1), new CacheConfig(), new DruidHttpClientConfig() {

        @Override
        public long getMaxQueuedBytes() {
            return 0L;
        }
    }, new DruidProcessingConfig() {

        @Override
        public String getFormatString() {
            return null;
        }
    }, ForkJoinPool.commonPool(), QueryStackTests.DEFAULT_NOOP_SCHEDULER, new MapJoinableFactory(ImmutableSet.of(), ImmutableMap.of()), new NoopServiceEmitter());
    ClientQuerySegmentWalker walker = new ClientQuerySegmentWalker(new ServiceEmitter("", "", null) {

        @Override
        public void emit(Event event) {
        }
    }, baseClient, null, /* local client; unused in this test, so pass in null */
    warehouse, new MapJoinableFactory(ImmutableSet.of(), ImmutableMap.of()), retryConfig, jsonMapper, serverConfig, null, new CacheConfig());
    defineMocks();
    QueryPlus queryPlus = QueryPlus.wrap(query);
    final Sequence<?> res = query.getRunner(walker).run(queryPlus);
    List actualResults = new ArrayList();
    actualResults = (List<MapBasedRow>) res.accumulate(actualResults, Accumulators.list());
    expectedResults = consistentTypeCasting(expectedResults);
    actualResults = consistentTypeCasting(actualResults);
    Assert.assertEquals(expectedResults, actualResults);
}

Example 5 with TimelineLookup

use of org.apache.druid.timeline.TimelineLookup in project druid by druid-io.

the class CoordinatorServerViewTest method testSingleServerAddedRemovedSegment.

@Test
public void testSingleServerAddedRemovedSegment() throws Exception {
    segmentViewInitLatch = new CountDownLatch(1);
    segmentAddedLatch = new CountDownLatch(1);
    segmentRemovedLatch = new CountDownLatch(1);
    setupViews();
    final DruidServer druidServer = new DruidServer("localhost:1234", "localhost:1234", null, 10000000L, ServerType.HISTORICAL, "default_tier", 0);
    setupZNodeForServer(druidServer, zkPathsConfig, jsonMapper);
    final DataSegment segment = dataSegmentWithIntervalAndVersion("2014-10-20T00:00:00Z/P1D", "v1");
    final int partition = segment.getShardSpec().getPartitionNum();
    final Interval intervals = Intervals.of("2014-10-20T00:00:00Z/P1D");
    announceSegmentForServer(druidServer, segment, zkPathsConfig, jsonMapper);
    Assert.assertTrue(timing.forWaiting().awaitLatch(segmentViewInitLatch));
    Assert.assertTrue(timing.forWaiting().awaitLatch(segmentAddedLatch));
    TimelineLookup timeline = overlordServerView.getTimeline(new TableDataSource("test_overlord_server_view"));
    List<TimelineObjectHolder> serverLookupRes = (List<TimelineObjectHolder>) timeline.lookup(intervals);
    Assert.assertEquals(1, serverLookupRes.size());
    TimelineObjectHolder<String, SegmentLoadInfo> actualTimelineObjectHolder = serverLookupRes.get(0);
    Assert.assertEquals(intervals, actualTimelineObjectHolder.getInterval());
    Assert.assertEquals("v1", actualTimelineObjectHolder.getVersion());
    PartitionHolder<SegmentLoadInfo> actualPartitionHolder = actualTimelineObjectHolder.getObject();
    Assert.assertTrue(actualPartitionHolder.isComplete());
    Assert.assertEquals(1, Iterables.size(actualPartitionHolder));
    SegmentLoadInfo segmentLoadInfo = actualPartitionHolder.iterator().next().getObject();
    Assert.assertFalse(segmentLoadInfo.isEmpty());
    Assert.assertEquals(druidServer.getMetadata(), Iterables.getOnlyElement(segmentLoadInfo.toImmutableSegmentLoadInfo().getServers()));
    Assert.assertNotNull(timeline.findChunk(intervals, "v1", partition));
    unannounceSegmentForServer(druidServer, segment);
    Assert.assertTrue(timing.forWaiting().awaitLatch(segmentRemovedLatch));
    Assert.assertEquals(0, ((List<TimelineObjectHolder>) timeline.lookup(Intervals.of("2014-10-20T00:00:00Z/P1D"))).size());
    Assert.assertNull(timeline.findChunk(intervals, "v1", partition));
}