Search in sources :

Example 16 with ServerSelector

use of org.apache.druid.client.selector.ServerSelector in project druid by druid-io.

the class BrokerServerViewTest method testIgnoredTiers.

@Test
public void testIgnoredTiers() throws Exception {
    segmentViewInitLatch = new CountDownLatch(1);
    segmentAddedLatch = new CountDownLatch(4);
    segmentRemovedLatch = new CountDownLatch(0);
    // Setup a Broker that does not watch Tier 1
    final String tier1 = "tier1";
    final String tier2 = "tier2";
    setupViews(null, Sets.newHashSet(tier1), false);
    // Historical Tier 1 has segments 1 and 2, Tier 2 has segments 2 and 3
    final DruidServer server11 = setupHistoricalServer(tier1, "localhost:1", 1);
    final DruidServer server21 = setupHistoricalServer(tier2, "localhost:2", 1);
    final DataSegment segment1 = dataSegmentWithIntervalAndVersion("2020-01-01/P1D", "v1");
    announceSegmentForServer(server11, segment1, zkPathsConfig, jsonMapper);
    final DataSegment segment2 = dataSegmentWithIntervalAndVersion("2020-01-02/P1D", "v1");
    announceSegmentForServer(server11, segment2, zkPathsConfig, jsonMapper);
    announceSegmentForServer(server21, segment2, zkPathsConfig, jsonMapper);
    final DataSegment segment3 = dataSegmentWithIntervalAndVersion("2020-01-03/P1D", "v1");
    announceSegmentForServer(server21, segment3, zkPathsConfig, jsonMapper);
    // Wait for the segments to be added
    Assert.assertTrue(timing.forWaiting().awaitLatch(segmentViewInitLatch));
    Assert.assertTrue(timing.forWaiting().awaitLatch(segmentAddedLatch));
    // Get the timeline for the datasource
    TimelineLookup<String, ServerSelector> timeline = brokerServerView.getTimeline(DataSourceAnalysis.forDataSource(new TableDataSource(segment1.getDataSource()))).get();
    // Verify that the timeline has no entry for the interval of segment 1
    Assert.assertTrue(timeline.lookup(segment1.getInterval()).isEmpty());
    // Verify that there is one entry for the interval of segment 2
    List<TimelineObjectHolder<String, ServerSelector>> timelineHolders = timeline.lookup(segment2.getInterval());
    Assert.assertEquals(1, timelineHolders.size());
    TimelineObjectHolder<String, ServerSelector> timelineHolder = timelineHolders.get(0);
    Assert.assertEquals(segment2.getInterval(), timelineHolder.getInterval());
    Assert.assertEquals(segment2.getVersion(), timelineHolder.getVersion());
    PartitionHolder<ServerSelector> partitionHolder = timelineHolder.getObject();
    Assert.assertTrue(partitionHolder.isComplete());
    Assert.assertEquals(1, Iterables.size(partitionHolder));
    ServerSelector selector = (partitionHolder.iterator().next()).getObject();
    Assert.assertFalse(selector.isEmpty());
    Assert.assertEquals(segment2, selector.getSegment());
    // Verify that the ServerSelector always picks Tier 1
    for (int i = 0; i < 5; ++i) {
        Assert.assertEquals(server21, selector.pick(null).getServer());
    }
    Assert.assertEquals(Collections.singletonList(server21.getMetadata()), selector.getCandidates(2));
}
Also used : ServerSelector(org.apache.druid.client.selector.ServerSelector) TimelineObjectHolder(org.apache.druid.timeline.TimelineObjectHolder) TableDataSource(org.apache.druid.query.TableDataSource) CountDownLatch(java.util.concurrent.CountDownLatch) DataSegment(org.apache.druid.timeline.DataSegment) Test(org.junit.Test)

Example 17 with ServerSelector

use of org.apache.druid.client.selector.ServerSelector in project druid by druid-io.

the class BrokerServerViewTest method testSingleServerAddedRemovedSegment.

@Test
public void testSingleServerAddedRemovedSegment() throws Exception {
    segmentViewInitLatch = new CountDownLatch(1);
    segmentAddedLatch = new CountDownLatch(1);
    segmentRemovedLatch = new CountDownLatch(1);
    setupViews();
    final DruidServer druidServer = setupHistoricalServer("default_tier", "localhost:1234", 0);
    final DataSegment segment = dataSegmentWithIntervalAndVersion("2014-10-20T00:00:00Z/P1D", "v1");
    final int partition = segment.getShardSpec().getPartitionNum();
    final Interval intervals = Intervals.of("2014-10-20T00:00:00Z/P1D");
    announceSegmentForServer(druidServer, segment, zkPathsConfig, jsonMapper);
    Assert.assertTrue(timing.forWaiting().awaitLatch(segmentViewInitLatch));
    Assert.assertTrue(timing.forWaiting().awaitLatch(segmentAddedLatch));
    TimelineLookup<String, ServerSelector> timeline = brokerServerView.getTimeline(DataSourceAnalysis.forDataSource(new TableDataSource("test_broker_server_view"))).get();
    List<TimelineObjectHolder<String, ServerSelector>> serverLookupRes = timeline.lookup(intervals);
    Assert.assertEquals(1, serverLookupRes.size());
    TimelineObjectHolder<String, ServerSelector> actualTimelineObjectHolder = serverLookupRes.get(0);
    Assert.assertEquals(intervals, actualTimelineObjectHolder.getInterval());
    Assert.assertEquals("v1", actualTimelineObjectHolder.getVersion());
    PartitionHolder<ServerSelector> actualPartitionHolder = actualTimelineObjectHolder.getObject();
    Assert.assertTrue(actualPartitionHolder.isComplete());
    Assert.assertEquals(1, Iterables.size(actualPartitionHolder));
    ServerSelector selector = (actualPartitionHolder.iterator().next()).getObject();
    Assert.assertFalse(selector.isEmpty());
    Assert.assertEquals(segment, selector.getSegment());
    Assert.assertEquals(druidServer, selector.pick(null).getServer());
    Assert.assertNotNull(timeline.findChunk(intervals, "v1", partition));
    unannounceSegmentForServer(druidServer, segment, zkPathsConfig);
    Assert.assertTrue(timing.forWaiting().awaitLatch(segmentRemovedLatch));
    Assert.assertEquals(0, timeline.lookup(intervals).size());
    Assert.assertNull(timeline.findChunk(intervals, "v1", partition));
}
Also used : CountDownLatch(java.util.concurrent.CountDownLatch) DataSegment(org.apache.druid.timeline.DataSegment) ServerSelector(org.apache.druid.client.selector.ServerSelector) TimelineObjectHolder(org.apache.druid.timeline.TimelineObjectHolder) TableDataSource(org.apache.druid.query.TableDataSource) Interval(org.joda.time.Interval) Test(org.junit.Test)

Example 18 with ServerSelector

use of org.apache.druid.client.selector.ServerSelector in project druid by druid-io.

the class CachingClusteredClientCacheKeyManagerTest method makeServerSelector.

/**
 * using partitionNumber, its possible to create segments with different ids
 */
private SegmentServerSelector makeServerSelector(boolean isHistorical, int partitionNumber) {
    ServerSelector serverSelector = mock(ServerSelector.class);
    QueryableDruidServer queryableDruidServer = mock(QueryableDruidServer.class);
    DruidServer server = mock(DruidServer.class);
    SegmentId segmentId = SegmentId.dummy("data-source", partitionNumber);
    DataSegment segment = new DataSegment(segmentId, null, null, null, new NumberedShardSpec(partitionNumber, 10), null, 0, 0);
    expect(server.isSegmentReplicationTarget()).andReturn(isHistorical).anyTimes();
    expect(serverSelector.pick(query)).andReturn(queryableDruidServer).anyTimes();
    expect(queryableDruidServer.getServer()).andReturn(server).anyTimes();
    expect(serverSelector.getSegment()).andReturn(segment).anyTimes();
    replay(serverSelector, queryableDruidServer, server);
    return new SegmentServerSelector(serverSelector, segmentId.toDescriptor());
}
Also used : ServerSelector(org.apache.druid.client.selector.ServerSelector) SegmentId(org.apache.druid.timeline.SegmentId) QueryableDruidServer(org.apache.druid.client.selector.QueryableDruidServer) DataSegment(org.apache.druid.timeline.DataSegment) NumberedShardSpec(org.apache.druid.timeline.partition.NumberedShardSpec) QueryableDruidServer(org.apache.druid.client.selector.QueryableDruidServer)

Example 19 with ServerSelector

use of org.apache.druid.client.selector.ServerSelector in project druid by druid-io.

the class SimpleServerView method addSegmentToServer.

private void addSegmentToServer(DruidServer server, DataSegment segment) {
    final ServerSelector selector = selectors.computeIfAbsent(segment.getId().toString(), k -> new ServerSelector(segment, tierSelectorStrategy));
    selector.addServerAndUpdateSegment(servers.get(server), segment);
    timelines.computeIfAbsent(segment.getDataSource(), k -> new VersionedIntervalTimeline<>(Ordering.natural())).add(segment.getInterval(), segment.getVersion(), segment.getShardSpec().createChunk(selector));
}
Also used : DataSourceAnalysis(org.apache.druid.query.planning.DataSourceAnalysis) HighestPriorityTierSelectorStrategy(org.apache.druid.client.selector.HighestPriorityTierSelectorStrategy) HttpClient(org.apache.druid.java.util.http.client.HttpClient) HashMap(java.util.HashMap) ServerSelector(org.apache.druid.client.selector.ServerSelector) QueryWatcher(org.apache.druid.query.QueryWatcher) Map(java.util.Map) ServerType(org.apache.druid.server.coordination.ServerType) TierSelectorStrategy(org.apache.druid.client.selector.TierSelectorStrategy) QueryRunner(org.apache.druid.query.QueryRunner) QueryToolChestWarehouse(org.apache.druid.query.QueryToolChestWarehouse) NoopServiceEmitter(org.apache.druid.server.metrics.NoopServiceEmitter) TimelineLookup(org.apache.druid.timeline.TimelineLookup) VersionedIntervalTimeline(org.apache.druid.timeline.VersionedIntervalTimeline) Executor(java.util.concurrent.Executor) ObjectMapper(com.fasterxml.jackson.databind.ObjectMapper) ISE(org.apache.druid.java.util.common.ISE) TableDataSource(org.apache.druid.query.TableDataSource) List(java.util.List) RandomServerSelectorStrategy(org.apache.druid.client.selector.RandomServerSelectorStrategy) Ordering(com.google.common.collect.Ordering) DataSegment(org.apache.druid.timeline.DataSegment) Optional(java.util.Optional) Preconditions(com.google.common.base.Preconditions) QueryableDruidServer(org.apache.druid.client.selector.QueryableDruidServer) Collections(java.util.Collections) ServerSelector(org.apache.druid.client.selector.ServerSelector) VersionedIntervalTimeline(org.apache.druid.timeline.VersionedIntervalTimeline)

Example 20 with ServerSelector

use of org.apache.druid.client.selector.ServerSelector in project druid by druid-io.

the class SimpleServerView method unannounceSegmentFromServer.

public void unannounceSegmentFromServer(DruidServer server, DataSegment segment) {
    final QueryableDruidServer queryableDruidServer = servers.get(server);
    if (queryableDruidServer == null) {
        throw new ISE("Unknown server [%s]", server);
    }
    final ServerSelector selector = selectors.get(segment.getId().toString());
    if (selector == null) {
        throw new ISE("Unknown segment [%s]", segment.getId());
    }
    if (!selector.removeServer(queryableDruidServer)) {
        throw new ISE("Failed to remove segment[%s] from server[%s]", segment.getId(), server);
    }
    final VersionedIntervalTimeline<String, ServerSelector> timeline = timelines.get(segment.getDataSource());
    if (timeline == null) {
        throw new ISE("Unknown datasource [%s]", segment.getDataSource());
    }
    timeline.remove(segment.getInterval(), segment.getVersion(), segment.getShardSpec().createChunk(selector));
}
Also used : ServerSelector(org.apache.druid.client.selector.ServerSelector) ISE(org.apache.druid.java.util.common.ISE) QueryableDruidServer(org.apache.druid.client.selector.QueryableDruidServer)

Aggregations

ServerSelector (org.apache.druid.client.selector.ServerSelector)26 DataSegment (org.apache.druid.timeline.DataSegment)17 QueryableDruidServer (org.apache.druid.client.selector.QueryableDruidServer)16 HighestPriorityTierSelectorStrategy (org.apache.druid.client.selector.HighestPriorityTierSelectorStrategy)11 Interval (org.joda.time.Interval)11 RandomServerSelectorStrategy (org.apache.druid.client.selector.RandomServerSelectorStrategy)10 Test (org.junit.Test)10 TableDataSource (org.apache.druid.query.TableDataSource)7 ResponseContext (org.apache.druid.query.context.ResponseContext)6 ArrayList (java.util.ArrayList)5 QueryRunner (org.apache.druid.query.QueryRunner)5 SegmentDescriptor (org.apache.druid.query.SegmentDescriptor)5 Map (java.util.Map)4 QueryPlus (org.apache.druid.query.QueryPlus)4 MultipleIntervalSegmentSpec (org.apache.druid.query.spec.MultipleIntervalSegmentSpec)4 ImmutableMap (com.google.common.collect.ImmutableMap)3 HashMap (java.util.HashMap)3 CountDownLatch (java.util.concurrent.CountDownLatch)3 Druids (org.apache.druid.query.Druids)3 FinalizeResultsQueryRunner (org.apache.druid.query.FinalizeResultsQueryRunner)3