Examples with TableDataSource org.apache.druid.query.TableDataSource used on opensource projects

Example 96 with TableDataSource

use of org.apache.druid.query.TableDataSource in project druid by druid-io.

the class DruidSchemaTest method testLocalSegmentCacheSetsDataSourceAsGlobalAndJoinable.

@Test
public void testLocalSegmentCacheSetsDataSourceAsGlobalAndJoinable() throws InterruptedException {
    DruidTable fooTable = (DruidTable) schema.getTableMap().get("foo");
    Assert.assertNotNull(fooTable);
    Assert.assertTrue(fooTable.getDataSource() instanceof TableDataSource);
    Assert.assertFalse(fooTable.getDataSource() instanceof GlobalTableDataSource);
    Assert.assertFalse(fooTable.isJoinable());
    Assert.assertFalse(fooTable.isBroadcast());
    Assert.assertTrue(buildTableLatch.await(1, TimeUnit.SECONDS));
    buildTableLatch = new CountDownLatch(1);
    final DataSegment someNewBrokerSegment = new DataSegment("foo", Intervals.of("2012/2013"), "version1", null, ImmutableList.of("dim1", "dim2"), ImmutableList.of("met1", "met2"), new NumberedShardSpec(2, 3), null, 1, 100L, PruneSpecsHolder.DEFAULT);
    segmentDataSourceNames.add("foo");
    joinableDataSourceNames.add("foo");
    serverView.addSegment(someNewBrokerSegment, ServerType.BROKER);
    Assert.assertTrue(markDataSourceLatch.await(2, TimeUnit.SECONDS));
    // wait for build twice
    Assert.assertTrue(buildTableLatch.await(2, TimeUnit.SECONDS));
    // wait for get again, just to make sure table has been updated (latch counts down just before tables are updated)
    Assert.assertTrue(getDatasourcesLatch.await(2, TimeUnit.SECONDS));
    fooTable = (DruidTable) schema.getTableMap().get("foo");
    Assert.assertNotNull(fooTable);
    Assert.assertTrue(fooTable.getDataSource() instanceof TableDataSource);
    Assert.assertTrue(fooTable.getDataSource() instanceof GlobalTableDataSource);
    Assert.assertTrue(fooTable.isJoinable());
    Assert.assertTrue(fooTable.isBroadcast());
    // now remove it
    markDataSourceLatch = new CountDownLatch(1);
    buildTableLatch = new CountDownLatch(1);
    getDatasourcesLatch = new CountDownLatch(1);
    joinableDataSourceNames.remove("foo");
    segmentDataSourceNames.remove("foo");
    serverView.removeSegment(someNewBrokerSegment, ServerType.BROKER);
    Assert.assertTrue(markDataSourceLatch.await(2, TimeUnit.SECONDS));
    // wait for build
    Assert.assertTrue(buildTableLatch.await(2, TimeUnit.SECONDS));
    // wait for get again, just to make sure table has been updated (latch counts down just before tables are updated)
    Assert.assertTrue(getDatasourcesLatch.await(2, TimeUnit.SECONDS));
    fooTable = (DruidTable) schema.getTableMap().get("foo");
    Assert.assertNotNull(fooTable);
    Assert.assertTrue(fooTable.getDataSource() instanceof TableDataSource);
    Assert.assertFalse(fooTable.getDataSource() instanceof GlobalTableDataSource);
    Assert.assertFalse(fooTable.isJoinable());
    Assert.assertFalse(fooTable.isBroadcast());
}

Example 97 with TableDataSource

use of org.apache.druid.query.TableDataSource in project druid by druid-io.

the class TestClusterQuerySegmentWalker method getQueryRunnerForSegments.

@Override
public <T> QueryRunner<T> getQueryRunnerForSegments(final Query<T> query, final Iterable<SegmentDescriptor> specs) {
    final QueryRunnerFactory<T, Query<T>> factory = conglomerate.findFactory(query);
    if (factory == null) {
        throw new ISE("Unknown query type[%s].", query.getClass());
    }
    final DataSourceAnalysis analysis = DataSourceAnalysis.forDataSource(query.getDataSource());
    if (!analysis.isConcreteTableBased()) {
        throw new ISE("Cannot handle datasource: %s", query.getDataSource());
    }
    final String dataSourceName = ((TableDataSource) analysis.getBaseDataSource()).getName();
    final QueryToolChest<T, Query<T>> toolChest = factory.getToolchest();
    // Make sure this query type can handle the subquery, if present.
    if (analysis.isQuery() && !toolChest.canPerformSubquery(((QueryDataSource) analysis.getDataSource()).getQuery())) {
        throw new ISE("Cannot handle subquery: %s", analysis.getDataSource());
    }
    final Function<SegmentReference, SegmentReference> segmentMapFn = joinableFactoryWrapper.createSegmentMapFn(analysis.getJoinBaseTableFilter().map(Filters::toFilter).orElse(null), analysis.getPreJoinableClauses(), new AtomicLong(), analysis.getBaseQuery().orElse(query));
    final QueryRunner<T> baseRunner = new FinalizeResultsQueryRunner<>(toolChest.postMergeQueryDecoration(toolChest.mergeResults(toolChest.preMergeQueryDecoration(makeTableRunner(toolChest, factory, getSegmentsForTable(dataSourceName, specs), segmentMapFn)))), toolChest);
    // to actually serve the queries
    return (theQuery, responseContext) -> {
        responseContext.initializeRemainingResponses();
        responseContext.addRemainingResponse(theQuery.getQuery().getMostSpecificId(), 0);
        if (scheduler != null) {
            Set<SegmentServerSelector> segments = new HashSet<>();
            specs.forEach(spec -> segments.add(new SegmentServerSelector(spec)));
            return scheduler.run(scheduler.prioritizeAndLaneQuery(theQuery, segments), new LazySequence<>(() -> baseRunner.run(theQuery.withQuery(Queries.withSpecificSegments(theQuery.getQuery(), ImmutableList.copyOf(specs))), responseContext)));
        } else {
            return baseRunner.run(theQuery.withQuery(Queries.withSpecificSegments(theQuery.getQuery(), ImmutableList.copyOf(specs))), responseContext);
        }
    };
}

Example 98 with TableDataSource

use of org.apache.druid.query.TableDataSource in project druid by druid-io.

the class CalciteJoinQueryTest method testFilterAndGroupByLookupUsingJoinOperator.

@Test
@Parameters(source = QueryContextForJoinProvider.class)
public void testFilterAndGroupByLookupUsingJoinOperator(Map<String, Object> queryContext) throws Exception {
    // Cannot vectorize JOIN operator.
    cannotVectorize();
    testQuery("SELECT lookyloo.k, COUNT(*)\n" + "FROM foo LEFT JOIN lookup.lookyloo ON foo.dim2 = lookyloo.k\n" + "WHERE lookyloo.v = 'xa'\n" + "GROUP BY lookyloo.k", queryContext, ImmutableList.of(GroupByQuery.builder().setDataSource(join(new TableDataSource(CalciteTests.DATASOURCE1), new LookupDataSource("lookyloo"), "j0.", equalsCondition(makeColumnExpression("dim2"), makeColumnExpression("j0.k")), JoinType.LEFT)).setInterval(querySegmentSpec(Filtration.eternity())).setDimFilter(selector("j0.v", "xa", null)).setGranularity(Granularities.ALL).setDimensions(dimensions(new DefaultDimensionSpec("j0.k", "d0"))).setAggregatorSpecs(aggregators(new CountAggregatorFactory("a0"))).setContext(queryContext).build()), ImmutableList.of(new Object[] { "a", 2L }));
}

Example 99 with TableDataSource

use of org.apache.druid.query.TableDataSource in project druid by druid-io.

the class CalciteJoinQueryTest method testExactTopNOnInnerJoinWithLimit.

@Test
public void testExactTopNOnInnerJoinWithLimit() throws Exception {
    // Adjust topN threshold, so that the topN engine keeps only 1 slot for aggregates, which should be enough
    // to compute the query with limit 1.
    minTopNThreshold = 1;
    Map<String, Object> context = new HashMap<>(QUERY_CONTEXT_DEFAULT);
    context.put(PlannerConfig.CTX_KEY_USE_APPROXIMATE_TOPN, false);
    testQuery("select f1.\"dim4\", sum(\"m1\") from numfoo f1 inner join (\n" + "  select \"dim4\" from numfoo where dim4 <> 'a' group by 1\n" + ") f2 on f1.\"dim4\" = f2.\"dim4\" group by 1 limit 1", // turn on exact topN
    context, ImmutableList.of(new TopNQueryBuilder().intervals(querySegmentSpec(Filtration.eternity())).granularity(Granularities.ALL).dimension(new DefaultDimensionSpec("dim4", "_d0")).aggregators(new DoubleSumAggregatorFactory("a0", "m1")).metric(new DimensionTopNMetricSpec(null, StringComparators.LEXICOGRAPHIC)).threshold(1).dataSource(JoinDataSource.create(new TableDataSource("numfoo"), new QueryDataSource(GroupByQuery.builder().setInterval(querySegmentSpec(Filtration.eternity())).setGranularity(Granularities.ALL).setDimFilter(new NotDimFilter(new SelectorDimFilter("dim4", "a", null))).setDataSource(new TableDataSource("numfoo")).setDimensions(new DefaultDimensionSpec("dim4", "_d0")).setContext(context).build()), "j0.", "(\"dim4\" == \"j0._d0\")", JoinType.INNER, null, ExprMacroTable.nil())).context(context).build()), ImmutableList.of(new Object[] { "b", 15.0 }));
}

Example 100 with TableDataSource

use of org.apache.druid.query.TableDataSource in project druid by druid-io.

the class CalciteJoinQueryTest method testGroupByInnerJoinOnLookupUsingJoinOperator.

@Test
@Parameters(source = QueryContextForJoinProvider.class)
public void testGroupByInnerJoinOnLookupUsingJoinOperator(Map<String, Object> queryContext) throws Exception {
    // Cannot vectorize JOIN operator.
    cannotVectorize();
    testQuery("SELECT lookyloo.v, COUNT(*)\n" + "FROM foo INNER JOIN lookup.lookyloo ON foo.dim1 = lookyloo.k\n" + "GROUP BY lookyloo.v", queryContext, ImmutableList.of(GroupByQuery.builder().setDataSource(join(new TableDataSource(CalciteTests.DATASOURCE1), new LookupDataSource("lookyloo"), "j0.", equalsCondition(makeColumnExpression("dim1"), makeColumnExpression("j0.k")), JoinType.INNER)).setInterval(querySegmentSpec(Filtration.eternity())).setGranularity(Granularities.ALL).setDimensions(dimensions(new DefaultDimensionSpec("j0.v", "d0"))).setAggregatorSpecs(aggregators(new CountAggregatorFactory("a0"))).setContext(queryContext).build()), ImmutableList.of(new Object[] { "xabc", 1L }));
}