Examples with LookupExtractionFn org.apache.druid.query.lookup.LookupExtractionFn used on opensource projects

Example 31 with LookupExtractionFn

use of org.apache.druid.query.lookup.LookupExtractionFn in project druid by druid-io.

the class GroupByQueryRunnerTest method testGroupByWithLookupAndLimitAndSortByDimsFirst.

@Test
public void testGroupByWithLookupAndLimitAndSortByDimsFirst() {
    // Cannot vectorize due to extraction dimension spec.
    cannotVectorize();
    Map<String, String> map = new HashMap<>();
    map.put("automotive", "9");
    map.put("business", "8");
    map.put("entertainment", "7");
    map.put("health", "6");
    map.put("mezzanine", "5");
    map.put("news", "4");
    map.put("premium", "3");
    map.put("technology", "2");
    map.put("travel", "1");
    GroupByQuery query = makeQueryBuilder().setDataSource(QueryRunnerTestHelper.DATA_SOURCE).setQuerySegmentSpec(QueryRunnerTestHelper.FIRST_TO_THIRD).setDimensions(new ExtractionDimensionSpec("quality", "alias", new LookupExtractionFn(new MapLookupExtractor(map, false), false, null, false, false))).setAggregatorSpecs(QueryRunnerTestHelper.ROWS_COUNT, new LongSumAggregatorFactory("idx", "index")).setLimitSpec(new DefaultLimitSpec(Collections.singletonList(new OrderByColumnSpec("alias", null, StringComparators.ALPHANUMERIC)), 11)).setGranularity(QueryRunnerTestHelper.DAY_GRAN).overrideContext(ImmutableMap.of("sortByDimsFirst", true)).build();
    List<ResultRow> expectedResults = Arrays.asList(makeRow(query, "2011-04-01", "alias", "1", "rows", 1L, "idx", 119L), makeRow(query, "2011-04-02", "alias", "1", "rows", 1L, "idx", 126L), makeRow(query, "2011-04-01", "alias", "2", "rows", 1L, "idx", 78L), makeRow(query, "2011-04-02", "alias", "2", "rows", 1L, "idx", 97L), makeRow(query, "2011-04-01", "alias", "3", "rows", 3L, "idx", 2900L), makeRow(query, "2011-04-02", "alias", "3", "rows", 3L, "idx", 2505L), makeRow(query, "2011-04-01", "alias", "4", "rows", 1L, "idx", 121L), makeRow(query, "2011-04-02", "alias", "4", "rows", 1L, "idx", 114L), makeRow(query, "2011-04-01", "alias", "5", "rows", 3L, "idx", 2870L), makeRow(query, "2011-04-02", "alias", "5", "rows", 3L, "idx", 2447L), makeRow(query, "2011-04-01", "alias", "6", "rows", 1L, "idx", 120L));
    Iterable<ResultRow> results = GroupByQueryRunnerTestHelper.runQuery(factory, runner, query);
    TestHelper.assertExpectedObjects(expectedResults, results, "lookup-limit");
}

Example 32 with LookupExtractionFn

use of org.apache.druid.query.lookup.LookupExtractionFn in project druid by druid-io.

the class GroupByQueryRunnerTest method testGroupByWithSimpleRenameRetainMissingNonInjective.

@Test
public void testGroupByWithSimpleRenameRetainMissingNonInjective() {
    // Cannot vectorize due to extraction dimension spec.
    cannotVectorize();
    Map<String, String> map = new HashMap<>();
    map.put("automotive", "automotive0");
    map.put("business", "business0");
    map.put("entertainment", "entertainment0");
    map.put("health", "health0");
    map.put("mezzanine", "mezzanine0");
    map.put("news", "news0");
    map.put("premium", "premium0");
    map.put("technology", "technology0");
    map.put("travel", "travel0");
    GroupByQuery query = makeQueryBuilder().setDataSource(QueryRunnerTestHelper.DATA_SOURCE).setQuerySegmentSpec(QueryRunnerTestHelper.FIRST_TO_THIRD).setDimensions(new ExtractionDimensionSpec("quality", "alias", new LookupExtractionFn(new MapLookupExtractor(map, false), true, null, false, false))).setAggregatorSpecs(QueryRunnerTestHelper.ROWS_COUNT, new LongSumAggregatorFactory("idx", "index")).setGranularity(QueryRunnerTestHelper.DAY_GRAN).build();
    List<ResultRow> expectedResults = Arrays.asList(makeRow(query, "2011-04-01", "alias", "automotive0", "rows", 1L, "idx", 135L), makeRow(query, "2011-04-01", "alias", "business0", "rows", 1L, "idx", 118L), makeRow(query, "2011-04-01", "alias", "entertainment0", "rows", 1L, "idx", 158L), makeRow(query, "2011-04-01", "alias", "health0", "rows", 1L, "idx", 120L), makeRow(query, "2011-04-01", "alias", "mezzanine0", "rows", 3L, "idx", 2870L), makeRow(query, "2011-04-01", "alias", "news0", "rows", 1L, "idx", 121L), makeRow(query, "2011-04-01", "alias", "premium0", "rows", 3L, "idx", 2900L), makeRow(query, "2011-04-01", "alias", "technology0", "rows", 1L, "idx", 78L), makeRow(query, "2011-04-01", "alias", "travel0", "rows", 1L, "idx", 119L), makeRow(query, "2011-04-02", "alias", "automotive0", "rows", 1L, "idx", 147L), makeRow(query, "2011-04-02", "alias", "business0", "rows", 1L, "idx", 112L), makeRow(query, "2011-04-02", "alias", "entertainment0", "rows", 1L, "idx", 166L), makeRow(query, "2011-04-02", "alias", "health0", "rows", 1L, "idx", 113L), makeRow(query, "2011-04-02", "alias", "mezzanine0", "rows", 3L, "idx", 2447L), makeRow(query, "2011-04-02", "alias", "news0", "rows", 1L, "idx", 114L), makeRow(query, "2011-04-02", "alias", "premium0", "rows", 3L, "idx", 2505L), makeRow(query, "2011-04-02", "alias", "technology0", "rows", 1L, "idx", 97L), makeRow(query, "2011-04-02", "alias", "travel0", "rows", 1L, "idx", 126L));
    Iterable<ResultRow> results = GroupByQueryRunnerTestHelper.runQuery(factory, runner, query);
    TestHelper.assertExpectedObjects(expectedResults, results, "non-injective");
}

Example 33 with LookupExtractionFn

use of org.apache.druid.query.lookup.LookupExtractionFn in project druid by druid-io.

the class TopNQueryRunnerTest method testTopNWithExtractionFilterAndFilteredAggregatorCaseNoExistingValue.

@Test
public void testTopNWithExtractionFilterAndFilteredAggregatorCaseNoExistingValue() {
    Map<String, String> extractionMap = new HashMap<>();
    MapLookupExtractor mapLookupExtractor = new MapLookupExtractor(extractionMap, false);
    LookupExtractionFn lookupExtractionFn;
    if (NullHandling.replaceWithDefault()) {
        lookupExtractionFn = new LookupExtractionFn(mapLookupExtractor, false, null, true, false);
        extractionMap.put("", "NULL");
    } else {
        lookupExtractionFn = new LookupExtractionFn(mapLookupExtractor, false, "NULL", true, false);
    }
    DimFilter extractionFilter = new ExtractionDimFilter("null_column", "NULL", lookupExtractionFn, null);
    TopNQueryBuilder topNQueryBuilder = new TopNQueryBuilder().dataSource(QueryRunnerTestHelper.DATA_SOURCE).granularity(QueryRunnerTestHelper.ALL_GRAN).dimension("null_column").metric(QueryRunnerTestHelper.INDEX_METRIC).threshold(4).intervals(QueryRunnerTestHelper.FULL_ON_INTERVAL_SPEC).aggregators(Lists.newArrayList(Iterables.concat(commonAggregators, Lists.newArrayList(new FilteredAggregatorFactory(new DoubleMaxAggregatorFactory("maxIndex", "index"), extractionFilter), new DoubleMinAggregatorFactory("minIndex", "index"))))).postAggregators(QueryRunnerTestHelper.ADD_ROWS_INDEX_CONSTANT);
    TopNQuery topNQueryWithNULLValueExtraction = topNQueryBuilder.filters(extractionFilter).build();
    Map<String, Object> map = new HashMap<>();
    map.put("null_column", null);
    map.put("rows", 1209L);
    map.put("index", 503332.5071372986D);
    map.put("addRowsIndexConstant", 504542.5071372986D);
    map.put("uniques", QueryRunnerTestHelper.UNIQUES_9);
    map.put("maxIndex", 1870.061029D);
    map.put("minIndex", 59.02102279663086D);
    List<Result<TopNResultValue>> expectedResults = Collections.singletonList(new Result<>(DateTimes.of("2011-01-12T00:00:00.000Z"), new TopNResultValue(Collections.singletonList(map))));
    assertExpectedResults(expectedResults, topNQueryWithNULLValueExtraction);
}

Example 34 with LookupExtractionFn

use of org.apache.druid.query.lookup.LookupExtractionFn in project druid by druid-io.

the class TopNQueryRunnerTest method testTopNDimExtractionFastTopNUnOptimalWithReplaceMissing.

@Test
public void testTopNDimExtractionFastTopNUnOptimalWithReplaceMissing() {
    TopNQuery query = new TopNQueryBuilder().dataSource(QueryRunnerTestHelper.DATA_SOURCE).granularity(QueryRunnerTestHelper.ALL_GRAN).dimension(new ExtractionDimensionSpec(QueryRunnerTestHelper.MARKET_DIMENSION, QueryRunnerTestHelper.MARKET_DIMENSION, new LookupExtractionFn(new MapLookupExtractor(ImmutableMap.of("spot", "2spot0", "total_market", "1total_market0", "upfront", "3upfront0"), false), false, "MISSING", false, false))).metric("rows").threshold(4).intervals(QueryRunnerTestHelper.FIRST_TO_THIRD).aggregators(commonAggregators).postAggregators(QueryRunnerTestHelper.ADD_ROWS_INDEX_CONSTANT).build();
    List<Result<TopNResultValue>> expectedResults = Collections.singletonList(new Result<>(DateTimes.of("2011-04-01T00:00:00.000Z"), new TopNResultValue(Arrays.<Map<String, Object>>asList(ImmutableMap.of(QueryRunnerTestHelper.MARKET_DIMENSION, "2spot0", "rows", 18L, "index", 2231.876812D, "addRowsIndexConstant", 2250.876812D, "uniques", QueryRunnerTestHelper.UNIQUES_9), ImmutableMap.of(QueryRunnerTestHelper.MARKET_DIMENSION, "1total_market0", "rows", 4L, "index", 5351.814783D, "addRowsIndexConstant", 5356.814783D, "uniques", QueryRunnerTestHelper.UNIQUES_2), ImmutableMap.of(QueryRunnerTestHelper.MARKET_DIMENSION, "3upfront0", "rows", 4L, "index", 4875.669692D, "addRowsIndexConstant", 4880.669692D, "uniques", QueryRunnerTestHelper.UNIQUES_2)))));
    assertExpectedResults(expectedResults, query);
}

Example 35 with LookupExtractionFn

use of org.apache.druid.query.lookup.LookupExtractionFn in project druid by druid-io.

the class TimeFilteringTest method testTimeFilterWithExtractionFn.

@Test
public void testTimeFilterWithExtractionFn() {
    final Map<String, String> stringMap = new HashMap<>();
    stringMap.put("0", "Monday");
    stringMap.put("1", "Tuesday");
    stringMap.put("2", "Wednesday");
    stringMap.put("3", "Thursday");
    stringMap.put("4", "Friday");
    stringMap.put("5", "Saturday");
    LookupExtractor mapExtractor = new MapLookupExtractor(stringMap, false);
    LookupExtractionFn exfn = new LookupExtractionFn(mapExtractor, false, "UNKNOWN", false, true);
    assertFilterMatches(new SelectorDimFilter(ColumnHolder.TIME_COLUMN_NAME, "Monday", exfn), ImmutableList.of("0"));
    assertFilterMatches(new SelectorDimFilter(ColumnHolder.TIME_COLUMN_NAME, "Notaday", exfn), ImmutableList.of());
    assertFilterMatches(new BoundDimFilter(ColumnHolder.TIME_COLUMN_NAME, "Fridax", "Fridaz", false, false, null, exfn, StringComparators.ALPHANUMERIC), ImmutableList.of("4"));
    assertFilterMatches(new BoundDimFilter(ColumnHolder.TIME_COLUMN_NAME, "Friday", "Friday", true, true, null, exfn, StringComparators.ALPHANUMERIC), ImmutableList.of());
    assertFilterMatches(new InDimFilter(ColumnHolder.TIME_COLUMN_NAME, Arrays.asList("Caturday", "Saturday", "Tuesday"), exfn), ImmutableList.of("1", "5"));
    // test InFilter HashSet implementation
    List<String> bigList = Arrays.asList("Saturday", "Tuesday", "Caturday", "Xanaday", "Vojuday", "Gribaday", "Kipoday", "Dheferday", "Fakeday", "Qeearaday", "Hello", "World", "1", "2", "3", "4", "5", "6", "7");
    assertFilterMatches(new InDimFilter(ColumnHolder.TIME_COLUMN_NAME, bigList, exfn), ImmutableList.of("1", "5"));
    String jsFn = "function(x) { return(x === 'Wednesday' || x === 'Thursday') }";
    assertFilterMatchesSkipVectorize(new JavaScriptDimFilter(ColumnHolder.TIME_COLUMN_NAME, jsFn, exfn, JavaScriptConfig.getEnabledInstance()), ImmutableList.of("2", "3"));
    assertFilterMatches(new RegexDimFilter(ColumnHolder.TIME_COLUMN_NAME, ".*day", exfn), ImmutableList.of("0", "1", "2", "3", "4", "5"));
    assertFilterMatches(new SearchQueryDimFilter(ColumnHolder.TIME_COLUMN_NAME, new ContainsSearchQuerySpec("s", true), exfn), ImmutableList.of("1", "2", "3"));
}