Examples with LookupExtractionFn io.druid.query.lookup.LookupExtractionFn used on opensource projects

Example 16 with LookupExtractionFn

use of io.druid.query.lookup.LookupExtractionFn in project druid by druid-io.

the class LongFilteringTest method testLongFilterWithExtractionFn.

@Test
public void testLongFilterWithExtractionFn() {
    final Map<String, String> stringMap = new HashMap<>();
    stringMap.put("1", "Monday");
    stringMap.put("2", "Tuesday");
    stringMap.put("3", "Wednesday");
    stringMap.put("4", "Thursday");
    stringMap.put("5", "Friday");
    stringMap.put("6", "Saturday");
    LookupExtractor mapExtractor = new MapLookupExtractor(stringMap, false);
    LookupExtractionFn exfn = new LookupExtractionFn(mapExtractor, false, "UNKNOWN", false, true);
    assertFilterMatches(new SelectorDimFilter(LONG_COLUMN, "Monday", exfn), ImmutableList.<String>of("1"));
    assertFilterMatches(new SelectorDimFilter(LONG_COLUMN, "Notaday", exfn), ImmutableList.<String>of());
    assertFilterMatches(new BoundDimFilter(LONG_COLUMN, "Fridax", "Fridaz", false, false, null, exfn, StringComparators.ALPHANUMERIC), ImmutableList.<String>of("5"));
    assertFilterMatches(new BoundDimFilter(LONG_COLUMN, "Friday", "Friday", true, true, null, exfn, StringComparators.ALPHANUMERIC), ImmutableList.<String>of());
    assertFilterMatches(new InDimFilter(LONG_COLUMN, Arrays.asList("Caturday", "Saturday", "Tuesday"), exfn), ImmutableList.<String>of("2", "6"));
    // test InFilter HashSet implementation
    List<String> bigList = Arrays.asList("Saturday", "Tuesday", "Caturday", "Xanaday", "Vojuday", "Gribaday", "Kipoday", "Dheferday", "Fakeday", "Qeearaday", "Hello", "World", "1", "2", "3", "4", "5", "6", "7");
    assertFilterMatches(new InDimFilter(LONG_COLUMN, bigList, exfn), ImmutableList.<String>of("2", "6"));
    String jsFn = "function(x) { return(x === 'Wednesday' || x === 'Thursday') }";
    assertFilterMatches(new JavaScriptDimFilter(LONG_COLUMN, jsFn, exfn, JavaScriptConfig.getEnabledInstance()), ImmutableList.<String>of("3", "4"));
    assertFilterMatches(new RegexDimFilter(LONG_COLUMN, ".*day", exfn), ImmutableList.<String>of("1", "2", "3", "4", "5", "6"));
    assertFilterMatches(new SearchQueryDimFilter(LONG_COLUMN, new ContainsSearchQuerySpec("s", true), exfn), ImmutableList.<String>of("2", "3", "4"));
}

Example 17 with LookupExtractionFn

use of io.druid.query.lookup.LookupExtractionFn in project druid by druid-io.

the class SelectorFilterTest method testSelectorWithLookupExtractionFn.

@Test
public void testSelectorWithLookupExtractionFn() {
    final Map<String, String> stringMap = ImmutableMap.of("1", "HELLO", "a", "HELLO", "def", "HELLO", "abc", "UNKNOWN");
    LookupExtractor mapExtractor = new MapLookupExtractor(stringMap, false);
    LookupExtractionFn lookupFn = new LookupExtractionFn(mapExtractor, false, "UNKNOWN", false, true);
    assertFilterMatches(new SelectorDimFilter("dim0", "HELLO", lookupFn), ImmutableList.of("1"));
    assertFilterMatches(new SelectorDimFilter("dim0", "UNKNOWN", lookupFn), ImmutableList.of("0", "2", "3", "4", "5"));
    assertFilterMatches(new SelectorDimFilter("dim1", "HELLO", lookupFn), ImmutableList.of("3", "4"));
    assertFilterMatches(new SelectorDimFilter("dim1", "UNKNOWN", lookupFn), ImmutableList.of("0", "1", "2", "5"));
    assertFilterMatches(new SelectorDimFilter("dim2", "HELLO", lookupFn), ImmutableList.of("0", "3"));
    assertFilterMatches(new SelectorDimFilter("dim2", "UNKNOWN", lookupFn), ImmutableList.of("0", "1", "2", "4", "5"));
    assertFilterMatches(new SelectorDimFilter("dim3", "HELLO", lookupFn), ImmutableList.<String>of());
    assertFilterMatches(new SelectorDimFilter("dim3", "UNKNOWN", lookupFn), ImmutableList.of("0", "1", "2", "3", "4", "5"));
    assertFilterMatches(new SelectorDimFilter("dim4", "HELLO", lookupFn), ImmutableList.<String>of());
    assertFilterMatches(new SelectorDimFilter("dim4", "UNKNOWN", lookupFn), ImmutableList.of("0", "1", "2", "3", "4", "5"));
    final Map<String, String> stringMap2 = ImmutableMap.of("2", "5");
    LookupExtractor mapExtractor2 = new MapLookupExtractor(stringMap2, false);
    LookupExtractionFn lookupFn2 = new LookupExtractionFn(mapExtractor2, true, null, false, true);
    assertFilterMatches(new SelectorDimFilter("dim0", "5", lookupFn2), ImmutableList.of("2", "5"));
    final Map<String, String> stringMap3 = ImmutableMap.of("1", "");
    LookupExtractor mapExtractor3 = new MapLookupExtractor(stringMap3, false);
    LookupExtractionFn lookupFn3 = new LookupExtractionFn(mapExtractor3, false, null, false, true);
    assertFilterMatches(new SelectorDimFilter("dim0", null, lookupFn3), ImmutableList.of("0", "1", "2", "3", "4", "5"));
    final Map<String, String> stringMap4 = ImmutableMap.of("9", "4");
    LookupExtractor mapExtractor4 = new MapLookupExtractor(stringMap4, false);
    LookupExtractionFn lookupFn4 = new LookupExtractionFn(mapExtractor4, true, null, false, true);
    final Map<String, String> stringMap5 = ImmutableMap.of("5", "44");
    LookupExtractor mapExtractor5 = new MapLookupExtractor(stringMap5, false);
    LookupExtractionFn lookupFn5 = new LookupExtractionFn(mapExtractor5, true, null, false, true);
    final Map<String, String> stringMap6 = ImmutableMap.of("5", "5");
    LookupExtractor mapExtractor6 = new MapLookupExtractor(stringMap6, false);
    LookupExtractionFn lookupFn6 = new LookupExtractionFn(mapExtractor6, true, null, false, true);
    // optimize() tests, check that filter was converted to the proper form
    SelectorDimFilter optFilter1 = new SelectorDimFilter("dim1", "UNKNOWN", lookupFn);
    SelectorDimFilter optFilter2 = new SelectorDimFilter("dim0", "5", lookupFn2);
    SelectorDimFilter optFilter3 = new SelectorDimFilter("dim0", null, lookupFn3);
    SelectorDimFilter optFilter4 = new SelectorDimFilter("dim0", "5", lookupFn4);
    SelectorDimFilter optFilter5 = new SelectorDimFilter("dim0", "5", lookupFn5);
    SelectorDimFilter optFilter6 = new SelectorDimFilter("dim0", "5", lookupFn6);
    InDimFilter optFilter2Optimized = new InDimFilter("dim0", Arrays.asList("2", "5"), null);
    SelectorDimFilter optFilter4Optimized = new SelectorDimFilter("dim0", "5", null);
    SelectorDimFilter optFilter6Optimized = new SelectorDimFilter("dim0", "5", null);
    Assert.assertTrue(optFilter1.equals(optFilter1.optimize()));
    Assert.assertTrue(optFilter2Optimized.equals(optFilter2.optimize()));
    Assert.assertTrue(optFilter3.equals(optFilter3.optimize()));
    Assert.assertTrue(optFilter4Optimized.equals(optFilter4.optimize()));
    Assert.assertTrue(optFilter5.equals(optFilter5.optimize()));
    Assert.assertTrue(optFilter6Optimized.equals(optFilter6.optimize()));
    assertFilterMatches(optFilter1, ImmutableList.of("0", "1", "2", "5"));
    assertFilterMatches(optFilter2, ImmutableList.of("2", "5"));
    assertFilterMatches(optFilter3, ImmutableList.of("0", "1", "2", "3", "4", "5"));
    assertFilterMatches(optFilter4, ImmutableList.of("5"));
    assertFilterMatches(optFilter5, ImmutableList.<String>of());
    assertFilterMatches(optFilter6, ImmutableList.of("5"));
    // tests that ExtractionDimFilter (identical to SelectorDimFilter now) optimize() with lookup works
    // remove these when ExtractionDimFilter is removed.
    assertFilterMatches(new ExtractionDimFilter("dim1", "UNKNOWN", lookupFn, null), ImmutableList.of("0", "1", "2", "5"));
    assertFilterMatches(new ExtractionDimFilter("dim0", "5", lookupFn2, null), ImmutableList.of("2", "5"));
    assertFilterMatches(new ExtractionDimFilter("dim0", null, lookupFn3, null), ImmutableList.of("0", "1", "2", "3", "4", "5"));
}

Example 18 with LookupExtractionFn

use of io.druid.query.lookup.LookupExtractionFn in project druid by druid-io.

the class GroupByQueryRunnerTest method testGroupByWithAlphaNumericDimensionOrder.

@Test
public void testGroupByWithAlphaNumericDimensionOrder() {
    Map<String, String> map = new HashMap<>();
    map.put("automotive", "health105");
    map.put("business", "health20");
    map.put("entertainment", "travel47");
    map.put("health", "health55");
    map.put("mezzanine", "health09");
    map.put("news", "health0000");
    map.put("premium", "health999");
    map.put("technology", "travel123");
    map.put("travel", "travel555");
    GroupByQuery query = GroupByQuery.builder().setDataSource(QueryRunnerTestHelper.dataSource).setQuerySegmentSpec(QueryRunnerTestHelper.firstToThird).setDimensions(Lists.<DimensionSpec>newArrayList(new ExtractionDimensionSpec("quality", "alias", new LookupExtractionFn(new MapLookupExtractor(map, false), false, null, false, false)))).setAggregatorSpecs(Arrays.asList(QueryRunnerTestHelper.rowsCount, new LongSumAggregatorFactory("idx", "index"))).setLimitSpec(new DefaultLimitSpec(Lists.<OrderByColumnSpec>newArrayList(new OrderByColumnSpec("alias", null, StringComparators.ALPHANUMERIC)), null)).setGranularity(QueryRunnerTestHelper.dayGran).build();
    List<Row> expectedResults = Arrays.asList(GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "health0000", "rows", 1L, "idx", 121L), GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "health09", "rows", 3L, "idx", 2870L), GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "health20", "rows", 1L, "idx", 118L), GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "health55", "rows", 1L, "idx", 120L), GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "health105", "rows", 1L, "idx", 135L), GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "health999", "rows", 3L, "idx", 2900L), GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "travel47", "rows", 1L, "idx", 158L), GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "travel123", "rows", 1L, "idx", 78L), GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-01", "alias", "travel555", "rows", 1L, "idx", 119L), GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "health0000", "rows", 1L, "idx", 114L), GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "health09", "rows", 3L, "idx", 2447L), GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "health20", "rows", 1L, "idx", 112L), GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "health55", "rows", 1L, "idx", 113L), GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "health105", "rows", 1L, "idx", 147L), GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "health999", "rows", 3L, "idx", 2505L), GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "travel47", "rows", 1L, "idx", 166L), GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "travel123", "rows", 1L, "idx", 97L), GroupByQueryRunnerTestHelper.createExpectedRow("2011-04-02", "alias", "travel555", "rows", 1L, "idx", 126L));
    Iterable<Row> results = GroupByQueryRunnerTestHelper.runQuery(factory, runner, query);
    TestHelper.assertExpectedObjects(expectedResults, results, "");
}

Example 19 with LookupExtractionFn

use of io.druid.query.lookup.LookupExtractionFn in project druid by druid-io.

the class SearchQueryRunnerTest method testSearchWithExtractionFilter1.

@Test
public void testSearchWithExtractionFilter1() {
    final String automotiveSnowman = "automotive☃";
    List<SearchHit> expectedHits = Lists.newLinkedList();
    expectedHits.add(new SearchHit(QueryRunnerTestHelper.qualityDimension, automotiveSnowman, 93));
    final LookupExtractionFn lookupExtractionFn = new LookupExtractionFn(new MapLookupExtractor(ImmutableMap.of("automotive", automotiveSnowman), false), true, null, true, true);
    SearchQuery query = Druids.newSearchQueryBuilder().dataSource(QueryRunnerTestHelper.dataSource).granularity(QueryRunnerTestHelper.allGran).filters(new ExtractionDimFilter(QueryRunnerTestHelper.qualityDimension, automotiveSnowman, lookupExtractionFn, null)).intervals(QueryRunnerTestHelper.fullOnInterval).dimensions(new ExtractionDimensionSpec(QueryRunnerTestHelper.qualityDimension, null, lookupExtractionFn)).query("☃").build();
    checkSearchQuery(query, expectedHits);
}

Example 20 with LookupExtractionFn

use of io.druid.query.lookup.LookupExtractionFn in project druid by druid-io.

the class TimeFilteringTest method testTimeFilterWithExtractionFn.

@Test
public void testTimeFilterWithExtractionFn() {
    final Map<String, String> stringMap = new HashMap<>();
    stringMap.put("0", "Monday");
    stringMap.put("1", "Tuesday");
    stringMap.put("2", "Wednesday");
    stringMap.put("3", "Thursday");
    stringMap.put("4", "Friday");
    stringMap.put("5", "Saturday");
    LookupExtractor mapExtractor = new MapLookupExtractor(stringMap, false);
    LookupExtractionFn exfn = new LookupExtractionFn(mapExtractor, false, "UNKNOWN", false, true);
    assertFilterMatches(new SelectorDimFilter(Column.TIME_COLUMN_NAME, "Monday", exfn), ImmutableList.<String>of("0"));
    assertFilterMatches(new SelectorDimFilter(Column.TIME_COLUMN_NAME, "Notaday", exfn), ImmutableList.<String>of());
    assertFilterMatches(new BoundDimFilter(Column.TIME_COLUMN_NAME, "Fridax", "Fridaz", false, false, null, exfn, StringComparators.ALPHANUMERIC), ImmutableList.<String>of("4"));
    assertFilterMatches(new BoundDimFilter(Column.TIME_COLUMN_NAME, "Friday", "Friday", true, true, null, exfn, StringComparators.ALPHANUMERIC), ImmutableList.<String>of());
    assertFilterMatches(new InDimFilter(Column.TIME_COLUMN_NAME, Arrays.asList("Caturday", "Saturday", "Tuesday"), exfn), ImmutableList.<String>of("1", "5"));
    // test InFilter HashSet implementation
    List<String> bigList = Arrays.asList("Saturday", "Tuesday", "Caturday", "Xanaday", "Vojuday", "Gribaday", "Kipoday", "Dheferday", "Fakeday", "Qeearaday", "Hello", "World", "1", "2", "3", "4", "5", "6", "7");
    assertFilterMatches(new InDimFilter(Column.TIME_COLUMN_NAME, bigList, exfn), ImmutableList.<String>of("1", "5"));
    String jsFn = "function(x) { return(x === 'Wednesday' || x === 'Thursday') }";
    assertFilterMatches(new JavaScriptDimFilter(Column.TIME_COLUMN_NAME, jsFn, exfn, JavaScriptConfig.getEnabledInstance()), ImmutableList.<String>of("2", "3"));
    assertFilterMatches(new RegexDimFilter(Column.TIME_COLUMN_NAME, ".*day", exfn), ImmutableList.<String>of("0", "1", "2", "3", "4", "5"));
    assertFilterMatches(new SearchQueryDimFilter(Column.TIME_COLUMN_NAME, new ContainsSearchQuerySpec("s", true), exfn), ImmutableList.<String>of("1", "2", "3"));
}