Examples with LookupExtractor org.apache.druid.query.lookup.LookupExtractor used on opensource projects

Example 11 with LookupExtractor

use of org.apache.druid.query.lookup.LookupExtractor in project druid by druid-io.

the class TimeFilteringTest method testTimeFilterWithExtractionFn.

@Test
public void testTimeFilterWithExtractionFn() {
    final Map<String, String> stringMap = new HashMap<>();
    stringMap.put("0", "Monday");
    stringMap.put("1", "Tuesday");
    stringMap.put("2", "Wednesday");
    stringMap.put("3", "Thursday");
    stringMap.put("4", "Friday");
    stringMap.put("5", "Saturday");
    LookupExtractor mapExtractor = new MapLookupExtractor(stringMap, false);
    LookupExtractionFn exfn = new LookupExtractionFn(mapExtractor, false, "UNKNOWN", false, true);
    assertFilterMatches(new SelectorDimFilter(ColumnHolder.TIME_COLUMN_NAME, "Monday", exfn), ImmutableList.of("0"));
    assertFilterMatches(new SelectorDimFilter(ColumnHolder.TIME_COLUMN_NAME, "Notaday", exfn), ImmutableList.of());
    assertFilterMatches(new BoundDimFilter(ColumnHolder.TIME_COLUMN_NAME, "Fridax", "Fridaz", false, false, null, exfn, StringComparators.ALPHANUMERIC), ImmutableList.of("4"));
    assertFilterMatches(new BoundDimFilter(ColumnHolder.TIME_COLUMN_NAME, "Friday", "Friday", true, true, null, exfn, StringComparators.ALPHANUMERIC), ImmutableList.of());
    assertFilterMatches(new InDimFilter(ColumnHolder.TIME_COLUMN_NAME, Arrays.asList("Caturday", "Saturday", "Tuesday"), exfn), ImmutableList.of("1", "5"));
    // test InFilter HashSet implementation
    List<String> bigList = Arrays.asList("Saturday", "Tuesday", "Caturday", "Xanaday", "Vojuday", "Gribaday", "Kipoday", "Dheferday", "Fakeday", "Qeearaday", "Hello", "World", "1", "2", "3", "4", "5", "6", "7");
    assertFilterMatches(new InDimFilter(ColumnHolder.TIME_COLUMN_NAME, bigList, exfn), ImmutableList.of("1", "5"));
    String jsFn = "function(x) { return(x === 'Wednesday' || x === 'Thursday') }";
    assertFilterMatchesSkipVectorize(new JavaScriptDimFilter(ColumnHolder.TIME_COLUMN_NAME, jsFn, exfn, JavaScriptConfig.getEnabledInstance()), ImmutableList.of("2", "3"));
    assertFilterMatches(new RegexDimFilter(ColumnHolder.TIME_COLUMN_NAME, ".*day", exfn), ImmutableList.of("0", "1", "2", "3", "4", "5"));
    assertFilterMatches(new SearchQueryDimFilter(ColumnHolder.TIME_COLUMN_NAME, new ContainsSearchQuerySpec("s", true), exfn), ImmutableList.of("1", "2", "3"));
}

Example 12 with LookupExtractor

use of org.apache.druid.query.lookup.LookupExtractor in project druid by druid-io.

the class InFilterTest method testMatchWithLookupExtractionFn.

@Test
public void testMatchWithLookupExtractionFn() {
    final Map<String, String> stringMap = ImmutableMap.of("a", "HELLO", "10", "HELLO", "def", "HELLO", "c", "BYE");
    LookupExtractor mapExtractor = new MapLookupExtractor(stringMap, false);
    LookupExtractionFn lookupFn = new LookupExtractionFn(mapExtractor, false, "UNKNOWN", false, true);
    assertFilterMatches(toInFilterWithFn("dim0", lookupFn, null, "HELLO"), ImmutableList.of("a"));
    assertFilterMatches(toInFilterWithFn("dim0", lookupFn, "HELLO", "BYE"), ImmutableList.of("a", "c"));
    assertFilterMatches(toInFilterWithFn("dim0", lookupFn, "UNKNOWN"), ImmutableList.of("b", "d", "e", "f"));
    assertFilterMatches(toInFilterWithFn("dim1", lookupFn, "HELLO"), ImmutableList.of("b", "e"));
    assertFilterMatches(toInFilterWithFn("dim1", lookupFn, "N/A"), ImmutableList.of());
    assertFilterMatches(toInFilterWithFn("dim2", lookupFn, "a"), ImmutableList.of());
    assertFilterMatches(toInFilterWithFn("dim2", lookupFn, "HELLO"), ImmutableList.of("a", "d"));
    assertFilterMatches(toInFilterWithFn("dim2", lookupFn, "HELLO", "BYE", "UNKNOWN"), ImmutableList.of("a", "b", "c", "d", "e", "f"));
    final Map<String, String> stringMap2 = ImmutableMap.of("a", "e");
    LookupExtractor mapExtractor2 = new MapLookupExtractor(stringMap2, false);
    LookupExtractionFn lookupFn2 = new LookupExtractionFn(mapExtractor2, true, null, false, true);
    assertFilterMatches(toInFilterWithFn("dim0", lookupFn2, null, "e"), ImmutableList.of("a", "e"));
    assertFilterMatches(toInFilterWithFn("dim0", lookupFn2, "a"), ImmutableList.of());
    final Map<String, String> stringMap3 = ImmutableMap.of("c", "500", "100", "e");
    LookupExtractor mapExtractor3 = new MapLookupExtractor(stringMap3, false);
    LookupExtractionFn lookupFn3 = new LookupExtractionFn(mapExtractor3, false, null, false, true);
    assertFilterMatches(toInFilterWithFn("dim0", lookupFn3, null, "c"), ImmutableList.of("a", "b", "d", "e", "f"));
    assertFilterMatches(toInFilterWithFn("dim0", lookupFn3, "e"), ImmutableList.of());
}

Example 13 with LookupExtractor

use of org.apache.druid.query.lookup.LookupExtractor in project druid by druid-io.

the class JavaScriptFilterTest method testJavascriptFilterWithLookupExtractionFn.

@Test
public void testJavascriptFilterWithLookupExtractionFn() {
    final Map<String, String> stringMap = ImmutableMap.of("1", "HELLO", "a", "HELLO", "abdef", "HELLO", "abc", "UNKNOWN");
    LookupExtractor mapExtractor = new MapLookupExtractor(stringMap, false);
    LookupExtractionFn lookupFn = new LookupExtractionFn(mapExtractor, false, "UNKNOWN", false, true);
    assertFilterMatchesSkipVectorize(newJavaScriptDimFilter("dim0", jsValueFilter("HELLO"), lookupFn), ImmutableList.of("1"));
    assertFilterMatchesSkipVectorize(newJavaScriptDimFilter("dim0", jsValueFilter("UNKNOWN"), lookupFn), ImmutableList.of("0", "2", "3", "4", "5"));
    assertFilterMatchesSkipVectorize(newJavaScriptDimFilter("dim1", jsValueFilter("HELLO"), lookupFn), ImmutableList.of("3", "4"));
    assertFilterMatchesSkipVectorize(newJavaScriptDimFilter("dim1", jsValueFilter("UNKNOWN"), lookupFn), ImmutableList.of("0", "1", "2", "5"));
    assertFilterMatchesSkipVectorize(newJavaScriptDimFilter("dim2", jsValueFilter("HELLO"), lookupFn), ImmutableList.of("0", "3"));
    assertFilterMatchesSkipVectorize(newJavaScriptDimFilter("dim2", jsValueFilter("UNKNOWN"), lookupFn), ImmutableList.of("0", "1", "2", "4", "5"));
    assertFilterMatchesSkipVectorize(newJavaScriptDimFilter("dim3", jsValueFilter("HELLO"), lookupFn), ImmutableList.of());
    assertFilterMatchesSkipVectorize(newJavaScriptDimFilter("dim3", jsValueFilter("UNKNOWN"), lookupFn), ImmutableList.of("0", "1", "2", "3", "4", "5"));
    assertFilterMatchesSkipVectorize(newJavaScriptDimFilter("dim4", jsValueFilter("HELLO"), lookupFn), ImmutableList.of());
    assertFilterMatchesSkipVectorize(newJavaScriptDimFilter("dim4", jsValueFilter("UNKNOWN"), lookupFn), ImmutableList.of("0", "1", "2", "3", "4", "5"));
}

Example 14 with LookupExtractor

use of org.apache.druid.query.lookup.LookupExtractor in project druid by druid-io.

the class InDimFilter method optimizeLookup.

private InDimFilter optimizeLookup() {
    if (extractionFn instanceof LookupExtractionFn && ((LookupExtractionFn) extractionFn).isOptimize()) {
        LookupExtractionFn exFn = (LookupExtractionFn) extractionFn;
        LookupExtractor lookup = exFn.getLookup();
        final Set<String> keys = new HashSet<>();
        for (String value : values) {
            // We cannot do an unapply()-based optimization if the selector value
            // and the replaceMissingValuesWith value are the same, since we have to match on
            // all values that are not present in the lookup.
            final String convertedValue = NullHandling.emptyToNullIfNeeded(value);
            if (!exFn.isRetainMissingValue() && Objects.equals(convertedValue, exFn.getReplaceMissingValueWith())) {
                return this;
            }
            keys.addAll(lookup.unapply(convertedValue));
            // If the selector value is overwritten in the lookup map, don't add selector value to keys.
            if (exFn.isRetainMissingValue() && NullHandling.isNullOrEquivalent(lookup.apply(convertedValue))) {
                keys.add(convertedValue);
            }
        }
        if (keys.isEmpty()) {
            return this;
        } else {
            return new InDimFilter(dimension, keys, null, filterTuning);
        }
    }
    return this;
}

Example 15 with LookupExtractor

use of org.apache.druid.query.lookup.LookupExtractor in project druid by apache.

the class InFilterTest method testMatchWithLookupExtractionFn.

@Test
public void testMatchWithLookupExtractionFn() {
    final Map<String, String> stringMap = ImmutableMap.of("a", "HELLO", "10", "HELLO", "def", "HELLO", "c", "BYE");
    LookupExtractor mapExtractor = new MapLookupExtractor(stringMap, false);
    LookupExtractionFn lookupFn = new LookupExtractionFn(mapExtractor, false, "UNKNOWN", false, true);
    assertFilterMatches(toInFilterWithFn("dim0", lookupFn, null, "HELLO"), ImmutableList.of("a"));
    assertFilterMatches(toInFilterWithFn("dim0", lookupFn, "HELLO", "BYE"), ImmutableList.of("a", "c"));
    assertFilterMatches(toInFilterWithFn("dim0", lookupFn, "UNKNOWN"), ImmutableList.of("b", "d", "e", "f"));
    assertFilterMatches(toInFilterWithFn("dim1", lookupFn, "HELLO"), ImmutableList.of("b", "e"));
    assertFilterMatches(toInFilterWithFn("dim1", lookupFn, "N/A"), ImmutableList.of());
    assertFilterMatches(toInFilterWithFn("dim2", lookupFn, "a"), ImmutableList.of());
    assertFilterMatches(toInFilterWithFn("dim2", lookupFn, "HELLO"), ImmutableList.of("a", "d"));
    assertFilterMatches(toInFilterWithFn("dim2", lookupFn, "HELLO", "BYE", "UNKNOWN"), ImmutableList.of("a", "b", "c", "d", "e", "f"));
    final Map<String, String> stringMap2 = ImmutableMap.of("a", "e");
    LookupExtractor mapExtractor2 = new MapLookupExtractor(stringMap2, false);
    LookupExtractionFn lookupFn2 = new LookupExtractionFn(mapExtractor2, true, null, false, true);
    assertFilterMatches(toInFilterWithFn("dim0", lookupFn2, null, "e"), ImmutableList.of("a", "e"));
    assertFilterMatches(toInFilterWithFn("dim0", lookupFn2, "a"), ImmutableList.of());
    final Map<String, String> stringMap3 = ImmutableMap.of("c", "500", "100", "e");
    LookupExtractor mapExtractor3 = new MapLookupExtractor(stringMap3, false);
    LookupExtractionFn lookupFn3 = new LookupExtractionFn(mapExtractor3, false, null, false, true);
    assertFilterMatches(toInFilterWithFn("dim0", lookupFn3, null, "c"), ImmutableList.of("a", "b", "d", "e", "f"));
    assertFilterMatches(toInFilterWithFn("dim0", lookupFn3, "e"), ImmutableList.of());
}