Examples with BoundFilter org.apache.druid.segment.filter.BoundFilter used on opensource projects

Example 1 with BoundFilter

use of org.apache.druid.segment.filter.BoundFilter in project druid by druid-io.

the class JoinFilterAnalyzerTest method test_filterPushDown_factToRegionToCountryLeftEnablePushDownDisableRewrite.

@Test
public void test_filterPushDown_factToRegionToCountryLeftEnablePushDownDisableRewrite() {
    JoinableClauses joinableClauses = JoinableClauses.fromList(ImmutableList.of(factToRegion(JoinType.LEFT), regionToCountry(JoinType.LEFT)));
    Filter originalFilter = new AndFilter(ImmutableList.of(new SelectorFilter("channel", "#en.wikipedia"), new SelectorFilter("rtc.countryName", "United States"), new OrFilter(ImmutableList.of(new SelectorFilter("page", "DirecTV"), new SelectorFilter("rtc.countryIsoCode", "US"))), new BoundFilter(new BoundDimFilter("namespace", "Main", "Main", false, false, null, null, null))));
    JoinFilterPreAnalysis joinFilterPreAnalysis = JoinFilterAnalyzer.computeJoinFilterPreAnalysis(new JoinFilterPreAnalysisKey(new JoinFilterRewriteConfig(true, false, true, QueryContexts.DEFAULT_ENABLE_REWRITE_JOIN_TO_FILTER, QueryContexts.DEFAULT_ENABLE_JOIN_FILTER_REWRITE_MAX_SIZE), joinableClauses.getJoinableClauses(), VirtualColumns.EMPTY, originalFilter));
    HashJoinSegmentStorageAdapter adapter = new HashJoinSegmentStorageAdapter(factSegment.asStorageAdapter(), joinableClauses.getJoinableClauses(), joinFilterPreAnalysis);
    JoinTestHelper.verifyCursors(adapter.makeCursors(originalFilter, Intervals.ETERNITY, VirtualColumns.EMPTY, Granularities.ALL, false, null), ImmutableList.of("page", FACT_TO_REGION_PREFIX + "regionName", REGION_TO_COUNTRY_PREFIX + "countryName"), ImmutableList.of(new Object[] { "President of India", "California", "United States" }, new Object[] { "Otjiwarongo Airport", "California", "United States" }, new Object[] { "DirecTV", "North Carolina", "United States" }, new Object[] { "Carlo Curti", "California", "United States" }, new Object[] { "Old Anatolian Turkish", "Virginia", "United States" }));
    JoinFilterSplit expectedFilterSplit = new JoinFilterSplit(new AndFilter(ImmutableList.of(new SelectorFilter("channel", "#en.wikipedia"), new BoundFilter(new BoundDimFilter("namespace", "Main", "Main", false, false, null, null, null)))), new AndFilter(ImmutableList.of(new SelectorFilter("rtc.countryName", "United States"), new OrFilter(ImmutableList.of(new SelectorFilter("page", "DirecTV"), new SelectorFilter("rtc.countryIsoCode", "US"))))), ImmutableSet.of());
    JoinFilterSplit actualFilterSplit = JoinFilterAnalyzer.splitFilter(joinFilterPreAnalysis);
    Assert.assertEquals(expectedFilterSplit, actualFilterSplit);
}

Example 2 with BoundFilter

use of org.apache.druid.segment.filter.BoundFilter in project druid by druid-io.

the class JoinFilterAnalyzerTest method test_filterPushDown_factToRegionToCountryLeftFilterOnRHSJoinConditionColumnsHelper.

private void test_filterPushDown_factToRegionToCountryLeftFilterOnRHSJoinConditionColumnsHelper(boolean hasLhsExpressionInJoinCondition) {
    Filter expressionFilter = new ExpressionDimFilter("\"rtc.countryIsoCode\" == 'CA'", ExprMacroTable.nil()).toFilter();
    Filter specialSelectorFilter = new SelectorFilter("rtc.countryIsoCode", "CA") {

        @Override
        public boolean supportsRequiredColumnRewrite() {
            return false;
        }
    };
    Filter originalFilter = new AndFilter(ImmutableList.of(new SelectorFilter("r1.regionIsoCode", "ON"), new SelectorFilter("rtc.countryIsoCode", "CA"), specialSelectorFilter, new BoundFilter(new BoundDimFilter("rtc.countryIsoCode", "CA", "CB", false, false, null, null, null)), expressionFilter, new InDimFilter("rtc.countryIsoCode", ImmutableSet.of("CA", "CA2", "CA3"), null, null).toFilter(), new OrFilter(ImmutableList.of(new SelectorFilter("channel", "#fr.wikipedia"), new SelectorFilter("rtc.countryIsoCode", "QQQ"), new BoundFilter(new BoundDimFilter("rtc.countryIsoCode", "YYY", "ZZZ", false, false, null, null, null)))), new OrFilter(ImmutableList.of(new SelectorFilter("namespace", "Main"), new SelectorFilter("rtc.countryIsoCode", "ABCDEF"), new SelectorFilter("rtc.countryName", "Canada"), new BoundFilter(new BoundDimFilter("rtc.countryIsoCode", "XYZXYZ", "XYZXYZ", false, false, null, null, null))))));
    JoinableClause factToRegionClause;
    if (hasLhsExpressionInJoinCondition) {
        factToRegionClause = new JoinableClause(FACT_TO_REGION_PREFIX, new IndexedTableJoinable(regionsTable), JoinType.LEFT, JoinConditionAnalysis.forExpression(StringUtils.format("\"%sregionIsoCode\" == upper(lower(regionIsoCode)) && \"%scountryIsoCode\" == upper(lower(countryIsoCode))", FACT_TO_REGION_PREFIX, FACT_TO_REGION_PREFIX), FACT_TO_REGION_PREFIX, ExprMacroTable.nil()));
    } else {
        factToRegionClause = factToRegion(JoinType.LEFT);
    }
    List<JoinableClause> joinableClauses = ImmutableList.of(factToRegionClause, regionToCountry(JoinType.LEFT));
    JoinFilterPreAnalysis joinFilterPreAnalysis = makeDefaultConfigPreAnalysis(originalFilter, joinableClauses, VirtualColumns.EMPTY);
    HashJoinSegmentStorageAdapter adapter = new HashJoinSegmentStorageAdapter(factSegment.asStorageAdapter(), joinableClauses, joinFilterPreAnalysis);
    String rewrittenCountryIsoCodeColumnName = hasLhsExpressionInJoinCondition ? "JOIN-FILTER-PUSHDOWN-VIRTUAL-COLUMN-1" : "countryIsoCode";
    String rewrittenRegionIsoCodeColumnName = hasLhsExpressionInJoinCondition ? "JOIN-FILTER-PUSHDOWN-VIRTUAL-COLUMN-0" : "regionIsoCode";
    Set<VirtualColumn> expectedVirtualColumns;
    if (hasLhsExpressionInJoinCondition) {
        expectedVirtualColumns = ImmutableSet.of(new ExpressionVirtualColumn(rewrittenRegionIsoCodeColumnName, "(upper [(lower [regionIsoCode])])", ColumnType.STRING, ExprMacroTable.nil()), new ExpressionVirtualColumn(rewrittenCountryIsoCodeColumnName, "(upper [(lower [countryIsoCode])])", ColumnType.STRING, ExprMacroTable.nil()));
    } else {
        expectedVirtualColumns = ImmutableSet.of();
    }
    JoinTestHelper.verifyCursors(adapter.makeCursors(originalFilter, Intervals.ETERNITY, VirtualColumns.EMPTY, Granularities.ALL, false, null), ImmutableList.of("page", FACT_TO_REGION_PREFIX + "regionName", REGION_TO_COUNTRY_PREFIX + "countryName"), ImmutableList.of(new Object[] { "Didier Leclair", "Ontario", "Canada" }));
    JoinFilterSplit expectedFilterSplit = new JoinFilterSplit(new AndFilter(ImmutableList.of(new SelectorFilter(rewrittenRegionIsoCodeColumnName, "ON"), new SelectorFilter(rewrittenCountryIsoCodeColumnName, "CA"), new InDimFilter(rewrittenCountryIsoCodeColumnName, ImmutableSet.of("CA"), null, null).toFilter(), new BoundFilter(new BoundDimFilter(rewrittenCountryIsoCodeColumnName, "CA", "CB", false, false, null, null, null)), new InDimFilter(rewrittenCountryIsoCodeColumnName, ImmutableSet.of("CA", "CA2", "CA3"), null, null).toFilter(), new OrFilter(ImmutableList.of(new SelectorFilter("channel", "#fr.wikipedia"), new SelectorFilter(rewrittenCountryIsoCodeColumnName, "QQQ"), new BoundFilter(new BoundDimFilter(rewrittenCountryIsoCodeColumnName, "YYY", "ZZZ", false, false, null, null, null)))), new OrFilter(ImmutableList.of(new SelectorFilter("namespace", "Main"), new SelectorFilter(rewrittenCountryIsoCodeColumnName, "ABCDEF"), new InDimFilter(rewrittenCountryIsoCodeColumnName, ImmutableSet.of("CA"), null, null).toFilter(), new BoundFilter(new BoundDimFilter(rewrittenCountryIsoCodeColumnName, "XYZXYZ", "XYZXYZ", false, false, null, null, null)))))), new AndFilter(ImmutableList.of(specialSelectorFilter, expressionFilter, new OrFilter(ImmutableList.of(new SelectorFilter("namespace", "Main"), new SelectorFilter("rtc.countryIsoCode", "ABCDEF"), new SelectorFilter("rtc.countryName", "Canada"), new BoundFilter(new BoundDimFilter("rtc.countryIsoCode", "XYZXYZ", "XYZXYZ", false, false, null, null, null)))))), expectedVirtualColumns);
    JoinFilterSplit actualFilterSplit = JoinFilterAnalyzer.splitFilter(joinFilterPreAnalysis);
    Assert.assertEquals(expectedFilterSplit, actualFilterSplit);
}

Example 3 with BoundFilter

use of org.apache.druid.segment.filter.BoundFilter in project druid by druid-io.

the class FilterPartitionBenchmark method setup.

@Setup
public void setup() throws IOException {
    log.info("SETUP CALLED AT " + System.currentTimeMillis());
    ComplexMetrics.registerSerde("hyperUnique", new HyperUniquesSerde());
    schemaInfo = GeneratorBasicSchemas.SCHEMA_MAP.get(schema);
    DataGenerator gen = new DataGenerator(schemaInfo.getColumnSchemas(), RNG_SEED, schemaInfo.getDataInterval(), rowsPerSegment);
    incIndex = makeIncIndex();
    for (int j = 0; j < rowsPerSegment; j++) {
        InputRow row = gen.nextRow();
        if (j % 10000 == 0) {
            log.info(j + " rows generated.");
        }
        incIndex.add(row);
    }
    tmpDir = FileUtils.createTempDir();
    log.info("Using temp dir: " + tmpDir.getAbsolutePath());
    indexFile = INDEX_MERGER_V9.persist(incIndex, tmpDir, new IndexSpec(), null);
    qIndex = INDEX_IO.loadIndex(indexFile);
    Interval interval = schemaInfo.getDataInterval();
    timeFilterNone = new BoundFilter(new BoundDimFilter(ColumnHolder.TIME_COLUMN_NAME, String.valueOf(Long.MAX_VALUE), String.valueOf(Long.MAX_VALUE), true, true, null, null, StringComparators.ALPHANUMERIC));
    long halfEnd = (interval.getEndMillis() + interval.getStartMillis()) / 2;
    timeFilterHalf = new BoundFilter(new BoundDimFilter(ColumnHolder.TIME_COLUMN_NAME, String.valueOf(interval.getStartMillis()), String.valueOf(halfEnd), true, true, null, null, StringComparators.ALPHANUMERIC));
    timeFilterAll = new BoundFilter(new BoundDimFilter(ColumnHolder.TIME_COLUMN_NAME, String.valueOf(interval.getStartMillis()), String.valueOf(interval.getEndMillis()), true, true, null, null, StringComparators.ALPHANUMERIC));
}

Example 4 with BoundFilter

use of org.apache.druid.segment.filter.BoundFilter in project druid by druid-io.

the class JoinFilterAnalyzerTest method test_filterPushDown_factToRegionToCountryLeftFilterNormalizedAlreadyPushDownVariety.

@Test
public void test_filterPushDown_factToRegionToCountryLeftFilterNormalizedAlreadyPushDownVariety() {
    Filter originalFilter = new AndFilter(ImmutableList.of(new SelectorFilter("channel", "#fr.wikipedia"), new BoundFilter(new BoundDimFilter("page", "Les Argonautes", "Les Argonautes", false, false, null, null, null)), new SelectorFilter("rtc.countryName", "Canada"), new BoundFilter(new BoundDimFilter("rtc.countryName", "Canada", "Canada", false, false, null, null, null)), new OrFilter(ImmutableList.of(new SelectorFilter("namespace", "main"), new BoundFilter(new BoundDimFilter("user", "24.122.168.111", "24.122.168.111", false, false, null, null, null)))), new OrFilter(ImmutableList.of(new SelectorFilter("namespace", "main"), new BoundFilter(new BoundDimFilter("r1.regionName", "Quebec", "Quebec", false, false, null, null, null))))));
    List<JoinableClause> joinableClauses = ImmutableList.of(factToRegion(JoinType.LEFT), regionToCountry(JoinType.LEFT));
    JoinFilterPreAnalysis joinFilterPreAnalysis = makeDefaultConfigPreAnalysis(originalFilter, joinableClauses, VirtualColumns.EMPTY);
    HashJoinSegmentStorageAdapter adapter = new HashJoinSegmentStorageAdapter(factSegment.asStorageAdapter(), joinableClauses, joinFilterPreAnalysis);
    JoinTestHelper.verifyCursors(adapter.makeCursors(originalFilter, Intervals.ETERNITY, VirtualColumns.EMPTY, Granularities.ALL, false, null), ImmutableList.of("page", FACT_TO_REGION_PREFIX + "regionName", REGION_TO_COUNTRY_PREFIX + "countryName"), ImmutableList.of(new Object[] { "Les Argonautes", "Quebec", "Canada" }));
    JoinFilterSplit expectedFilterSplit = new JoinFilterSplit(new AndFilter(ImmutableList.of(new SelectorFilter("channel", "#fr.wikipedia"), new BoundFilter(new BoundDimFilter("page", "Les Argonautes", "Les Argonautes", false, false, null, null, null)), new OrFilter(ImmutableList.of(new SelectorFilter("namespace", "main"), new BoundFilter(new BoundDimFilter("user", "24.122.168.111", "24.122.168.111", false, false, null, null, null)))), new InDimFilter("countryIsoCode", ImmutableSet.of("CA"), null, null).toFilter())), new AndFilter(ImmutableList.of(new SelectorFilter("rtc.countryName", "Canada"), new BoundFilter(new BoundDimFilter("rtc.countryName", "Canada", "Canada", false, false, null, null, null)), new OrFilter(ImmutableList.of(new SelectorFilter("namespace", "main"), new BoundFilter(new BoundDimFilter("r1.regionName", "Quebec", "Quebec", false, false, null, null, null)))))), ImmutableSet.of());
    JoinFilterSplit actualFilterSplit = JoinFilterAnalyzer.splitFilter(joinFilterPreAnalysis);
    Assert.assertEquals(expectedFilterSplit, actualFilterSplit);
}