use of org.apache.druid.segment.join.filter.rewrite.JoinFilterRewriteConfig in project druid by druid-io.
the class JoinFilterAnalyzerTest method test_filterPushDown_factToRegionToCountryLeftEnablePushDownDisableRewrite.
@Test
public void test_filterPushDown_factToRegionToCountryLeftEnablePushDownDisableRewrite() {
JoinableClauses joinableClauses = JoinableClauses.fromList(ImmutableList.of(factToRegion(JoinType.LEFT), regionToCountry(JoinType.LEFT)));
Filter originalFilter = new AndFilter(ImmutableList.of(new SelectorFilter("channel", "#en.wikipedia"), new SelectorFilter("rtc.countryName", "United States"), new OrFilter(ImmutableList.of(new SelectorFilter("page", "DirecTV"), new SelectorFilter("rtc.countryIsoCode", "US"))), new BoundFilter(new BoundDimFilter("namespace", "Main", "Main", false, false, null, null, null))));
JoinFilterPreAnalysis joinFilterPreAnalysis = JoinFilterAnalyzer.computeJoinFilterPreAnalysis(new JoinFilterPreAnalysisKey(new JoinFilterRewriteConfig(true, false, true, QueryContexts.DEFAULT_ENABLE_REWRITE_JOIN_TO_FILTER, QueryContexts.DEFAULT_ENABLE_JOIN_FILTER_REWRITE_MAX_SIZE), joinableClauses.getJoinableClauses(), VirtualColumns.EMPTY, originalFilter));
HashJoinSegmentStorageAdapter adapter = new HashJoinSegmentStorageAdapter(factSegment.asStorageAdapter(), joinableClauses.getJoinableClauses(), joinFilterPreAnalysis);
JoinTestHelper.verifyCursors(adapter.makeCursors(originalFilter, Intervals.ETERNITY, VirtualColumns.EMPTY, Granularities.ALL, false, null), ImmutableList.of("page", FACT_TO_REGION_PREFIX + "regionName", REGION_TO_COUNTRY_PREFIX + "countryName"), ImmutableList.of(new Object[] { "President of India", "California", "United States" }, new Object[] { "Otjiwarongo Airport", "California", "United States" }, new Object[] { "DirecTV", "North Carolina", "United States" }, new Object[] { "Carlo Curti", "California", "United States" }, new Object[] { "Old Anatolian Turkish", "Virginia", "United States" }));
JoinFilterSplit expectedFilterSplit = new JoinFilterSplit(new AndFilter(ImmutableList.of(new SelectorFilter("channel", "#en.wikipedia"), new BoundFilter(new BoundDimFilter("namespace", "Main", "Main", false, false, null, null, null)))), new AndFilter(ImmutableList.of(new SelectorFilter("rtc.countryName", "United States"), new OrFilter(ImmutableList.of(new SelectorFilter("page", "DirecTV"), new SelectorFilter("rtc.countryIsoCode", "US"))))), ImmutableSet.of());
JoinFilterSplit actualFilterSplit = JoinFilterAnalyzer.splitFilter(joinFilterPreAnalysis);
Assert.assertEquals(expectedFilterSplit, actualFilterSplit);
}
use of org.apache.druid.segment.join.filter.rewrite.JoinFilterRewriteConfig in project druid by druid-io.
the class JoinFilterAnalyzerTest method test_filterPushDown_factToRegionExprToCountryLeftFilterOnCountryNameWithMultiLevelMode.
@Test
public void test_filterPushDown_factToRegionExprToCountryLeftFilterOnCountryNameWithMultiLevelMode() {
Filter originalFilter = new SelectorFilter("rtc.countryName", "United States");
JoinableClause regionExprToCountry = new JoinableClause(REGION_TO_COUNTRY_PREFIX, new IndexedTableJoinable(countriesTable), JoinType.LEFT, JoinConditionAnalysis.forExpression(StringUtils.format("reverse(\"%scountryIsoCode\") == \"%scountryIsoCode\"", FACT_TO_REGION_PREFIX, REGION_TO_COUNTRY_PREFIX), REGION_TO_COUNTRY_PREFIX, ExprMacroTable.nil()));
List<JoinableClause> joinableClauses = ImmutableList.of(factToRegion(JoinType.LEFT), regionExprToCountry);
JoinFilterPreAnalysis joinFilterPreAnalysis = JoinFilterAnalyzer.computeJoinFilterPreAnalysis(new JoinFilterPreAnalysisKey(new JoinFilterRewriteConfig(true, true, true, QueryContexts.DEFAULT_ENABLE_REWRITE_JOIN_TO_FILTER, QueryContexts.DEFAULT_ENABLE_JOIN_FILTER_REWRITE_MAX_SIZE), joinableClauses, VirtualColumns.EMPTY, originalFilter));
HashJoinSegmentStorageAdapter adapter = new HashJoinSegmentStorageAdapter(factSegment.asStorageAdapter(), joinableClauses, joinFilterPreAnalysis);
JoinTestHelper.verifyCursors(adapter.makeCursors(originalFilter, Intervals.ETERNITY, VirtualColumns.EMPTY, Granularities.ALL, false, null), ImmutableList.of("page", FACT_TO_REGION_PREFIX + "regionName", REGION_TO_COUNTRY_PREFIX + "countryName"), ImmutableList.of(new Object[] { "Cream Soda", "Ainigriv", "United States" }));
JoinFilterSplit expectedFilterSplit = new JoinFilterSplit(null, new SelectorFilter("rtc.countryName", "United States"), ImmutableSet.of());
JoinFilterSplit actualFilterSplit = JoinFilterAnalyzer.splitFilter(joinFilterPreAnalysis);
Assert.assertEquals(expectedFilterSplit, actualFilterSplit);
}
use of org.apache.druid.segment.join.filter.rewrite.JoinFilterRewriteConfig in project druid by druid-io.
the class JoinFilterAnalyzerTest method test_filterPushDown_factToRegionToCountryLeftFilterOnPageDisablePushDown.
@Test
public void test_filterPushDown_factToRegionToCountryLeftFilterOnPageDisablePushDown() {
JoinableClauses joinableClauses = JoinableClauses.fromList(ImmutableList.of(factToRegion(JoinType.LEFT), regionToCountry(JoinType.LEFT)));
Filter originalFilter = new SelectorFilter("page", "Peremptory norm");
JoinFilterPreAnalysis joinFilterPreAnalysis = JoinFilterAnalyzer.computeJoinFilterPreAnalysis(new JoinFilterPreAnalysisKey(new JoinFilterRewriteConfig(false, true, true, QueryContexts.DEFAULT_ENABLE_REWRITE_JOIN_TO_FILTER, QueryContexts.DEFAULT_ENABLE_JOIN_FILTER_REWRITE_MAX_SIZE), joinableClauses.getJoinableClauses(), VirtualColumns.EMPTY, originalFilter));
HashJoinSegmentStorageAdapter adapter = new HashJoinSegmentStorageAdapter(factSegment.asStorageAdapter(), joinableClauses.getJoinableClauses(), joinFilterPreAnalysis);
JoinTestHelper.verifyCursors(adapter.makeCursors(originalFilter, Intervals.ETERNITY, VirtualColumns.EMPTY, Granularities.ALL, false, null), ImmutableList.of("page", FACT_TO_REGION_PREFIX + "regionName", REGION_TO_COUNTRY_PREFIX + "countryName"), ImmutableList.of(new Object[] { "Peremptory norm", "New South Wales", "Australia" }));
JoinFilterSplit expectedFilterSplit = new JoinFilterSplit(null, new SelectorFilter("page", "Peremptory norm"), ImmutableSet.of());
JoinFilterSplit actualFilterSplit = JoinFilterAnalyzer.splitFilter(joinFilterPreAnalysis);
Assert.assertEquals(expectedFilterSplit, actualFilterSplit);
}
use of org.apache.druid.segment.join.filter.rewrite.JoinFilterRewriteConfig in project druid by druid-io.
the class JoinAndLookupBenchmark method setup.
@Setup()
public void setup() throws IOException {
tmpDir = FileUtils.createTempDir();
ColumnConfig columnConfig = () -> columnCacheSizeBytes;
index = JoinTestHelper.createFactIndexBuilder(columnConfig, tmpDir, rows).buildMMappedIndex();
final String prefix = "c.";
baseSegment = new QueryableIndexSegment(index, SegmentId.dummy("join"));
List<JoinableClause> joinableClausesLookupStringKey = ImmutableList.of(new JoinableClause(prefix, LookupJoinable.wrap(JoinTestHelper.createCountryIsoCodeToNameLookup()), JoinType.LEFT, JoinConditionAnalysis.forExpression(StringUtils.format("countryIsoCode == \"%sk\"", prefix), prefix, ExprMacroTable.nil())));
JoinFilterPreAnalysis preAnalysisLookupStringKey = JoinFilterAnalyzer.computeJoinFilterPreAnalysis(new JoinFilterPreAnalysisKey(new JoinFilterRewriteConfig(false, false, false, false, 0), joinableClausesLookupStringKey, VirtualColumns.EMPTY, null));
hashJoinLookupStringKeySegment = new HashJoinSegment(ReferenceCountingSegment.wrapRootGenerationSegment(baseSegment), null, joinableClausesLookupStringKey, preAnalysisLookupStringKey);
List<JoinableClause> joinableClausesLookupLongKey = ImmutableList.of(new JoinableClause(prefix, LookupJoinable.wrap(JoinTestHelper.createCountryIsoCodeToNameLookup()), JoinType.LEFT, JoinConditionAnalysis.forExpression(StringUtils.format("countryIsoCode == \"%sk\"", prefix), prefix, ExprMacroTable.nil())));
JoinFilterPreAnalysis preAnalysisLookupLongKey = JoinFilterAnalyzer.computeJoinFilterPreAnalysis(new JoinFilterPreAnalysisKey(new JoinFilterRewriteConfig(false, false, false, false, 0), joinableClausesLookupLongKey, VirtualColumns.EMPTY, null));
hashJoinLookupLongKeySegment = new HashJoinSegment(ReferenceCountingSegment.wrapRootGenerationSegment(baseSegment), null, joinableClausesLookupLongKey, preAnalysisLookupLongKey);
List<JoinableClause> joinableClausesIndexedTableStringKey = ImmutableList.of(new JoinableClause(prefix, new IndexedTableJoinable(JoinTestHelper.createCountriesIndexedTable()), JoinType.LEFT, JoinConditionAnalysis.forExpression(StringUtils.format("countryIsoCode == \"%scountryIsoCode\"", prefix), prefix, ExprMacroTable.nil())));
JoinFilterPreAnalysis preAnalysisIndexedStringKey = JoinFilterAnalyzer.computeJoinFilterPreAnalysis(new JoinFilterPreAnalysisKey(new JoinFilterRewriteConfig(false, false, false, false, 0), joinableClausesLookupLongKey, VirtualColumns.EMPTY, null));
hashJoinIndexedTableStringKeySegment = new HashJoinSegment(ReferenceCountingSegment.wrapRootGenerationSegment(baseSegment), null, joinableClausesIndexedTableStringKey, preAnalysisIndexedStringKey);
List<JoinableClause> joinableClausesIndexedTableLongKey = ImmutableList.of(new JoinableClause(prefix, new IndexedTableJoinable(JoinTestHelper.createCountriesIndexedTable()), JoinType.LEFT, JoinConditionAnalysis.forExpression(StringUtils.format("countryNumber == \"%scountryNumber\"", prefix), prefix, ExprMacroTable.nil())));
JoinFilterPreAnalysis preAnalysisIndexedLongKey = JoinFilterAnalyzer.computeJoinFilterPreAnalysis(new JoinFilterPreAnalysisKey(new JoinFilterRewriteConfig(false, false, false, false, 0), joinableClausesIndexedTableLongKey, VirtualColumns.EMPTY, null));
hashJoinIndexedTableLongKeySegment = new HashJoinSegment(ReferenceCountingSegment.wrapRootGenerationSegment(baseSegment), null, joinableClausesIndexedTableLongKey, preAnalysisIndexedLongKey);
final Map<String, String> countryCodeToNameMap = JoinTestHelper.createCountryIsoCodeToNameLookup().getMap();
final Map<String, String> countryNumberToNameMap = JoinTestHelper.createCountryNumberToNameLookup().getMap();
final ExprMacroTable exprMacroTable = new ExprMacroTable(ImmutableList.of(new LookupExprMacro(new LookupExtractorFactoryContainerProvider() {
@Override
public Set<String> getAllLookupNames() {
return ImmutableSet.of(LOOKUP_COUNTRY_CODE_TO_NAME, LOOKUP_COUNTRY_NUMBER_TO_NAME);
}
@Override
public Optional<LookupExtractorFactoryContainer> get(String lookupName) {
if (LOOKUP_COUNTRY_CODE_TO_NAME.equals(lookupName)) {
return Optional.of(new LookupExtractorFactoryContainer("0", new MapLookupExtractorFactory(countryCodeToNameMap, false)));
} else if (LOOKUP_COUNTRY_NUMBER_TO_NAME.equals(lookupName)) {
return Optional.of(new LookupExtractorFactoryContainer("0", new MapLookupExtractorFactory(countryNumberToNameMap, false)));
} else {
return Optional.empty();
}
}
})));
lookupVirtualColumns = VirtualColumns.create(ImmutableList.of(new ExpressionVirtualColumn(LOOKUP_COUNTRY_CODE_TO_NAME, "lookup(countryIsoCode, '" + LOOKUP_COUNTRY_CODE_TO_NAME + "')", ColumnType.STRING, exprMacroTable), new ExpressionVirtualColumn(LOOKUP_COUNTRY_NUMBER_TO_NAME, "lookup(countryNumber, '" + LOOKUP_COUNTRY_NUMBER_TO_NAME + "')", ColumnType.STRING, exprMacroTable)));
}
use of org.apache.druid.segment.join.filter.rewrite.JoinFilterRewriteConfig in project druid by druid-io.
the class IndexedTableJoinCursorBenchmark method setup.
@Setup(Level.Trial)
public void setup() {
baseSegment = makeQueryableIndexSegment(closer, "regular", rowsPerSegment);
joinSegment = makeQueryableIndexSegment(closer, "join", rowsPerTableSegment);
table = closer.register(makeTable(indexedTableType, keyColumns, joinSegment));
final String prefix = "j0.";
projectionColumns = PROJECTIONS.get(projection);
final String[] split = joinColumns.split(",");
final String lhsJoinColumn = split[0];
final String rhsJoinColumn = split[1];
final List<JoinableClause> clauses = ImmutableList.of(new JoinableClause(prefix, new IndexedTableJoinable(table), JoinType.LEFT, JoinConditionAnalysis.forExpression(StringUtils.format("%s == \"%s%s\"", lhsJoinColumn, prefix, rhsJoinColumn), prefix, ExprMacroTable.nil())));
final JoinFilterPreAnalysis preAnalysis = JoinFilterAnalyzer.computeJoinFilterPreAnalysis(new JoinFilterPreAnalysisKey(new JoinFilterRewriteConfig(enableFilterPushdown, enableFilterRewrite, enableFilterRewriteValueFilters, QueryContexts.DEFAULT_ENABLE_REWRITE_JOIN_TO_FILTER, QueryContexts.DEFAULT_ENABLE_JOIN_FILTER_REWRITE_MAX_SIZE), clauses, VirtualColumns.EMPTY, null));
hashJoinSegment = closer.register(new HashJoinSegment(ReferenceCountingSegment.wrapRootGenerationSegment(baseSegment), null, clauses, preAnalysis));
}
Aggregations