use of org.apache.calcite.rel.RelRoot in project hazelcast by hazelcast.
the class QueryConverter method convert.
public QueryConvertResult convert(SqlNode node) {
SqlToRelConverter converter = new HazelcastSqlToRelConverter(viewExpander, validator, catalogReader, cluster, StandardConvertletTable.INSTANCE, CONFIG);
// 1. Perform initial conversion.
RelRoot root = converter.convertQuery(node, false, true);
// 2. Remove subquery expressions, converting them to Correlate nodes.
RelNode relNoSubqueries = rewriteSubqueries(root.project());
// 3. Perform decorrelation, i.e. rewrite a nested loop where the right side depends on the value of the left side,
// to a variation of joins, semijoins and aggregations, which could be executed much more efficiently.
// See "Unnesting Arbitrary Queries", Thomas Neumann and Alfons Kemper.
RelNode result = converter.decorrelate(node, relNoSubqueries);
// The bug is likely in decorrelation which produces LogicalAggregate with 0 output columns.
if (!hasNestedExists(root.rel)) {
result = converter.trimUnusedFields(true, result);
}
// 5. Collect original field names.
return new QueryConvertResult(result, Pair.right(root.fields));
}
use of org.apache.calcite.rel.RelRoot in project hazelcast by hazelcast.
the class HazelcastViewExpander method expandView.
@Override
public RelRoot expandView(RelDataType rowType, String queryString, List<String> schemaPath, @Nullable List<String> viewPath) {
if (expansionStack.contains(viewPath)) {
throw QueryException.error("Cycle detected in view references");
}
expansionStack.push(viewPath);
SqlNode sqlNode = parser.parse(queryString).getNode();
final RelRoot root = sqlToRelConverter.convertQuery(sqlNode, true, true);
expansionStack.pop();
final RelRoot root2 = root.withRel(sqlToRelConverter.flattenTypes(root.rel, true));
final RelBuilder relBuilder = QueryConverter.CONFIG.getRelBuilderFactory().create(relOptCluster, null);
return root2.withRel(RelDecorrelator.decorrelateQuery(root.rel, relBuilder));
}
use of org.apache.calcite.rel.RelRoot in project samza by apache.
the class TestQueryPlanner method testRemoteJoinNoFilterPushDownWithUdfInFilterAndOptimizer.
@Test
public void testRemoteJoinNoFilterPushDownWithUdfInFilterAndOptimizer() throws SamzaSqlValidatorException {
Map<String, String> staticConfigs = SamzaSqlTestConfig.fetchStaticConfigsWithFactories(1);
String sql = "Insert into testavro.enrichedPageViewTopic " + "select pv.pageKey as __key__, pv.pageKey as pageKey, coalesce(null, 'N/A') as companyName," + " p.name as profileName, p.address as profileAddress " + "from testRemoteStore.Profile.`$table` as p " + "join testavro.PAGEVIEW as pv " + " on p.__key__ = pv.profileId" + " where p.name = pv.pageKey AND p.name = 'Mike' AND pv.profileId = MyTestPoly(p.name)";
staticConfigs.put(SamzaSqlApplicationConfig.CFG_SQL_STMT, sql);
staticConfigs.put(SamzaSqlApplicationConfig.CFG_SQL_ENABLE_PLAN_OPTIMIZER, Boolean.toString(true));
Config samzaConfig = new MapConfig(staticConfigs);
DslConverter dslConverter = new SamzaSqlDslConverterFactory().create(samzaConfig);
Collection<RelRoot> relRootsWithOptimization = dslConverter.convertDsl(sql);
staticConfigs.put(SamzaSqlApplicationConfig.CFG_SQL_ENABLE_PLAN_OPTIMIZER, Boolean.toString(false));
samzaConfig = new MapConfig(staticConfigs);
dslConverter = new SamzaSqlDslConverterFactory().create(samzaConfig);
Collection<RelRoot> relRootsWithoutOptimization = dslConverter.convertDsl(sql);
/*
LogicalProject(__key__=[$9], pageKey=[$9], companyName=['N/A'], profileName=[$2], profileAddress=[$4])
LogicalFilter(condition=[AND(=($2, $9), =($2, 'Mike'), =($10, CAST(MyTestPoly($10)):INTEGER))])
LogicalJoin(condition=[=($0, $10)], joinType=[inner])
LogicalTableScan(table=[[testRemoteStore, Profile, $table]])
LogicalTableScan(table=[[testavro, PAGEVIEW]])
*/
// None of the conditions in the filter could be pushed down as they all require a remote call. Hence the plans
// with and without optimization should be the same.
assertEquals(RelOptUtil.toString(relRootsWithOptimization.iterator().next().rel, SqlExplainLevel.EXPPLAN_ATTRIBUTES), RelOptUtil.toString(relRootsWithoutOptimization.iterator().next().rel, SqlExplainLevel.EXPPLAN_ATTRIBUTES));
}
use of org.apache.calcite.rel.RelRoot in project samza by apache.
the class TestQueryPlanner method testRemoteJoinWithFilterHelper.
void testRemoteJoinWithFilterHelper(boolean enableOptimizer) throws SamzaSqlValidatorException {
Map<String, String> staticConfigs = SamzaSqlTestConfig.fetchStaticConfigsWithFactories(1);
String sql = "Insert into testavro.enrichedPageViewTopic " + "select pv.pageKey as __key__, pv.pageKey as pageKey, coalesce(null, 'N/A') as companyName," + " p.name as profileName, p.address as profileAddress " + "from testavro.PAGEVIEW as pv " + "join testRemoteStore.Profile.`$table` as p " + " on p.__key__ = pv.profileId" + " where p.name = pv.pageKey AND p.name = 'Mike' AND pv.profileId = 1";
staticConfigs.put(SamzaSqlApplicationConfig.CFG_SQL_STMT, sql);
staticConfigs.put(SamzaSqlApplicationConfig.CFG_SQL_ENABLE_PLAN_OPTIMIZER, Boolean.toString(enableOptimizer));
Config samzaConfig = new MapConfig(staticConfigs);
DslConverter dslConverter = new SamzaSqlDslConverterFactory().create(samzaConfig);
Collection<RelRoot> relRoots = dslConverter.convertDsl(sql);
/*
Query plan without optimization:
LogicalProject(__key__=[$1], pageKey=[$1], companyName=['N/A'], profileName=[$5], profileAddress=[$7])
LogicalFilter(condition=[AND(=($5, $1), =($5, 'Mike'), =($2, 1))])
LogicalJoin(condition=[=($3, $2)], joinType=[inner])
LogicalTableScan(table=[[testavro, PAGEVIEW]])
LogicalTableScan(table=[[testRemoteStore, Profile, $table]])
Query plan with optimization:
LogicalProject(__key__=[$1], pageKey=[$1], companyName=['N/A'], profileName=[$5], profileAddress=[$7])
LogicalFilter(condition=[AND(=($5, $1), =($5, 'Mike'))])
LogicalJoin(condition=[=($3, $2)], joinType=[inner])
LogicalFilter(condition=[=($2, 1)])
LogicalTableScan(table=[[testavro, PAGEVIEW]])
LogicalTableScan(table=[[testRemoteStore, Profile, $table]])
*/
assertEquals(1, relRoots.size());
RelRoot relRoot = relRoots.iterator().next();
RelNode relNode = relRoot.rel;
assertTrue(relNode instanceof LogicalProject);
relNode = relNode.getInput(0);
assertTrue(relNode instanceof LogicalFilter);
if (enableOptimizer) {
assertEquals("AND(=($1, $5), =($5, 'Mike'))", ((LogicalFilter) relNode).getCondition().toString());
} else {
assertEquals("AND(=(1, $2), =($1, $5), =($5, 'Mike'))", ((LogicalFilter) relNode).getCondition().toString());
}
relNode = relNode.getInput(0);
assertTrue(relNode instanceof LogicalJoin);
assertEquals(2, relNode.getInputs().size());
LogicalJoin join = (LogicalJoin) relNode;
RelNode left = join.getLeft();
RelNode right = join.getRight();
assertTrue(right instanceof LogicalTableScan);
if (enableOptimizer) {
assertTrue(left instanceof LogicalFilter);
assertEquals("=(1, $2)", ((LogicalFilter) left).getCondition().toString());
assertTrue(left.getInput(0) instanceof LogicalTableScan);
} else {
assertTrue(left instanceof LogicalTableScan);
}
}
use of org.apache.calcite.rel.RelRoot in project samza by apache.
the class SamzaSqlApplication method describe.
@Override
public void describe(StreamApplicationDescriptor appDescriptor) {
try {
// TODO: Introduce an API to return a dsl string containing one or more sql statements.
List<String> dslStmts = SamzaSqlDslConverter.fetchSqlFromConfig(appDescriptor.getConfig());
Map<Integer, TranslatorContext> translatorContextMap = new HashMap<>();
// 1. Get Calcite plan
List<String> inputSystemStreams = new LinkedList<>();
List<String> outputSystemStreams = new LinkedList<>();
Collection<RelRoot> relRoots = SamzaSqlApplicationConfig.populateSystemStreamsAndGetRelRoots(dslStmts, appDescriptor.getConfig(), inputSystemStreams, outputSystemStreams);
// 2. Populate configs
SamzaSqlApplicationConfig sqlConfig = new SamzaSqlApplicationConfig(appDescriptor.getConfig(), inputSystemStreams, outputSystemStreams);
// 3. Translate Calcite plan to Samza stream operators
QueryTranslator queryTranslator = new QueryTranslator(appDescriptor, sqlConfig);
SamzaSqlExecutionContext executionContext = new SamzaSqlExecutionContext(sqlConfig);
// QueryId implies the index of the query in multiple query statements scenario. It should always start with 0.
int queryId = 0;
for (RelRoot relRoot : relRoots) {
LOG.info("Translating relRoot {} to samza stream graph with queryId {}", relRoot, queryId);
TranslatorContext translatorContext = new TranslatorContext(appDescriptor, relRoot, executionContext);
translatorContextMap.put(queryId, translatorContext);
queryTranslator.translate(relRoot, sqlConfig.getOutputSystemStreams().get(queryId), translatorContext, queryId);
queryId++;
}
// 4. Set all translator contexts
/*
* TODO When serialization of ApplicationDescriptor is actually needed, then something will need to be updated here,
* since translatorContext is not Serializable. Currently, a new ApplicationDescriptor instance is created in each
* container, so it does not need to be serialized. Therefore, the translatorContext is recreated in each container
* and does not need to be serialized.
*/
appDescriptor.withApplicationTaskContextFactory(new ApplicationTaskContextFactory<SamzaSqlApplicationContext>() {
@Override
public SamzaSqlApplicationContext create(ExternalContext externalContext, JobContext jobContext, ContainerContext containerContext, TaskContext taskContext, ApplicationContainerContext applicationContainerContext) {
return new SamzaSqlApplicationContext(translatorContextMap);
}
});
} catch (RuntimeException e) {
LOG.error("SamzaSqlApplication threw exception.", e);
throw e;
}
}
Aggregations