Search in sources :

Example 91 with Table

use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.schema.Table in project beam by apache.

the class ThreeTablesSchema method testSystemNotReorderingWithoutRules.

@Test
public void testSystemNotReorderingWithoutRules() {
    TestTableProvider tableProvider = new TestTableProvider();
    createThreeTables(tableProvider);
    List<RelOptRule> ruleSet = BeamRuleSets.getRuleSets().stream().flatMap(rules -> StreamSupport.stream(rules.spliterator(), false)).filter(rule -> !(rule instanceof BeamJoinPushThroughJoinRule)).filter(rule -> !(rule instanceof BeamJoinAssociateRule)).filter(rule -> !(rule instanceof JoinCommuteRule)).collect(Collectors.toList());
    BeamSqlEnv env = BeamSqlEnv.builder(tableProvider).setPipelineOptions(PipelineOptionsFactory.create()).setRuleSets(ImmutableList.of(RuleSets.ofList(ruleSet))).build();
    // This is Join(Join(medium, large), small) which should be converted to a join that large table
    // is on the top.
    BeamRelNode parsedQuery = env.parseQuery("select * from medium_table " + " JOIN large_table on large_table.medium_key = medium_table.large_key " + " JOIN small_table on medium_table.small_key = small_table.medium_key ");
    assertTopTableInJoins(parsedQuery, "small_table");
}
Also used : TestTableProvider(org.apache.beam.sdk.extensions.sql.meta.provider.test.TestTableProvider) Linq4j(org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.linq4j.Linq4j) RelNode(org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rel.RelNode) Arrays(java.util.Arrays) AbstractTable(org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.schema.impl.AbstractTable) Enumerable(org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.linq4j.Enumerable) BeamRelNode(org.apache.beam.sdk.extensions.sql.impl.rel.BeamRelNode) BeamRuleSets(org.apache.beam.sdk.extensions.sql.impl.planner.BeamRuleSets) Frameworks(org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.tools.Frameworks) RelFieldCollation(org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rel.RelFieldCollation) ImmutableBitSet(org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.util.ImmutableBitSet) Planner(org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.tools.Planner) Map(java.util.Map) EnumerableConvention(org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.adapter.enumerable.EnumerableConvention) JoinCommuteRule(org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rel.rules.JoinCommuteRule) RelRoot(org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rel.RelRoot) ImmutableMap(org.apache.beam.vendor.calcite.v1_28_0.com.google.common.collect.ImmutableMap) TestTableProvider(org.apache.beam.sdk.extensions.sql.meta.provider.test.TestTableProvider) Collectors(java.util.stream.Collectors) Table(org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.schema.Table) TableScan(org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rel.core.TableScan) Statistic(org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.schema.Statistic) List(java.util.List) ImmutableList(org.apache.beam.vendor.calcite.v1_28_0.com.google.common.collect.ImmutableList) RelCollationTraitDef(org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rel.RelCollationTraitDef) AbstractSchema(org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.schema.impl.AbstractSchema) RuleSets(org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.tools.RuleSets) RuleSet(org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.tools.RuleSet) Programs(org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.tools.Programs) RelDataTypeFactory(org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rel.type.RelDataTypeFactory) FrameworkConfig(org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.tools.FrameworkConfig) RelCollations(org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rel.RelCollations) PipelineOptionsFactory(org.apache.beam.sdk.options.PipelineOptionsFactory) Function(java.util.function.Function) ArrayList(java.util.ArrayList) Statistics(org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.schema.Statistics) RelOptRule(org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.plan.RelOptRule) Join(org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rel.core.Join) ScannableTable(org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.schema.ScannableTable) StreamSupport(java.util.stream.StreamSupport) BeamSqlEnv(org.apache.beam.sdk.extensions.sql.impl.BeamSqlEnv) Row(org.apache.beam.sdk.values.Row) SqlNode(org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.sql.SqlNode) SqlParser(org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.sql.parser.SqlParser) EnumerableRules(org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.adapter.enumerable.EnumerableRules) Test(org.junit.Test) ConventionTraitDef(org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.plan.ConventionTraitDef) RelDataType(org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rel.type.RelDataType) RelTraitSet(org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.plan.RelTraitSet) DataContext(org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.DataContext) SchemaPlus(org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.schema.SchemaPlus) Assert(org.junit.Assert) CoreRules(org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rel.rules.CoreRules) BeamRelNode(org.apache.beam.sdk.extensions.sql.impl.rel.BeamRelNode) BeamSqlEnv(org.apache.beam.sdk.extensions.sql.impl.BeamSqlEnv) JoinCommuteRule(org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rel.rules.JoinCommuteRule) RelOptRule(org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.plan.RelOptRule) Test(org.junit.Test)

Example 92 with Table

use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.schema.Table in project beam by apache.

the class BeamSideInputJoinRel method sideInputJoin.

public PCollection<Row> sideInputJoin(PCollection<Row> leftRows, PCollection<Row> rightRows, FieldAccessDescriptor leftKeyFields, FieldAccessDescriptor rightKeyFields) {
    // we always make the Unbounded table on the left to do the sideInput join
    // (will convert the result accordingly before return)
    boolean swapped = (leftRows.isBounded() == PCollection.IsBounded.BOUNDED);
    JoinRelType realJoinType = joinType;
    if (swapped && joinType != JoinRelType.INNER) {
        Preconditions.checkArgument(realJoinType != JoinRelType.LEFT);
        realJoinType = JoinRelType.LEFT;
    }
    PCollection<Row> realLeftRows = swapped ? rightRows : leftRows;
    PCollection<Row> realRightRows = swapped ? leftRows : rightRows;
    FieldAccessDescriptor realLeftKeyFields = swapped ? rightKeyFields : leftKeyFields;
    FieldAccessDescriptor realRightKeyFields = swapped ? leftKeyFields : rightKeyFields;
    PCollection<Row> joined;
    switch(realJoinType) {
        case INNER:
            joined = realLeftRows.apply(org.apache.beam.sdk.schemas.transforms.Join.<Row, Row>innerBroadcastJoin(realRightRows).on(FieldsEqual.left(realLeftKeyFields).right(realRightKeyFields)));
            break;
        case LEFT:
            joined = realLeftRows.apply(org.apache.beam.sdk.schemas.transforms.Join.<Row, Row>leftOuterBroadcastJoin(realRightRows).on(FieldsEqual.left(realLeftKeyFields).right(realRightKeyFields)));
            break;
        default:
            throw new RuntimeException("Unexpected join type " + realJoinType);
    }
    Schema schema = CalciteUtils.toSchema(getRowType());
    String lhsSelect = org.apache.beam.sdk.schemas.transforms.Join.LHS_TAG + ".*";
    String rhsSelect = org.apache.beam.sdk.schemas.transforms.Join.RHS_TAG + ".*";
    PCollection<Row> selected = !swapped ? joined.apply(Select.<Row>fieldNames(lhsSelect, rhsSelect).withOutputSchema(schema)) : joined.apply(Select.<Row>fieldNames(rhsSelect, lhsSelect).withOutputSchema(schema));
    return selected;
}
Also used : JoinRelType(org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.rel.core.JoinRelType) FieldAccessDescriptor(org.apache.beam.sdk.schemas.FieldAccessDescriptor) Schema(org.apache.beam.sdk.schemas.Schema) Row(org.apache.beam.sdk.values.Row)

Example 93 with Table

use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.schema.Table in project storm by apache.

the class TestCompilerUtils method sqlOverDummyTable.

public static CalciteState sqlOverDummyTable(String sql) throws RelConversionException, ValidationException, SqlParseException {
    SchemaPlus schema = Frameworks.createRootSchema(true);
    JavaTypeFactory typeFactory = new JavaTypeFactoryImpl(RelDataTypeSystem.DEFAULT);
    StreamableTable streamableTable = new CompilerUtil.TableBuilderInfo(typeFactory).field("ID", SqlTypeName.INTEGER, new ColumnConstraint.PrimaryKey(SqlMonotonicity.MONOTONIC, SqlParserPos.ZERO)).field("NAME", typeFactory.createType(String.class)).field("ADDR", typeFactory.createType(String.class)).build();
    Table table = streamableTable.stream();
    schema.add("FOO", table);
    schema.add("BAR", table);
    schema.add("MYPLUS", ScalarFunctionImpl.create(MyPlus.class, "eval"));
    QueryPlanner queryPlanner = new QueryPlanner(schema);
    StreamsRel tree = queryPlanner.getPlan(sql);
    System.out.println(StormRelUtils.explain(tree, SqlExplainLevel.ALL_ATTRIBUTES));
    return new CalciteState(schema, tree);
}
Also used : StreamableTable(org.apache.calcite.schema.StreamableTable) Table(org.apache.calcite.schema.Table) StreamableTable(org.apache.calcite.schema.StreamableTable) JavaTypeFactoryImpl(org.apache.calcite.jdbc.JavaTypeFactoryImpl) SchemaPlus(org.apache.calcite.schema.SchemaPlus) JavaTypeFactory(org.apache.calcite.adapter.java.JavaTypeFactory) QueryPlanner(org.apache.storm.sql.planner.streams.QueryPlanner) StreamsRel(org.apache.storm.sql.planner.streams.rel.StreamsRel)

Example 94 with Table

use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.schema.Table in project storm by apache.

the class StormSqlContext method interpretCreateTable.

public void interpretCreateTable(SqlCreateTable n) {
    CompilerUtil.TableBuilderInfo builder = new CompilerUtil.TableBuilderInfo(typeFactory);
    List<FieldInfo> fields = new ArrayList<>();
    for (ColumnDefinition col : n.fieldList()) {
        builder.field(col.name(), col.type(), col.constraint());
        RelDataType dataType = col.type().deriveType(typeFactory);
        Class<?> javaType = (Class<?>) typeFactory.getJavaClass(dataType);
        ColumnConstraint constraint = col.constraint();
        boolean isPrimary = constraint != null && constraint instanceof ColumnConstraint.PrimaryKey;
        fields.add(new FieldInfo(col.name(), javaType, isPrimary));
    }
    if (n.parallelism() != null) {
        builder.parallelismHint(n.parallelism());
    }
    Table table = builder.build();
    schema.add(n.tableName(), table);
    ISqlStreamsDataSource ds = DataSourcesRegistry.constructStreamsDataSource(n.location(), n.inputFormatClass(), n.outputFormatClass(), n.properties(), fields);
    if (ds == null) {
        throw new RuntimeException("Failed to find data source for " + n.tableName() + " URI: " + n.location());
    } else if (dataSources.containsKey(n.tableName())) {
        throw new RuntimeException("Duplicated definition for table " + n.tableName());
    }
    dataSources.put(n.tableName(), ds);
}
Also used : CompilerUtil(org.apache.storm.sql.compiler.CompilerUtil) SqlOperatorTable(org.apache.calcite.sql.SqlOperatorTable) Table(org.apache.calcite.schema.Table) ChainedSqlOperatorTable(org.apache.calcite.sql.util.ChainedSqlOperatorTable) SqlStdOperatorTable(org.apache.calcite.sql.fun.SqlStdOperatorTable) SqlCreateTable(org.apache.storm.sql.parser.SqlCreateTable) ArrayList(java.util.ArrayList) RelDataType(org.apache.calcite.rel.type.RelDataType) ColumnDefinition(org.apache.storm.sql.parser.ColumnDefinition) ISqlStreamsDataSource(org.apache.storm.sql.runtime.ISqlStreamsDataSource) ColumnConstraint(org.apache.storm.sql.parser.ColumnConstraint) FieldInfo(org.apache.storm.sql.runtime.FieldInfo)

Example 95 with Table

use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.schema.Table in project calcite by apache.

the class EnumerableTableScan method create.

/**
 * Creates an EnumerableTableScan.
 */
public static EnumerableTableScan create(RelOptCluster cluster, RelOptTable relOptTable) {
    final Table table = relOptTable.unwrap(Table.class);
    Class elementType = EnumerableTableScan.deduceElementType(table);
    final RelTraitSet traitSet = cluster.traitSetOf(EnumerableConvention.INSTANCE).replaceIfs(RelCollationTraitDef.INSTANCE, new Supplier<List<RelCollation>>() {

        public List<RelCollation> get() {
            if (table != null) {
                return table.getStatistic().getCollations();
            }
            return ImmutableList.of();
        }
    });
    return new EnumerableTableScan(cluster, traitSet, relOptTable, elementType);
}
Also used : RelOptTable(org.apache.calcite.plan.RelOptTable) QueryableTable(org.apache.calcite.schema.QueryableTable) ProjectableFilterableTable(org.apache.calcite.schema.ProjectableFilterableTable) ScannableTable(org.apache.calcite.schema.ScannableTable) Table(org.apache.calcite.schema.Table) FilterableTable(org.apache.calcite.schema.FilterableTable) StreamableTable(org.apache.calcite.schema.StreamableTable) ArrayList(java.util.ArrayList) ImmutableList(com.google.common.collect.ImmutableList) List(java.util.List) RelTraitSet(org.apache.calcite.plan.RelTraitSet)

Aggregations

Table (org.apache.calcite.schema.Table)104 Test (org.junit.Test)43 SqlStdOperatorTable (org.apache.calcite.sql.fun.SqlStdOperatorTable)38 RelOptTable (org.apache.calcite.plan.RelOptTable)33 RelDataType (org.apache.calcite.rel.type.RelDataType)22 SqlOperatorTable (org.apache.calcite.sql.SqlOperatorTable)22 SchemaPlus (org.apache.calcite.schema.SchemaPlus)20 ArrayList (java.util.ArrayList)19 List (java.util.List)19 ProjectableFilterableTable (org.apache.calcite.schema.ProjectableFilterableTable)17 ScannableTable (org.apache.calcite.schema.ScannableTable)17 JavaTypeFactoryImpl (org.apache.calcite.jdbc.JavaTypeFactoryImpl)16 FilterableTable (org.apache.calcite.schema.FilterableTable)15 AbstractTable (org.apache.calcite.schema.impl.AbstractTable)15 ResultSet (java.sql.ResultSet)14 Schema (org.apache.beam.sdk.schemas.Schema)14 Row (org.apache.beam.sdk.values.Row)14 StreamableTable (org.apache.calcite.schema.StreamableTable)14 CalciteConnection (org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.jdbc.CalciteConnection)13 Map (java.util.Map)12