Examples with AND org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.sql.SqlKind.AND used on opensource projects

Example 6 with AND

use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.sql.SqlKind.AND in project beam by apache.

the class CalciteQueryPlanner method convertToBeamRel.

/**
 * It parses and validate the input query, then convert into a {@link BeamRelNode} tree. Note that
 * query parameters are not yet supported.
 */
@Override
public BeamRelNode convertToBeamRel(String sqlStatement, QueryParameters queryParameters) throws ParseException, SqlConversionException {
    Preconditions.checkArgument(queryParameters.getKind() == Kind.NONE, "Beam SQL Calcite dialect does not yet support query parameters.");
    BeamRelNode beamRelNode;
    try {
        SqlNode parsed = planner.parse(sqlStatement);
        TableResolutionUtils.setupCustomTableResolution(connection, parsed);
        SqlNode validated = planner.validate(parsed);
        LOG.info("SQL:\n" + validated);
        // root of original logical plan
        RelRoot root = planner.rel(validated);
        LOG.info("SQLPlan>\n" + RelOptUtil.toString(root.rel));
        RelTraitSet desiredTraits = root.rel.getTraitSet().replace(BeamLogicalConvention.INSTANCE).replace(root.collation).simplify();
        // beam physical plan
        root.rel.getCluster().setMetadataProvider(ChainedRelMetadataProvider.of(ImmutableList.of(NonCumulativeCostImpl.SOURCE, RelMdNodeStats.SOURCE, root.rel.getCluster().getMetadataProvider())));
        root.rel.getCluster().setMetadataQuerySupplier(BeamRelMetadataQuery::instance);
        RelMetadataQuery.THREAD_PROVIDERS.set(JaninoRelMetadataProvider.of(root.rel.getCluster().getMetadataProvider()));
        root.rel.getCluster().invalidateMetadataQuery();
        beamRelNode = (BeamRelNode) planner.transform(0, desiredTraits, root.rel);
        LOG.info("BEAMPlan>\n" + RelOptUtil.toString(beamRelNode));
    } catch (RelConversionException | CannotPlanException e) {
        throw new SqlConversionException(String.format("Unable to convert query %s", sqlStatement), e);
    } catch (SqlParseException | ValidationException e) {
        throw new ParseException(String.format("Unable to parse query %s", sqlStatement), e);
    } finally {
        planner.close();
    }
    return beamRelNode;
}

Example 7 with AND

use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.sql.SqlKind.AND in project beam by apache.

the class SqlTransform method toTableMap.

@SuppressWarnings("unchecked")
private Map<String, BeamSqlTable> toTableMap(PInput inputs) {
    /**
     * A single PCollection is transformed to a table named PCOLLECTION, other input types are
     * expanded and converted to tables using the tags as names.
     */
    if (inputs instanceof PCollection) {
        PCollection<?> pCollection = (PCollection<?>) inputs;
        return ImmutableMap.of(PCOLLECTION_NAME, new BeamPCollectionTable(pCollection));
    }
    ImmutableMap.Builder<String, BeamSqlTable> tables = ImmutableMap.builder();
    for (Map.Entry<TupleTag<?>, PValue> input : inputs.expand().entrySet()) {
        PCollection<?> pCollection = (PCollection<?>) input.getValue();
        tables.put(input.getKey().getId(), new BeamPCollectionTable(pCollection));
    }
    return tables.build();
}

Example 8 with AND

use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.sql.SqlKind.AND in project beam by apache.

the class BeamCalcRelTest method testNodeStatsNumberOfConditions.

@Test
public void testNodeStatsNumberOfConditions() {
    String equalSql = "SELECT * FROM ORDER_DETAILS_BOUNDED where order_id=1";
    String doubleEqualSql = "SELECT * FROM ORDER_DETAILS_BOUNDED WHERE order_id=1 AND site_id=2 ";
    RelNode equalRoot = env.parseQuery(equalSql);
    RelNode doubleEqualRoot = env.parseQuery(doubleEqualSql);
    NodeStats equalEstimate = BeamSqlRelUtils.getNodeStats(equalRoot, ((BeamRelMetadataQuery) equalRoot.getCluster().getMetadataQuery()));
    NodeStats doubleEqualEstimate = BeamSqlRelUtils.getNodeStats(doubleEqualRoot, ((BeamRelMetadataQuery) doubleEqualRoot.getCluster().getMetadataQuery()));
    Assert.assertTrue(doubleEqualEstimate.getRowCount() < equalEstimate.getRowCount());
    Assert.assertTrue(doubleEqualEstimate.getWindow() < equalEstimate.getWindow());
}

Example 9 with AND

use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.sql.SqlKind.AND in project beam by apache.

the class BeamCoGBKJoinRelBoundedVsBoundedTest method testNodeStatsOfMoreConditions.

@Test
public void testNodeStatsOfMoreConditions() {
    String sql1 = "SELECT *  " + " FROM ORDER_DETAILS1 o1 " + " JOIN ORDER_DETAILS2 o2 " + " on " + " o1.order_id=o2.site_id ";
    String sql2 = "SELECT *  " + " FROM ORDER_DETAILS1 o1 " + " JOIN ORDER_DETAILS2 o2 " + " on " + " o1.order_id=o2.site_id AND o2.price=o1.site_id";
    RelNode root1 = env.parseQuery(sql1);
    while (!(root1 instanceof BeamCoGBKJoinRel)) {
        root1 = root1.getInput(0);
    }
    RelNode root2 = env.parseQuery(sql2);
    while (!(root2 instanceof BeamCoGBKJoinRel)) {
        root2 = root2.getInput(0);
    }
    NodeStats estimate1 = BeamSqlRelUtils.getNodeStats(root1, ((BeamRelMetadataQuery) root1.getCluster().getMetadataQuery()));
    NodeStats estimate2 = BeamSqlRelUtils.getNodeStats(root2, ((BeamRelMetadataQuery) root1.getCluster().getMetadataQuery()));
    Assert.assertNotEquals(0d, estimate2.getRowCount(), 0.001);
    // A join with two conditions should have lower estimate.
    Assert.assertTrue(estimate2.getRowCount() < estimate1.getRowCount());
}

Example 10 with AND

use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.sql.SqlKind.AND in project beam by apache.

the class PubsubTableProviderIT method testSQLWithBytePayload.

@Test
public void testSQLWithBytePayload() throws Exception {
    // Prepare messages to send later
    List<PubsubMessage> messages = ImmutableList.of(objectsProvider.messageIdName(ts(1), 3, "foo"), objectsProvider.messageIdName(ts(2), 5, "bar"), objectsProvider.messageIdName(ts(3), 7, "baz"));
    String createTableString = String.format("CREATE EXTERNAL TABLE message (\n" + "event_timestamp TIMESTAMP, \n" + "attributes MAP<VARCHAR, VARCHAR>, \n" + "payload VARBINARY \n" + ") \n" + "TYPE '%s' \n" + "LOCATION '%s' \n" + "TBLPROPERTIES '{ " + "\"protoClass\" : \"%s\", " + "\"timestampAttributeKey\" : \"ts\" }'", tableProvider.getTableType(), eventsTopic.topicPath(), PayloadMessages.SimpleMessage.class.getName());
    String queryString = "SELECT message.payload AS some_bytes FROM message";
    // Initialize SQL environment and create the pubsub table
    BeamSqlEnv sqlEnv = BeamSqlEnv.inMemory(new PubsubTableProvider());
    sqlEnv.executeDdl(createTableString);
    // Apply the PTransform to query the pubsub topic
    PCollection<Row> queryOutput = query(sqlEnv, pipeline, queryString);
    // Observe the query results and send success signal after seeing the expected messages
    Schema justBytesSchema = Schema.builder().addField("some_bytes", FieldType.BYTES.withNullable(true)).build();
    Row expectedRow0 = row(justBytesSchema, (Object) messages.get(0).getPayload());
    Row expectedRow1 = row(justBytesSchema, (Object) messages.get(1).getPayload());
    Row expectedRow2 = row(justBytesSchema, (Object) messages.get(2).getPayload());
    Set<Row> expected = ImmutableSet.of(expectedRow0, expectedRow1, expectedRow2);
    queryOutput.apply("waitForSuccess", resultSignal.signalSuccessWhen(SchemaCoder.of(justBytesSchema), observedRows -> observedRows.equals(expected)));
    // Start the pipeline
    pipeline.run();
    // Block until a subscription for this topic exists
    eventsTopic.assertSubscriptionEventuallyCreated(pipeline.getOptions().as(GcpOptions.class).getProject(), Duration.standardMinutes(5));
    // Start publishing the messages when main pipeline is started and signaling topic is ready
    eventsTopic.publish(messages);
    // Poll the signaling topic for success message
    resultSignal.waitForSuccess(timeout);
}