Examples with AND org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.sql.SqlKind.AND used on opensource projects

Example 11 with AND

use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.sql.SqlKind.AND in project beam by apache.

the class PubsubTableProviderIT method testSQLSelectsArrayAttributes.

@Ignore("https://issues.apache.org/jira/browse/BEAM-12320")
@Test
public void testSQLSelectsArrayAttributes() throws Exception {
    String createTableString = String.format("CREATE EXTERNAL TABLE message (\n" + "event_timestamp TIMESTAMP, \n" + "attributes ARRAY<ROW<key VARCHAR, `value` VARCHAR>>, \n" + "payload ROW< \n" + "             id INTEGER, \n" + "             name VARCHAR \n" + "           > \n" + ") \n" + "TYPE '%s' \n" + "LOCATION '%s' \n" + "TBLPROPERTIES '{ " + "%s" + "\"protoClass\" : \"%s\", " + "\"timestampAttributeKey\" : \"ts\" }'", tableProvider.getTableType(), eventsTopic.topicPath(), payloadFormatParam(), PayloadMessages.SimpleMessage.class.getName());
    String queryString = "SELECT message.payload.id, attributes[1].key AS a1, attributes[2].key AS a2 FROM message";
    // Initialize SQL environment and create the pubsub table
    BeamSqlEnv sqlEnv = BeamSqlEnv.inMemory(new PubsubTableProvider());
    sqlEnv.executeDdl(createTableString);
    // Apply the PTransform to query the pubsub topic
    PCollection<Row> queryOutput = query(sqlEnv, pipeline, queryString);
    // Observe the query results and send success signal after seeing the expected messages
    queryOutput.apply("waitForSuccess", resultSignal.signalSuccessWhen(SchemaCoder.of(PAYLOAD_SCHEMA), observedRows -> {
        Map<Integer, String> entries = new HashMap<>();
        for (Row row : observedRows) {
            if ("ts".equals(row.getString("a1"))) {
                entries.put(row.getInt32("id"), row.getString("a2"));
            } else {
                entries.put(row.getInt32("id"), row.getString("a1"));
            }
        }
        LOG.info("Entries: {}", entries);
        return entries.equals(ImmutableMap.of(3, "foo", 5, "bar", 7, "baz"));
    }));
    // Start the pipeline
    pipeline.run();
    // Block until a subscription for this topic exists
    eventsTopic.assertSubscriptionEventuallyCreated(pipeline.getOptions().as(GcpOptions.class).getProject(), Duration.standardMinutes(5));
    // Start publishing the messages when main pipeline is started and signaling topic is ready
    eventsTopic.publish(ImmutableList.of(objectsProvider.messageIdName(ts(1), 3, "foo"), objectsProvider.messageIdName(ts(2), 5, "bar"), objectsProvider.messageIdName(ts(3), 7, "baz")));
    // Poll the signaling topic for success message
    resultSignal.waitForSuccess(timeout);
}

Example 12 with AND

use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.sql.SqlKind.AND in project beam by apache.

the class PubsubTableProviderIT method testUsesDlq.

@Test
@SuppressWarnings("unchecked")
public void testUsesDlq() throws Exception {
    String createTableString = String.format("CREATE EXTERNAL TABLE message (\n" + "event_timestamp TIMESTAMP, \n" + "attributes MAP<VARCHAR, VARCHAR>, \n" + "payload ROW< \n" + "             id INTEGER, \n" + "             name VARCHAR \n" + "           > \n" + ") \n" + "TYPE '%s' \n" + "LOCATION '%s' \n" + "TBLPROPERTIES " + "    '{ " + "       %s" + "       \"timestampAttributeKey\" : \"ts\", " + "       \"deadLetterQueue\" : \"%s\", " + "       \"protoClass\" : \"%s\" " + "     }'", tableProvider.getTableType(), eventsTopic.topicPath(), payloadFormatParam(), dlqTopic.topicPath(), PayloadMessages.SimpleMessage.class.getName());
    String queryString = "SELECT message.payload.id, message.payload.name from message";
    // Initialize SQL environment and create the pubsub table
    BeamSqlEnv sqlEnv = BeamSqlEnv.inMemory(new PubsubTableProvider());
    sqlEnv.executeDdl(createTableString);
    // Apply the PTransform to query the pubsub topic
    PCollection<Row> queryOutput = query(sqlEnv, pipeline, queryString);
    // Observe the query results and send success signal after seeing the expected messages
    queryOutput.apply("waitForSuccess", resultSignal.signalSuccessWhen(SchemaCoder.of(PAYLOAD_SCHEMA), observedRows -> observedRows.equals(ImmutableSet.of(row(PAYLOAD_SCHEMA, 3, "foo"), row(PAYLOAD_SCHEMA, 5, "bar"), row(PAYLOAD_SCHEMA, 7, "baz")))));
    // Start the pipeline
    pipeline.run();
    // Block until a subscription for this topic exists
    eventsTopic.assertSubscriptionEventuallyCreated(pipeline.getOptions().as(GcpOptions.class).getProject(), Duration.standardMinutes(5));
    // Start publishing the messages when main pipeline is started and signaling topics are ready
    eventsTopic.publish(ImmutableList.of(objectsProvider.messageIdName(ts(1), 3, "foo"), objectsProvider.messageIdName(ts(2), 5, "bar"), objectsProvider.messageIdName(ts(3), 7, "baz"), // invalid message, will go to DLQ
    messagePayload(ts(4), "{ - }", ImmutableMap.of()), // invalid message, will go to DLQ
    messagePayload(ts(5), "{ + }", ImmutableMap.of())));
    // Poll the signaling topic for success message
    resultSignal.waitForSuccess(timeout);
    dlqTopic.assertThatTopicEventuallyReceives(matcherPayload(ts(4), "{ - }"), matcherPayload(ts(5), "{ + }")).waitForUpTo(Duration.standardSeconds(40));
}

Example 13 with AND

use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.sql.SqlKind.AND in project beam by apache.

the class BeamZetaSqlCatalog method addTableToLeafCatalog.

/**
 * Assume last element in tablePath is a table name, and everything before is catalogs. So the
 * logic is to create nested catalogs until the last level, then add a table at the last level.
 *
 * <p>Table schema is extracted from Calcite schema based on the table name resolution strategy,
 * e.g. either by drilling down the schema.getSubschema() path or joining the table name with dots
 * to construct a single compound identifier (e.g. Data Catalog use case).
 */
private void addTableToLeafCatalog(List<String> tablePath, QueryTrait queryTrait) {
    SimpleCatalog leafCatalog = createNestedCatalogs(zetaSqlCatalog, tablePath);
    org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.schema.Table calciteTable = TableResolution.resolveCalciteTable(calciteSchema, tablePath);
    if (calciteTable == null) {
        throw new ZetaSqlException("Wasn't able to resolve the path " + tablePath + " in schema: " + calciteSchema.getName());
    }
    RelDataType rowType = calciteTable.getRowType(typeFactory);
    TableResolution.SimpleTableWithPath tableWithPath = TableResolution.SimpleTableWithPath.of(tablePath);
    queryTrait.addResolvedTable(tableWithPath);
    addFieldsToTable(tableWithPath, rowType);
    leafCatalog.addSimpleTable(tableWithPath.getTable());
}

Example 14 with AND

use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.sql.SqlKind.AND in project beam by apache.

the class JdbcDriverTest method testDriverManager_simple.

@Test
public void testDriverManager_simple() throws Exception {
    Connection connection = DriverManager.getConnection(JdbcDriver.CONNECT_STRING_PREFIX);
    Statement statement = connection.createStatement();
    // SELECT 1 is a special case and does not reach the parser
    assertTrue(statement.execute("SELECT 1"));
}

Example 15 with AND

use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.sql.SqlKind.AND in project beam by apache.

the class BeamIOPushDownRule method canDropCalc.

/**
 * Perform a series of checks to determine whether a Calc can be dropped. Following conditions
 * need to be met in order for that to happen (logical AND):<br>
 * 1. Program should do simple projects, project each field once, and project fields in the same
 * order when field reordering is not supported.<br>
 * 2. Predicate can be completely pushed-down.<br>
 * 3. Project push-down is supported by the IO or all fields are projected by a Calc.
 *
 * @param program A {@code RexProgram} of a {@code Calc}.
 * @param projectSupport An enum containing information about IO project push-down capabilities.
 * @param tableFilter A class containing information about IO predicate push-down capabilities.
 * @return True when Calc can be dropped, false otherwise.
 */
private boolean canDropCalc(RexProgram program, ProjectSupport projectSupport, BeamSqlTableFilter tableFilter) {
    RelDataType calcInputRowType = program.getInputRowType();
    // Program should do simple projects, project each field once, and project fields in the same
    // order when field reordering is not supported.
    boolean fieldReorderingSupported = projectSupport.withFieldReordering();
    if (!isProjectRenameOnlyProgram(program, fieldReorderingSupported)) {
        return false;
    }
    // Predicate can be completely pushed-down
    if (!tableFilter.getNotSupported().isEmpty()) {
        return false;
    }
    // Project push-down is supported by the IO or all fields are projected by a Calc.
    boolean isProjectSupported = projectSupport.isSupported();
    boolean allFieldsProjected = program.getProjectList().stream().map(ref -> program.getInputRowType().getFieldList().get(ref.getIndex()).getName()).collect(Collectors.toList()).equals(calcInputRowType.getFieldNames());
    return isProjectSupported || allFieldsProjected;
}