Examples with CalciteConnection org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.jdbc.CalciteConnection used on opensource projects

Example 41 with CalciteConnection

use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.jdbc.CalciteConnection in project beam by apache.

the class PubsubTableProviderIT method connect.

@SuppressWarnings("unchecked")
private CalciteConnection connect(PipelineOptions options, TableProvider... tableProviders) {
    // HACK: PipelineOptions should expose a prominent method to do this reliably
    // The actual options are in the "options" field of the converted map
    Map<String, String> argsMap = ((Map<String, Object>) MAPPER.convertValue(pipeline.getOptions(), Map.class).get("options")).entrySet().stream().filter((entry) -> {
        if (entry.getValue() instanceof List) {
            if (!((List) entry.getValue()).isEmpty()) {
                throw new IllegalArgumentException("Cannot encode list arguments");
            }
            // We can encode empty lists, just omit them.
            return false;
        }
        return true;
    }).collect(Collectors.toMap(Map.Entry::getKey, entry -> toArg(entry.getValue())));
    InMemoryMetaStore inMemoryMetaStore = new InMemoryMetaStore();
    for (TableProvider tableProvider : tableProviders) {
        inMemoryMetaStore.registerProvider(tableProvider);
    }
    JdbcConnection connection = JdbcDriver.connect(inMemoryMetaStore, options);
    connection.setPipelineOptionsMap(argsMap);
    return connection;
}

Example 42 with CalciteConnection

use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.jdbc.CalciteConnection in project beam by apache.

the class PubsubTableProviderIT method testSQLLimit.

@Test
@SuppressWarnings({ "unchecked", "rawtypes" })
public void testSQLLimit() throws Exception {
    String createTableString = String.format("CREATE EXTERNAL TABLE message (\n" + "event_timestamp TIMESTAMP, \n" + "attributes MAP<VARCHAR, VARCHAR>, \n" + "payload ROW< \n" + "             id INTEGER, \n" + "             name VARCHAR \n" + "           > \n" + ") \n" + "TYPE '%s' \n" + "LOCATION '%s' \n" + "TBLPROPERTIES " + "    '{ " + "       %s" + "       \"timestampAttributeKey\" : \"ts\", " + "       \"deadLetterQueue\" : \"%s\", " + "       \"protoClass\" : \"%s\" " + "     }'", tableProvider.getTableType(), eventsTopic.topicPath(), payloadFormatParam(), dlqTopic.topicPath(), PayloadMessages.SimpleMessage.class.getName());
    List<PubsubMessage> messages = ImmutableList.of(objectsProvider.messageIdName(ts(1), 3, "foo"), objectsProvider.messageIdName(ts(2), 5, "bar"), objectsProvider.messageIdName(ts(3), 7, "baz"), objectsProvider.messageIdName(ts(4), 9, "ba2"), objectsProvider.messageIdName(ts(5), 10, "ba3"), objectsProvider.messageIdName(ts(6), 13, "ba4"), objectsProvider.messageIdName(ts(7), 15, "ba5"));
    // We need the default options on the schema to include the project passed in for the
    // integration test
    CalciteConnection connection = connect(pipeline.getOptions(), new PubsubTableProvider());
    Statement statement = connection.createStatement();
    statement.execute(createTableString);
    // Because Pubsub only allow new subscription receives message after the subscription is
    // created, eventsTopic.publish(messages) can only be called after statement.executeQuery.
    // However, because statement.executeQuery is a blocking call, it has to be put into a
    // separate thread to execute.
    ExecutorService pool = Executors.newFixedThreadPool(1);
    Future<List<String>> queryResult = pool.submit((Callable) () -> {
        ResultSet resultSet = statement.executeQuery("SELECT message.payload.id FROM message LIMIT 3");
        ImmutableList.Builder<String> result = ImmutableList.builder();
        while (resultSet.next()) {
            result.add(resultSet.getString(1));
        }
        return result.build();
    });
    try {
        eventsTopic.assertSubscriptionEventuallyCreated(pipeline.getOptions().as(GcpOptions.class).getProject(), Duration.standardMinutes(5));
    } catch (AssertionError assertionError) {
        // Check if the forked thread had an exception.
        try {
            queryResult.get(0, TimeUnit.SECONDS);
        } catch (TimeoutException e) {
        // Nothing went wrong on the forked thread, but a subscription still wasn't created.
        } catch (ExecutionException e) {
            // up to the user.
            throw new AssertionError("Exception occurred in statement.executeQuery thread", e);
        }
        // Just re-throw the timeout assertion.
        throw assertionError;
    }
    eventsTopic.publish(messages);
    assertThat(queryResult.get(2, TimeUnit.MINUTES).size(), equalTo(3));
    pool.shutdown();
}

Example 43 with CalciteConnection

use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.jdbc.CalciteConnection in project beam by apache.

the class JdbcDriverTest method testDriverManager_setUserAgent.

/**
 * Tests that userAgent can be overridden on the querystring.
 */
@Test
public void testDriverManager_setUserAgent() throws Exception {
    Connection connection = DriverManager.getConnection(JdbcDriver.CONNECT_STRING_PREFIX + "beam.userAgent=Secret Agent");
    SchemaPlus rootSchema = ((CalciteConnection) connection).getRootSchema();
    BeamCalciteSchema beamSchema = (BeamCalciteSchema) CalciteSchema.from(rootSchema.getSubSchema("beam")).schema;
    Map<String, String> pipelineOptions = beamSchema.getPipelineOptions();
    assertThat(pipelineOptions.get("userAgent"), equalTo("Secret Agent"));
}

Example 44 with CalciteConnection

use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.jdbc.CalciteConnection in project beam by apache.

the class JdbcDriverTest method testInternalConnect_setDirectRunner.

@Test
public void testInternalConnect_setDirectRunner() throws Exception {
    CalciteConnection connection = JdbcDriver.connect(BOUNDED_TABLE, PipelineOptionsFactory.create());
    Statement statement = connection.createStatement();
    assertEquals(0, statement.executeUpdate("SET runner = direct"));
    assertTrue(statement.execute("SELECT * FROM test"));
}

Example 45 with CalciteConnection

use of org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.jdbc.CalciteConnection in project beam by apache.

the class JdbcDriverTest method testInternalConnect_unbounded_limit.

@Test
public void testInternalConnect_unbounded_limit() throws Exception {
    ReadOnlyTableProvider tableProvider = new ReadOnlyTableProvider("test", ImmutableMap.of("test", TestUnboundedTable.of(Schema.FieldType.INT32, "order_id", Schema.FieldType.INT32, "site_id", Schema.FieldType.INT32, "price", Schema.FieldType.DATETIME, "order_time").timestampColumnIndex(3).addRows(Duration.ZERO, 1, 1, 1, FIRST_DATE, 1, 2, 6, FIRST_DATE)));
    CalciteConnection connection = JdbcDriver.connect(tableProvider, PipelineOptionsFactory.create());
    Statement statement = connection.createStatement();
    ResultSet resultSet1 = statement.executeQuery("SELECT * FROM test LIMIT 1");
    assertTrue(resultSet1.next());
    assertFalse(resultSet1.next());
    ResultSet resultSet2 = statement.executeQuery("SELECT * FROM test LIMIT 2");
    assertTrue(resultSet2.next());
    assertTrue(resultSet2.next());
    assertFalse(resultSet2.next());
}