Examples with TableEnvironmentInternal org.apache.flink.table.api.internal.TableEnvironmentInternal used on opensource projects

Example 6 with TableEnvironmentInternal

use of org.apache.flink.table.api.internal.TableEnvironmentInternal in project flink by apache.

the class StreamSQLTestProgram method main.

public static void main(String[] args) throws Exception {
    ParameterTool params = ParameterTool.fromArgs(args);
    String outputPath = params.getRequired("outputPath");
    final StreamExecutionEnvironment sEnv = StreamExecutionEnvironment.getExecutionEnvironment();
    sEnv.setRestartStrategy(RestartStrategies.fixedDelayRestart(3, Time.of(10, TimeUnit.SECONDS)));
    sEnv.enableCheckpointing(4000);
    sEnv.getConfig().setAutoWatermarkInterval(1000);
    final StreamTableEnvironment tEnv = StreamTableEnvironment.create(sEnv);
    ((TableEnvironmentInternal) tEnv).registerTableSourceInternal("table1", new GeneratorTableSource(10, 100, 60, 0));
    ((TableEnvironmentInternal) tEnv).registerTableSourceInternal("table2", new GeneratorTableSource(5, 0.2f, 60, 5));
    int overWindowSizeSeconds = 1;
    int tumbleWindowSizeSeconds = 10;
    String overQuery = String.format("SELECT " + "  key, " + "  rowtime, " + "  COUNT(*) OVER (PARTITION BY key ORDER BY rowtime RANGE BETWEEN INTERVAL '%d' SECOND PRECEDING AND CURRENT ROW) AS cnt " + "FROM table1", overWindowSizeSeconds);
    String tumbleQuery = String.format("SELECT " + "  key, " + "  CASE SUM(cnt) / COUNT(*) WHEN 101 THEN 1 ELSE 99 END AS correct, " + "  TUMBLE_START(rowtime, INTERVAL '%d' SECOND) AS wStart, " + "  TUMBLE_ROWTIME(rowtime, INTERVAL '%d' SECOND) AS rowtime " + "FROM (%s) " + "WHERE rowtime > TIMESTAMP '1970-01-01 00:00:01' " + "GROUP BY key, TUMBLE(rowtime, INTERVAL '%d' SECOND)", tumbleWindowSizeSeconds, tumbleWindowSizeSeconds, overQuery, tumbleWindowSizeSeconds);
    String joinQuery = String.format("SELECT " + "  t1.key, " + "  t2.rowtime AS rowtime, " + "  t2.correct," + "  t2.wStart " + "FROM table2 t1, (%s) t2 " + "WHERE " + "  t1.key = t2.key AND " + "  t1.rowtime BETWEEN t2.rowtime AND t2.rowtime + INTERVAL '%d' SECOND", tumbleQuery, tumbleWindowSizeSeconds);
    String finalAgg = String.format("SELECT " + "  SUM(correct) AS correct, " + "  TUMBLE_START(rowtime, INTERVAL '20' SECOND) AS rowtime " + "FROM (%s) " + "GROUP BY TUMBLE(rowtime, INTERVAL '20' SECOND)", joinQuery);
    // get Table for SQL query
    Table result = tEnv.sqlQuery(finalAgg);
    // convert Table into append-only DataStream
    DataStream<Row> resultStream = tEnv.toAppendStream(result, Types.ROW(Types.INT, Types.SQL_TIMESTAMP));
    final StreamingFileSink<Row> sink = StreamingFileSink.forRowFormat(new Path(outputPath), (Encoder<Row>) (element, stream) -> {
        PrintStream out = new PrintStream(stream);
        out.println(element.toString());
    }).withBucketAssigner(new KeyBucketAssigner()).withRollingPolicy(OnCheckpointRollingPolicy.build()).build();
    resultStream.map(new KillMapper()).setParallelism(1).addSink(sink).setParallelism(1);
    sEnv.execute();
}

Example 7 with TableEnvironmentInternal

use of org.apache.flink.table.api.internal.TableEnvironmentInternal in project flink by apache.

the class TpcdsTestProgram method main.

public static void main(String[] args) throws Exception {
    ParameterTool params = ParameterTool.fromArgs(args);
    String sourceTablePath = params.getRequired("sourceTablePath");
    String queryPath = params.getRequired("queryPath");
    String sinkTablePath = params.getRequired("sinkTablePath");
    Boolean useTableStats = params.getBoolean("useTableStats");
    TableEnvironment tableEnvironment = prepareTableEnv(sourceTablePath, useTableStats);
    // execute TPC-DS queries
    for (String queryId : TPCDS_QUERIES) {
        System.out.println("[INFO]Run TPC-DS query " + queryId + " ...");
        String queryName = QUERY_PREFIX + queryId + QUERY_SUFFIX;
        String queryFilePath = queryPath + FILE_SEPARATOR + queryName;
        String queryString = loadFile2String(queryFilePath);
        Table resultTable = tableEnvironment.sqlQuery(queryString);
        // register sink table
        String sinkTableName = QUERY_PREFIX + queryId + "_sinkTable";
        ((TableEnvironmentInternal) tableEnvironment).registerTableSinkInternal(sinkTableName, new CsvTableSink(sinkTablePath + FILE_SEPARATOR + queryId + RESULT_SUFFIX, COL_DELIMITER, 1, FileSystem.WriteMode.OVERWRITE, resultTable.getSchema().getFieldNames(), resultTable.getSchema().getFieldDataTypes()));
        TableResult tableResult = resultTable.executeInsert(sinkTableName);
        // wait job finish
        tableResult.getJobClient().get().getJobExecutionResult().get();
        System.out.println("[INFO]Run TPC-DS query " + queryId + " success.");
    }
}

Example 8 with TableEnvironmentInternal

use of org.apache.flink.table.api.internal.TableEnvironmentInternal in project flink by apache.

the class BatchSQLTestProgram method main.

public static void main(String[] args) throws Exception {
    ParameterTool params = ParameterTool.fromArgs(args);
    String outputPath = params.getRequired("outputPath");
    String sqlStatement = params.getRequired("sqlStatement");
    TableEnvironment tEnv = TableEnvironment.create(EnvironmentSettings.inBatchMode());
    ((TableEnvironmentInternal) tEnv).registerTableSourceInternal("table1", new GeneratorTableSource(10, 100, 60, 0));
    ((TableEnvironmentInternal) tEnv).registerTableSourceInternal("table2", new GeneratorTableSource(5, 0.2f, 60, 5));
    ((TableEnvironmentInternal) tEnv).registerTableSinkInternal("sinkTable", new CsvTableSink(outputPath).configure(new String[] { "f0", "f1" }, new TypeInformation[] { Types.INT, Types.SQL_TIMESTAMP }));
    TableResult result = tEnv.executeSql(sqlStatement);
    // wait job finish
    result.getJobClient().get().getJobExecutionResult().get();
}

Example 9 with TableEnvironmentInternal

use of org.apache.flink.table.api.internal.TableEnvironmentInternal in project flink by apache.

the class CassandraConnectorITCase method testCassandraTableSink.

@Test
public void testCassandraTableSink() throws Exception {
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    env.setParallelism(4);
    StreamTableEnvironment tEnv = StreamTableEnvironment.create(env);
    DataStreamSource<Row> source = env.fromCollection(rowCollection);
    tEnv.createTemporaryView("testFlinkTable", source);
    ((TableEnvironmentInternal) tEnv).registerTableSinkInternal("cassandraTable", new CassandraAppendTableSink(builderForWriting, injectTableName(INSERT_DATA_QUERY)).configure(new String[] { "f0", "f1", "f2" }, new TypeInformation[] { Types.STRING, Types.INT, Types.INT }));
    tEnv.sqlQuery("select * from testFlinkTable").executeInsert("cassandraTable").await();
    ResultSet rs = session.execute(injectTableName(SELECT_DATA_QUERY));
    // validate that all input was correctly written to Cassandra
    List<Row> input = new ArrayList<>(rowCollection);
    List<com.datastax.driver.core.Row> output = rs.all();
    for (com.datastax.driver.core.Row o : output) {
        Row cmp = new Row(3);
        cmp.setField(0, o.getString(0));
        cmp.setField(1, o.getInt(2));
        cmp.setField(2, o.getInt(1));
        Assert.assertTrue("Row " + cmp + " was written to Cassandra but not in input.", input.remove(cmp));
    }
    Assert.assertTrue("The input data was not completely written to Cassandra", input.isEmpty());
}

Example 10 with TableEnvironmentInternal

use of org.apache.flink.table.api.internal.TableEnvironmentInternal in project flink by apache.

the class LocalExecutor method executeOperation.

@Override
public TableResultInternal executeOperation(String sessionId, Operation operation) throws SqlExecutionException {
    final ExecutionContext context = getExecutionContext(sessionId);
    final TableEnvironmentInternal tEnv = (TableEnvironmentInternal) context.getTableEnvironment();
    try {
        return context.wrapClassLoader(() -> tEnv.executeInternal(operation));
    } catch (Exception e) {
        throw new SqlExecutionException(MESSAGE_SQL_EXECUTION_ERROR, e);
    }
}