Examples with ISqlTridentDataSource org.apache.storm.sql.runtime.ISqlTridentDataSource used on opensource projects

Example 1 with ISqlTridentDataSource

use of org.apache.storm.sql.runtime.ISqlTridentDataSource in project storm by apache.

the class StormSqlImpl method handleCreateTableForTrident.

private void handleCreateTableForTrident(SqlCreateTable n, Map<String, ISqlTridentDataSource> dataSources) {
    List<FieldInfo> fields = updateSchema(n);
    ISqlTridentDataSource ds = DataSourcesRegistry.constructTridentDataSource(n.location(), n.inputFormatClass(), n.outputFormatClass(), n.properties(), fields);
    if (ds == null) {
        throw new RuntimeException("Failed to find data source for " + n.tableName() + " URI: " + n.location());
    } else if (dataSources.containsKey(n.tableName())) {
        throw new RuntimeException("Duplicated definition for table " + n.tableName());
    }
    dataSources.put(n.tableName(), ds);
}

Example 2 with ISqlTridentDataSource

use of org.apache.storm.sql.runtime.ISqlTridentDataSource in project storm by apache.

the class StormSqlImpl method submit.

@Override
public void submit(String name, Iterable<String> statements, Map<String, ?> stormConf, SubmitOptions opts, StormSubmitter.ProgressListener progressListener, String asUser) throws Exception {
    Map<String, ISqlTridentDataSource> dataSources = new HashMap<>();
    for (String sql : statements) {
        StormParser parser = new StormParser(sql);
        SqlNode node = parser.impl().parseSqlStmtEof();
        if (node instanceof SqlCreateTable) {
            handleCreateTableForTrident((SqlCreateTable) node, dataSources);
        } else if (node instanceof SqlCreateFunction) {
            handleCreateFunction((SqlCreateFunction) node);
        } else {
            QueryPlanner planner = new QueryPlanner(schema);
            AbstractTridentProcessor processor = planner.compile(dataSources, sql);
            TridentTopology topo = processor.build();
            Path jarPath = null;
            try {
                // QueryPlanner on Trident mode configures the topology with compiled classes,
                // so we need to add new classes into topology jar
                // Topology will be serialized and sent to Nimbus, and deserialized and executed in workers.
                jarPath = Files.createTempFile("storm-sql", ".jar");
                System.setProperty("storm.jar", jarPath.toString());
                packageTopology(jarPath, processor);
                StormSubmitter.submitTopologyAs(name, stormConf, topo.build(), opts, progressListener, asUser);
            } finally {
                if (jarPath != null) {
                    Files.delete(jarPath);
                }
            }
        }
    }
}

Example 3 with ISqlTridentDataSource

use of org.apache.storm.sql.runtime.ISqlTridentDataSource in project storm by apache.

the class TestPlanCompiler method testCaseStatement.

@Test
public void testCaseStatement() throws Exception {
    int EXPECTED_VALUE_SIZE = 5;
    String sql = "SELECT CASE WHEN NAME IN ('a', 'abc', 'abcde') THEN UPPER('a') " + "WHEN UPPER(NAME) = 'AB' THEN 'b' ELSE {fn CONCAT(NAME, '#')} END FROM FOO";
    TestCompilerUtils.CalciteState state = TestCompilerUtils.sqlOverDummyTable(sql);
    final Map<String, ISqlTridentDataSource> data = new HashMap<>();
    data.put("FOO", new TestUtils.MockSqlTridentDataSource());
    QueryPlanner planner = new QueryPlanner(state.schema());
    AbstractTridentProcessor proc = planner.compile(data, sql);
    final TridentTopology topo = proc.build();
    Fields f = proc.outputStream().getOutputFields();
    proc.outputStream().partitionPersist(new TestUtils.MockStateFactory(), f, new TestUtils.MockStateUpdater(), new Fields());
    runTridentTopology(EXPECTED_VALUE_SIZE, proc, topo);
    Assert.assertArrayEquals(new Values[] { new Values("A"), new Values("b"), new Values("A"), new Values("abcd#"), new Values("A") }, getCollectedValues().toArray());
}

Example 4 with ISqlTridentDataSource

use of org.apache.storm.sql.runtime.ISqlTridentDataSource in project storm by apache.

the class TestPlanCompiler method testCompile.

@Test
public void testCompile() throws Exception {
    final int EXPECTED_VALUE_SIZE = 2;
    String sql = "SELECT ID FROM FOO WHERE ID > 2";
    TestCompilerUtils.CalciteState state = TestCompilerUtils.sqlOverDummyTable(sql);
    final Map<String, ISqlTridentDataSource> data = new HashMap<>();
    data.put("FOO", new TestUtils.MockSqlTridentDataSource());
    QueryPlanner planner = new QueryPlanner(state.schema());
    AbstractTridentProcessor proc = planner.compile(data, sql);
    final TridentTopology topo = proc.build();
    Fields f = proc.outputStream().getOutputFields();
    proc.outputStream().partitionPersist(new TestUtils.MockStateFactory(), f, new TestUtils.MockStateUpdater(), new Fields());
    runTridentTopology(EXPECTED_VALUE_SIZE, proc, topo);
    Assert.assertArrayEquals(new Values[] { new Values(3), new Values(4) }, getCollectedValues().toArray());
}

Example 5 with ISqlTridentDataSource

use of org.apache.storm.sql.runtime.ISqlTridentDataSource in project storm by apache.

the class TestSocketDataSourceProvider method testSocketSink.

@Test
public void testSocketSink() throws IOException {
    ISqlTridentDataSource ds = DataSourcesRegistry.constructTridentDataSource(URI.create("socket://localhost:8888"), null, null, new Properties(), FIELDS);
    Assert.assertNotNull(ds);
    ISqlTridentDataSource.SqlTridentConsumer consumer = ds.getConsumer();
    Assert.assertEquals(SocketState.Factory.class, consumer.getStateFactory().getClass());
    Assert.assertEquals(SocketStateUpdater.class, consumer.getStateUpdater().getClass());
    // makeState() fails on creating State so we just mock SocketState anyway
    SocketState mockState = mock(SocketState.class);
    StateUpdater stateUpdater = consumer.getStateUpdater();
    List<TridentTuple> tupleList = mockTupleList();
    stateUpdater.updateState(mockState, tupleList, null);
    for (TridentTuple t : tupleList) {
        String serializedValue = new String(SERIALIZER.write(t.getValues(), null).array());
        verify(mockState).write(serializedValue + "\n");
    }
}