Examples with ISqlStreamsDataSource org.apache.storm.sql.runtime.ISqlStreamsDataSource used on opensource projects

Example 6 with ISqlStreamsDataSource

use of org.apache.storm.sql.runtime.ISqlStreamsDataSource in project storm by apache.

the class StormSqlContext method interpretCreateTable.

public void interpretCreateTable(SqlCreateTable n) {
    CompilerUtil.TableBuilderInfo builder = new CompilerUtil.TableBuilderInfo(typeFactory);
    List<FieldInfo> fields = new ArrayList<>();
    for (ColumnDefinition col : n.fieldList()) {
        builder.field(col.name(), col.type(), col.constraint());
        RelDataType dataType = col.type().deriveType(typeFactory);
        Class<?> javaType = (Class<?>) typeFactory.getJavaClass(dataType);
        ColumnConstraint constraint = col.constraint();
        boolean isPrimary = constraint != null && constraint instanceof ColumnConstraint.PrimaryKey;
        fields.add(new FieldInfo(col.name(), javaType, isPrimary));
    }
    if (n.parallelism() != null) {
        builder.parallelismHint(n.parallelism());
    }
    Table table = builder.build();
    schema.add(n.tableName(), table);
    ISqlStreamsDataSource ds = DataSourcesRegistry.constructStreamsDataSource(n.location(), n.inputFormatClass(), n.outputFormatClass(), n.properties(), fields);
    if (ds == null) {
        throw new RuntimeException("Failed to find data source for " + n.tableName() + " URI: " + n.location());
    } else if (dataSources.containsKey(n.tableName())) {
        throw new RuntimeException("Duplicated definition for table " + n.tableName());
    }
    dataSources.put(n.tableName(), ds);
}

Example 7 with ISqlStreamsDataSource

use of org.apache.storm.sql.runtime.ISqlStreamsDataSource in project storm by apache.

the class StreamsStreamScanRel method streamsPlan.

@Override
public void streamsPlan(StreamsPlanCreator planCreator) throws Exception {
    String sourceName = Joiner.on('.').join(getTable().getQualifiedName());
    Map<String, ISqlStreamsDataSource> sources = planCreator.getSources();
    if (!sources.containsKey(sourceName)) {
        throw new RuntimeException("Cannot find table " + sourceName);
    }
    List<String> fieldNames = getRowType().getFieldNames();
    final Stream<Values> finalStream = planCreator.getStreamBuilder().newStream(sources.get(sourceName).getProducer(), new StreamsScanTupleValueMapper(fieldNames), parallelismHint);
    planCreator.addStream(finalStream);
}

Example 8 with ISqlStreamsDataSource

use of org.apache.storm.sql.runtime.ISqlStreamsDataSource in project storm by apache.

the class TestPlanCompiler method testUdf.

@Test
public void testUdf() throws Exception {
    int EXPECTED_VALUE_SIZE = 1;
    String sql = "SELECT MYPLUS(ID, 3)" + "FROM FOO " + "WHERE ID = 2";
    TestCompilerUtils.CalciteState state = TestCompilerUtils.sqlOverDummyTable(sql);
    Map<String, ISqlStreamsDataSource> data = new HashMap<>();
    data.put("FOO", new TestUtils.MockSqlStreamsDataSource());
    QueryPlanner planner = new QueryPlanner(state.schema());
    AbstractStreamsProcessor proc = planner.compile(data, sql);
    // inject output bolt
    proc.outputStream().to(new TestUtils.MockBolt());
    final StormTopology topo = proc.build();
    SqlTestUtil.runStormTopology(cluster, TestUtils.MockBolt.getCollectedValues(), EXPECTED_VALUE_SIZE, proc, topo);
    Assert.assertArrayEquals(new Values[] { new Values(5) }, TestUtils.MockBolt.getCollectedValues().toArray());
}

Example 9 with ISqlStreamsDataSource

use of org.apache.storm.sql.runtime.ISqlStreamsDataSource in project storm by apache.

the class TestPlanCompiler method testInsert.

@Test
public void testInsert() throws Exception {
    final int EXPECTED_VALUE_SIZE = 1;
    String sql = "INSERT INTO BAR SELECT ID, NAME, ADDR FROM FOO WHERE ID > 3";
    TestCompilerUtils.CalciteState state = TestCompilerUtils.sqlOverDummyTable(sql);
    final Map<String, ISqlStreamsDataSource> data = new HashMap<>();
    data.put("FOO", new TestUtils.MockSqlStreamsDataSource());
    data.put("BAR", new TestUtils.MockSqlStreamsOutputDataSource());
    QueryPlanner planner = new QueryPlanner(state.schema());
    AbstractStreamsProcessor proc = planner.compile(data, sql);
    final StormTopology topo = proc.build();
    SqlTestUtil.runStormTopology(cluster, TestUtils.MockInsertBolt.getCollectedValues(), EXPECTED_VALUE_SIZE, proc, topo);
    Assert.assertArrayEquals(new Pair[] { Pair.of(4, new Values(4, "abcde", "y")) }, TestUtils.MockInsertBolt.getCollectedValues().toArray());
}

Example 10 with ISqlStreamsDataSource

use of org.apache.storm.sql.runtime.ISqlStreamsDataSource in project storm by apache.

the class TestKafkaDataSourcesProvider method testKafkaSink.

@SuppressWarnings("unchecked")
@Test
public void testKafkaSink() throws Exception {
    ISqlStreamsDataSource ds = DataSourcesRegistry.constructStreamsDataSource(URI.create("kafka://foo?bootstrap-servers=foo"), null, null, TBL_PROPERTIES, FIELDS);
    Assert.assertNotNull(ds);
    IRichBolt consumer = ds.getConsumer();
    Assert.assertEquals(KafkaBolt.class, consumer.getClass());
}