Examples with StreamSchema com.ibm.streams.operator.StreamSchema used on opensource projects

Example 36 with StreamSchema

use of com.ibm.streams.operator.StreamSchema in project streamsx.kafka by IBMStreams.

the class KafkaOperatorsAttrNameParamsTest method kafkaAttrNameParamsTest.

@Test
public void kafkaAttrNameParamsTest() throws Exception {
    Topology topo = getTopology();
    StreamSchema producerSchema = com.ibm.streams.operator.Type.Factory.getStreamSchema("tuple<int32 " + PROD_KEY_ATTR_NAME + ", rstring " + PROD_MSG_ATTR_NAME + ", rstring " + PROD_TOPIC_ATTR_NAME + ", int32 " + PROD_PARTITION_ATTR_NAME + ">");
    // create the producer (produces tuples after a short delay)
    Map<String, Object> producerProps = new HashMap<>();
    producerProps.put("propertiesFile", Constants.PROPERTIES_FILE_PATH);
    producerProps.put("messageAttribute", producerSchema.getAttribute(PROD_MSG_ATTR_NAME));
    producerProps.put("keyAttribute", producerSchema.getAttribute(PROD_KEY_ATTR_NAME));
    producerProps.put("topicAttribute", producerSchema.getAttribute(PROD_TOPIC_ATTR_NAME));
    producerProps.put("partitionAttribute", producerSchema.getAttribute(PROD_PARTITION_ATTR_NAME));
    TStream<String> srcStream = topo.strings(MSG).modify(new Delay<>(5000));
    SPL.invokeSink(Constants.KafkaProducerOp, SPLStreams.convertStream(srcStream, new ProducerConverter(), producerSchema), producerProps);
    // create the consumer
    StreamSchema consumerSchema = com.ibm.streams.operator.Type.Factory.getStreamSchema("tuple<int32 " + CONS_KEY_ATTR_NAME + ", rstring " + CONS_MSG_ATTR_NAME + ", rstring " + CONS_TOPIC_ATTR_NAME + ">");
    Map<String, Object> consumerProps = new HashMap<String, Object>();
    consumerProps.put("propertiesFile", Constants.PROPERTIES_FILE_PATH);
    consumerProps.put("outputMessageAttributeName", CONS_MSG_ATTR_NAME);
    consumerProps.put("outputKeyAttributeName", CONS_KEY_ATTR_NAME);
    consumerProps.put("outputTopicAttributeName", CONS_TOPIC_ATTR_NAME);
    consumerProps.put("topic", Constants.TOPIC_TEST);
    SPLStream consumerStream = SPL.invokeSource(topo, Constants.KafkaConsumerOp, consumerProps, consumerSchema);
    SPLStream msgStream = SPLStreams.stringToSPLStream(consumerStream.convert(t -> {
        return t.getString(CONS_TOPIC_ATTR_NAME) + ":" + t.getInt(CONS_KEY_ATTR_NAME) + ":" + t.getString(CONS_MSG_ATTR_NAME);
    }));
    // test the output of the consumer
    StreamsContext<?> context = StreamsContextFactory.getStreamsContext(Type.DISTRIBUTED_TESTER);
    Tester tester = topo.getTester();
    Condition<List<String>> stringContentsUnordered = tester.stringContentsUnordered(msgStream.toStringStream(), Constants.TOPIC_TEST + ":" + KEY + ":" + MSG);
    HashMap<String, Object> config = new HashMap<>();
    // config.put (ContextProperties.TRACING_LEVEL, java.util.logging.Level.FINE);
    // config.put(ContextProperties.KEEP_ARTIFACTS,  new Boolean(true));
    tester.complete(context, config, stringContentsUnordered, 60, TimeUnit.SECONDS);
    // check the results
    Assert.assertTrue(stringContentsUnordered.valid());
    Assert.assertTrue(stringContentsUnordered.getResult().size() > 0);
}

Example 37 with StreamSchema

use of com.ibm.streams.operator.StreamSchema in project streamsx.kafka by IBMStreams.

the class KafkaOperatorsIntTypeTest method kafkaIntTypeTest.

@Test
public void kafkaIntTypeTest() throws Exception {
    Topology topo = getTopology();
    StreamSchema schema = KafkaSPLStreamsUtils.INT_SCHEMA;
    // create the producer (produces tuples after a short delay)
    TStream<Integer> srcStream = topo.strings(DATA).transform(s -> Integer.valueOf(s)).modify(new Delay<>(5000));
    SPLStream splSrcStream = SPLStreams.convertStream(srcStream, new Converter(), schema);
    SPL.invokeSink(Constants.KafkaProducerOp, splSrcStream, getKafkaParams());
    // create the consumer
    SPLStream consumerStream = SPL.invokeSource(topo, Constants.KafkaConsumerOp, getKafkaParams(), schema);
    SPLStream msgStream = SPLStreams.stringToSPLStream(consumerStream.convert(t -> String.valueOf(t.getInt("message"))));
    // test the output of the consumer
    StreamsContext<?> context = StreamsContextFactory.getStreamsContext(Type.DISTRIBUTED_TESTER);
    Tester tester = topo.getTester();
    Condition<List<String>> stringContentsUnordered = tester.stringContentsUnordered(msgStream.toStringStream(), DATA);
    HashMap<String, Object> config = new HashMap<>();
    // config.put (ContextProperties.KEEP_ARTIFACTS, new Boolean (true));
    // config.put (ContextProperties.TRACING_LEVEL, java.util.logging.Level.FINE);
    tester.complete(context, config, stringContentsUnordered, 60, TimeUnit.SECONDS);
    // check the results
    Assert.assertTrue(stringContentsUnordered.valid());
    Assert.assertTrue(stringContentsUnordered.getResult().size() == DATA.length);
}

Example 38 with StreamSchema

use of com.ibm.streams.operator.StreamSchema in project streamsx.kafka by IBMStreams.

the class KafkaProducerOPortSchemaTest method kafkaProducerOPortOptionalUstringTest.

@Test
public void kafkaProducerOPortOptionalUstringTest() throws Exception {
    final StreamSchema oSchema = com.ibm.streams.operator.Type.Factory.getStreamSchema("tuple<tuple<int32 key, rstring message> inTuple, optional<ustring> failureDescription>");
    doTestWithSPLSchema(oSchema);
}

Example 39 with StreamSchema

use of com.ibm.streams.operator.StreamSchema in project streamsx.topology by IBMStreams.

the class EmbeddedGraph method declareInputs.

private void declareInputs(OperatorInvocation<? extends Operator> opDecl, JsonArray inputs) {
    if (jisEmpty(inputs))
        return;
    // Ensure we deal with them in port order.
    JsonObject[] ports = new JsonObject[inputs.size()];
    for (JsonElement e : inputs) {
        JsonObject input = e.getAsJsonObject();
        ports[input.get("index").getAsInt()] = input;
    }
    for (JsonObject input : ports) {
        String name = jstring(input, "name");
        StreamSchema schema = Type.Factory.getTupleType(jstring(input, "type")).getTupleSchema();
        InputPortDeclaration port = opDecl.addInput(name, schema);
        assert !inputPorts.containsKey(name);
        inputPorts.put(name, port);
        if (input.has("window"))
            windowInput(input, port);
    }
}

Example 40 with StreamSchema

use of com.ibm.streams.operator.StreamSchema in project streamsx.topology by IBMStreams.

the class EmbeddedGraph method declareOutputs.

private void declareOutputs(OperatorInvocation<? extends Operator> opDecl, JsonArray outputs) {
    if (GsonUtilities.jisEmpty(outputs))
        return;
    // Ensure we deal with them in port order.
    JsonObject[] ports = new JsonObject[outputs.size()];
    for (JsonElement e : outputs) {
        JsonObject output = e.getAsJsonObject();
        ports[output.get("index").getAsInt()] = output;
    }
    for (JsonObject output : ports) {
        String name = jstring(output, "name");
        StreamSchema schema = Type.Factory.getTupleType(jstring(output, "type")).getTupleSchema();
        OutputPortDeclaration port = opDecl.addOutput(name, schema);
        assert !outputPorts.containsKey(name);
        outputPorts.put(name, port);
    }
}