Examples with Tuple3 org.apache.flink.api.java.tuple.Tuple3 used on opensource projects

Example 66 with Tuple3

use of org.apache.flink.api.java.tuple.Tuple3 in project flink by apache.

the class SemanticPropertiesPrecedenceTest method testFunctionSkipCodeAnalysisAnnotationPrecedence.

@Test
public void testFunctionSkipCodeAnalysisAnnotationPrecedence() {
    ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
    env.getConfig().setCodeAnalysisMode(CodeAnalysisMode.OPTIMIZE);
    @SuppressWarnings("unchecked") DataSet<Tuple3<Long, String, Integer>> input = env.fromElements(Tuple3.of(3l, "test", 42));
    input.map(new WildcardForwardedMapperWithSkipAnnotation<Tuple3<Long, String, Integer>>()).output(new DiscardingOutputFormat<Tuple3<Long, String, Integer>>());
    Plan plan = env.createProgramPlan();
    GenericDataSinkBase<?> sink = plan.getDataSinks().iterator().next();
    MapOperatorBase<?, ?, ?> mapper = (MapOperatorBase<?, ?, ?>) sink.getInput();
    SingleInputSemanticProperties semantics = mapper.getSemanticProperties();
    FieldSet fw1 = semantics.getForwardingTargetFields(0, 0);
    FieldSet fw2 = semantics.getForwardingTargetFields(0, 1);
    FieldSet fw3 = semantics.getForwardingTargetFields(0, 2);
    assertNotNull(fw1);
    assertNotNull(fw2);
    assertNotNull(fw3);
    assertFalse(fw1.contains(0));
    assertFalse(fw2.contains(1));
    assertFalse(fw3.contains(2));
}

Example 67 with Tuple3

use of org.apache.flink.api.java.tuple.Tuple3 in project flink by apache.

the class SemanticPropertiesPrecedenceTest method testFunctionAnalyzerPrecedence.

@Test
public void testFunctionAnalyzerPrecedence() {
    ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
    env.getConfig().setCodeAnalysisMode(CodeAnalysisMode.OPTIMIZE);
    @SuppressWarnings("unchecked") DataSet<Tuple3<Long, String, Integer>> input = env.fromElements(Tuple3.of(3l, "test", 42));
    input.map(new WildcardForwardedMapper<Tuple3<Long, String, Integer>>()).output(new DiscardingOutputFormat<Tuple3<Long, String, Integer>>());
    Plan plan = env.createProgramPlan();
    GenericDataSinkBase<?> sink = plan.getDataSinks().iterator().next();
    MapOperatorBase<?, ?, ?> mapper = (MapOperatorBase<?, ?, ?>) sink.getInput();
    SingleInputSemanticProperties semantics = mapper.getSemanticProperties();
    FieldSet fw1 = semantics.getForwardingTargetFields(0, 0);
    FieldSet fw2 = semantics.getForwardingTargetFields(0, 1);
    FieldSet fw3 = semantics.getForwardingTargetFields(0, 2);
    assertNotNull(fw1);
    assertNotNull(fw2);
    assertNotNull(fw3);
    assertTrue(fw1.contains(0));
    assertTrue(fw2.contains(1));
    assertTrue(fw3.contains(2));
}

Example 68 with Tuple3

use of org.apache.flink.api.java.tuple.Tuple3 in project flink by apache.

the class AvroOutputFormatITCase method testProgram.

@Override
protected void testProgram() throws Exception {
    ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
    DataSet<Tuple3<String, Integer, String>> input = env.readCsvFile(inputPath).fieldDelimiter("|").types(String.class, Integer.class, String.class);
    //output the data with AvroOutputFormat for specific user type
    DataSet<User> specificUser = input.map(new ConvertToUser());
    AvroOutputFormat<User> avroOutputFormat = new AvroOutputFormat<User>(User.class);
    // FLINK-4771: use a codec
    avroOutputFormat.setCodec(AvroOutputFormat.Codec.SNAPPY);
    //FLINK-3304: Ensure the OF is properly serializing the schema
    avroOutputFormat.setSchema(User.SCHEMA$);
    specificUser.write(avroOutputFormat, outputPath1);
    //output the data with AvroOutputFormat for reflect user type
    DataSet<ReflectiveUser> reflectiveUser = specificUser.map(new ConvertToReflective());
    reflectiveUser.write(new AvroOutputFormat<ReflectiveUser>(ReflectiveUser.class), outputPath2);
    env.execute();
}

Example 69 with Tuple3

use of org.apache.flink.api.java.tuple.Tuple3 in project flink by apache.

the class CassandraConnectorITCase method testCassandraBatchFormats.

@Test
public void testCassandraBatchFormats() throws Exception {
    ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
    env.setParallelism(1);
    DataSet<Tuple3<String, Integer, Integer>> dataSet = env.fromCollection(collection);
    dataSet.output(new CassandraOutputFormat<Tuple3<String, Integer, Integer>>(INSERT_DATA_QUERY, builder));
    env.execute("Write data");
    DataSet<Tuple3<String, Integer, Integer>> inputDS = env.createInput(new CassandraInputFormat<Tuple3<String, Integer, Integer>>(SELECT_DATA_QUERY, builder), TypeInformation.of(new TypeHint<Tuple3<String, Integer, Integer>>() {
    }));
    long count = inputDS.count();
    Assert.assertEquals(count, 20L);
}

Example 70 with Tuple3

use of org.apache.flink.api.java.tuple.Tuple3 in project flink by apache.

the class KafkaConsumerTestBase method runProduceConsumeMultipleTopics.

/**
	 * Test producing and consuming into multiple topics
	 * @throws java.lang.Exception
	 */
public void runProduceConsumeMultipleTopics() throws java.lang.Exception {
    final int NUM_TOPICS = 5;
    final int NUM_ELEMENTS = 20;
    StreamExecutionEnvironment env = StreamExecutionEnvironment.createRemoteEnvironment("localhost", flinkPort);
    env.getConfig().disableSysoutLogging();
    // create topics with content
    final List<String> topics = new ArrayList<>();
    for (int i = 0; i < NUM_TOPICS; i++) {
        final String topic = "topic-" + i;
        topics.add(topic);
        // create topic
        createTestTopic(topic, i + 1, /*partitions*/
        1);
    }
    // run first job, producing into all topics
    DataStream<Tuple3<Integer, Integer, String>> stream = env.addSource(new RichParallelSourceFunction<Tuple3<Integer, Integer, String>>() {

        @Override
        public void run(SourceContext<Tuple3<Integer, Integer, String>> ctx) throws Exception {
            int partition = getRuntimeContext().getIndexOfThisSubtask();
            for (int topicId = 0; topicId < NUM_TOPICS; topicId++) {
                for (int i = 0; i < NUM_ELEMENTS; i++) {
                    ctx.collect(new Tuple3<>(partition, i, "topic-" + topicId));
                }
            }
        }

        @Override
        public void cancel() {
        }
    });
    Tuple2WithTopicSchema schema = new Tuple2WithTopicSchema(env.getConfig());
    Properties props = new Properties();
    props.putAll(standardProps);
    props.putAll(secureProps);
    kafkaServer.produceIntoKafka(stream, "dummy", schema, props, null);
    env.execute("Write to topics");
    // run second job consuming from multiple topics
    env = StreamExecutionEnvironment.createRemoteEnvironment("localhost", flinkPort);
    env.getConfig().disableSysoutLogging();
    stream = env.addSource(kafkaServer.getConsumer(topics, schema, props));
    stream.flatMap(new FlatMapFunction<Tuple3<Integer, Integer, String>, Integer>() {

        Map<String, Integer> countPerTopic = new HashMap<>(NUM_TOPICS);

        @Override
        public void flatMap(Tuple3<Integer, Integer, String> value, Collector<Integer> out) throws Exception {
            Integer count = countPerTopic.get(value.f2);
            if (count == null) {
                count = 1;
            } else {
                count++;
            }
            countPerTopic.put(value.f2, count);
            // check map:
            for (Map.Entry<String, Integer> el : countPerTopic.entrySet()) {
                if (el.getValue() < NUM_ELEMENTS) {
                    // not enough yet
                    break;
                }
                if (el.getValue() > NUM_ELEMENTS) {
                    throw new RuntimeException("There is a failure in the test. I've read " + el.getValue() + " from topic " + el.getKey());
                }
            }
            // we've seen messages from all topics
            throw new SuccessException();
        }
    }).setParallelism(1);
    tryExecute(env, "Count elements from the topics");
    // delete all topics again
    for (int i = 0; i < NUM_TOPICS; i++) {
        final String topic = "topic-" + i;
        deleteTestTopic(topic);
    }
}