Examples with SimpleStringEncoder org.apache.flink.api.common.serialization.SimpleStringEncoder used on opensource projects

Example 6 with SimpleStringEncoder

use of org.apache.flink.api.common.serialization.SimpleStringEncoder in project flink by apache.

the class IterateExample method main.

// *************************************************************************
// PROGRAM
// *************************************************************************
public static void main(String[] args) throws Exception {
    // Checking input parameters
    final ParameterTool params = ParameterTool.fromArgs(args);
    // set up input for the stream of integer pairs
    // obtain execution environment and set setBufferTimeout to 1 to enable
    // continuous flushing of the output buffers (lowest latency)
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment().setBufferTimeout(1);
    // make parameters available in the web interface
    env.getConfig().setGlobalJobParameters(params);
    // create input stream of integer pairs
    DataStream<Tuple2<Integer, Integer>> inputStream;
    if (params.has("input")) {
        inputStream = env.readTextFile(params.get("input")).map(new FibonacciInputMap());
    } else {
        System.out.println("Executing Iterate example with default input data set.");
        System.out.println("Use --input to specify file input.");
        inputStream = env.addSource(new RandomFibonacciSource());
    }
    // create an iterative data stream from the input with 5 second timeout
    IterativeStream<Tuple5<Integer, Integer, Integer, Integer, Integer>> it = inputStream.map(new InputMap()).iterate(5000L);
    // apply the step function to get the next Fibonacci number
    // increment the counter and split the output
    SingleOutputStreamOperator<Tuple5<Integer, Integer, Integer, Integer, Integer>> step = it.process(new Step());
    // close the iteration by selecting the tuples that were directed to the
    // 'iterate' channel in the output selector
    it.closeWith(step.getSideOutput(ITERATE_TAG));
    // to produce the final get the input pairs that have the greatest iteration counter
    // on a 1 second sliding window
    DataStream<Tuple2<Tuple2<Integer, Integer>, Integer>> numbers = step.map(new OutputMap());
    // emit results
    if (params.has("output")) {
        numbers.sinkTo(FileSink.<Tuple2<Tuple2<Integer, Integer>, Integer>>forRowFormat(new Path(params.get("output")), new SimpleStringEncoder<>()).withRollingPolicy(DefaultRollingPolicy.builder().withMaxPartSize(MemorySize.ofMebiBytes(1)).withRolloverInterval(Duration.ofSeconds(10)).build()).build());
    } else {
        System.out.println("Printing result to stdout. Use --output to specify output path.");
        numbers.print();
    }
    // execute the program
    env.execute("Streaming Iteration Example");
}

Example 7 with SimpleStringEncoder

use of org.apache.flink.api.common.serialization.SimpleStringEncoder in project flink by apache.

the class ContinuousFileReaderOperatorITCase method testChainedOperatorsAreNotPrematurelyClosed.

/**
 * Tests https://issues.apache.org/jira/browse/FLINK-20888.
 */
@Test
public void testChainedOperatorsAreNotPrematurelyClosed() throws Exception {
    final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    env.setParallelism(1);
    File input = temp.newFile("input");
    FileUtils.write(input, "test", StandardCharsets.UTF_8);
    DataStream<String> stream = env.readTextFile(input.getAbsolutePath());
    final FileSink<String> sink = FileSink.forRowFormat(new Path(temp.newFolder("output").getAbsolutePath()), new SimpleStringEncoder<String>()).withOutputFileConfig(OutputFileConfig.builder().build()).withRollingPolicy(DefaultRollingPolicy.builder().withMaxPartSize(MemorySize.ofMebiBytes(1)).build()).build();
    stream.sinkTo(sink);
    env.execute("test");
}