Examples with StreamingQueryException org.apache.spark.sql.streaming.StreamingQueryException used on opensource projects

Example 1 with StreamingQueryException

use of org.apache.spark.sql.streaming.StreamingQueryException in project net.jgp.labs.spark by jgperrin.

the class ReadLinesFromMultipleFileStreams method start.

private void start() {
    log.debug("-> start()");
    SparkSession spark = SparkSession.builder().appName("Read lines over a file stream").master("local").getOrCreate();
    // @formatter:off
    Dataset<Row> df = spark.readStream().format("text").load(StreamingUtils.getInputDirectory());
    // @formatter:on
    StreamingQuery query = df.writeStream().outputMode(OutputMode.Update()).format("console").start();
    try {
        query.awaitTermination();
    } catch (StreamingQueryException e) {
        log.error("Exception while waiting for query to end {}.", e.getMessage(), e);
    }
    // In this case everything is a string
    df.show();
    df.printSchema();
}

Example 2 with StreamingQueryException

use of org.apache.spark.sql.streaming.StreamingQueryException in project net.jgp.labs.spark by jgperrin.

the class ReadLinesFromFileStream method start.

private void start() {
    log.debug("-> start()");
    SparkSession spark = SparkSession.builder().appName("Read lines over a file stream").master("local").getOrCreate();
    Dataset<Row> df = spark.readStream().format("text").load(StreamingUtils.getInputDirectory());
    StreamingQuery query = df.writeStream().outputMode(OutputMode.Update()).format("console").start();
    try {
        query.awaitTermination();
    } catch (StreamingQueryException e) {
        log.error("Exception while waiting for query to end {}.", e.getMessage(), e);
    }
    // Never executed
    df.show();
    df.printSchema();
}