Examples with RetractStreamTableSink org.apache.flink.table.sinks.RetractStreamTableSink used on opensource projects

Example 1 with RetractStreamTableSink

use of org.apache.flink.table.sinks.RetractStreamTableSink in project zeppelin by apache.

the class AbstractStreamSqlJob method run.

public String run(Table table, String tableName) throws IOException {
    try {
        this.table = table;
        int parallelism = Integer.parseInt(context.getLocalProperties().getOrDefault("parallelism", defaultParallelism + ""));
        this.schema = removeTimeAttributes(flinkShims, table.getSchema());
        checkTableSchema(schema);
        LOGGER.info("ResultTable Schema: " + this.schema);
        final RowTypeInfo outputType = new RowTypeInfo(schema.getFieldTypes(), schema.getFieldNames());
        // create socket stream iterator
        TypeInformation<Tuple2<Boolean, Row>> socketType = Types.TUPLE(Types.BOOLEAN, outputType);
        TypeSerializer<Tuple2<Boolean, Row>> serializer = socketType.createSerializer(senv.getConfig());
        // pass gateway port and address such that iterator knows where to bind to
        iterator = new SocketStreamIterator<>(0, InetAddress.getByName(RemoteInterpreterUtils.findAvailableHostAddress()), serializer);
        // create table sink
        // pass binding address and port such that sink knows where to send to
        LOGGER.debug("Collecting data at address: " + iterator.getBindAddress() + ":" + iterator.getPort());
        RetractStreamTableSink collectTableSink = (RetractStreamTableSink) flinkShims.getCollectStreamTableSink(iterator.getBindAddress(), iterator.getPort(), serializer);
        // new CollectStreamTableSink(iterator.getBindAddress(), iterator.getPort(), serializer);
        collectTableSink = (RetractStreamTableSink) collectTableSink.configure(outputType.getFieldNames(), outputType.getFieldTypes());
        // workaround, otherwise it won't find the sink properly
        String originalCatalog = stenv.getCurrentCatalog();
        String originalDatabase = stenv.getCurrentDatabase();
        try {
            stenv.useCatalog("default_catalog");
            stenv.useDatabase("default_database");
            flinkShims.registerTableSink(stenv, tableName, collectTableSink);
            table.insertInto(tableName);
        } finally {
            stenv.useCatalog(originalCatalog);
            stenv.useDatabase(originalDatabase);
        }
        long delay = 1000L;
        long period = Long.parseLong(context.getLocalProperties().getOrDefault("refreshInterval", "3000"));
        refreshScheduler.scheduleAtFixedRate(new RefreshTask(context), delay, period, MILLISECONDS);
        ResultRetrievalThread retrievalThread = new ResultRetrievalThread(refreshScheduler);
        retrievalThread.start();
        LOGGER.info("Run job: " + tableName + ", parallelism: " + parallelism);
        String jobName = context.getStringLocalProperty("jobName", tableName);
        stenv.execute(jobName);
        LOGGER.info("Flink Job is finished, jobName: " + jobName);
        // wait for retrieve thread consume all data
        LOGGER.info("Waiting for retrieve thread to be done");
        retrievalThread.join();
        refresh(context);
        String finalResult = buildResult();
        LOGGER.info("Final Result: " + finalResult);
        return finalResult;
    } catch (Exception e) {
        LOGGER.error("Fail to run stream sql job", e);
        throw new IOException("Fail to run stream sql job", e);
    } finally {
        refreshScheduler.shutdownNow();
    }
}

Example 2 with RetractStreamTableSink

use of org.apache.flink.table.sinks.RetractStreamTableSink in project flink by apache.

the class CommonExecLegacySink method translateToPlanInternal.

@SuppressWarnings("unchecked")
@Override
protected Transformation<T> translateToPlanInternal(PlannerBase planner, ExecNodeConfig config) {
    if (tableSink instanceof StreamTableSink) {
        final Transformation<T> transform;
        if (tableSink instanceof RetractStreamTableSink) {
            transform = translateToTransformation(planner, config, true);
        } else if (tableSink instanceof UpsertStreamTableSink) {
            UpsertStreamTableSink<T> upsertSink = (UpsertStreamTableSink<T>) tableSink;
            final boolean isAppendOnlyTable = !needRetraction;
            upsertSink.setIsAppendOnly(isAppendOnlyTable);
            if (upsertKeys != null) {
                upsertSink.setKeyFields(upsertKeys);
            } else {
                if (isAppendOnlyTable) {
                    upsertSink.setKeyFields(null);
                } else {
                    throw new TableException("UpsertStreamTableSink requires that Table has a full primary keys if it is updated.");
                }
            }
            transform = translateToTransformation(planner, config, true);
        } else if (tableSink instanceof AppendStreamTableSink) {
            // verify table is an insert-only (append-only) table
            if (needRetraction) {
                throw new TableException("AppendStreamTableSink requires that Table has only insert changes.");
            }
            transform = translateToTransformation(planner, config, false);
        } else {
            if (isStreaming) {
                throw new TableException("Stream Tables can only be emitted by AppendStreamTableSink, " + "RetractStreamTableSink, or UpsertStreamTableSink.");
            } else {
                transform = translateToTransformation(planner, config, false);
            }
        }
        final DataStream<T> dataStream = new DataStream<T>(planner.getExecEnv(), transform);
        final DataStreamSink<T> dsSink = (DataStreamSink<T>) ((StreamTableSink<T>) tableSink).consumeDataStream(dataStream);
        if (dsSink == null) {
            throw new TableException(String.format("The StreamTableSink#consumeDataStream(DataStream) must be implemented " + "and return the sink transformation DataStreamSink. " + "However, %s doesn't implement this method.", tableSink.getClass().getCanonicalName()));
        }
        return dsSink.getLegacyTransformation();
    } else if (tableSink instanceof DataStreamTableSink) {
        // is no real table sink, so we just need translate its input to Transformation.
        return translateToTransformation(planner, config, ((DataStreamTableSink<T>) tableSink).withChangeFlag());
    } else {
        throw new TableException(String.format("Only Support StreamTableSink! However %s is not a StreamTableSink.", tableSink.getClass().getCanonicalName()));
    }
}