Examples with KafkaSinkInfo org.apache.inlong.sort.protocol.sink.KafkaSinkInfo used on opensource projects

Example 1 with KafkaSinkInfo

use of org.apache.inlong.sort.protocol.sink.KafkaSinkInfo in project incubator-inlong by apache.

the class Entrance method buildSinkStream.

private static void buildSinkStream(DataStream<Row> sourceStream, Configuration config, SinkInfo sinkInfo, Map<String, Object> properties, long dataflowId) throws IOException, ClassNotFoundException {
    final String sinkType = checkNotNull(config.getString(Constants.SINK_TYPE));
    final int sinkParallelism = config.getInteger(Constants.SINK_PARALLELISM);
    switch(sinkType) {
        case Constants.SINK_TYPE_CLICKHOUSE:
            checkState(sinkInfo instanceof ClickHouseSinkInfo);
            ClickHouseSinkInfo clickHouseSinkInfo = (ClickHouseSinkInfo) sinkInfo;
            sourceStream.addSink(new ClickhouseRowSinkFunction(clickHouseSinkInfo)).uid(Constants.SINK_UID).name("Clickhouse Sink").setParallelism(sinkParallelism);
            break;
        case Constants.SINK_TYPE_HIVE:
            checkState(sinkInfo instanceof HiveSinkInfo);
            HiveSinkInfo hiveSinkInfo = (HiveSinkInfo) sinkInfo;
            if (hiveSinkInfo.getPartitions().length == 0) {
                // The committer operator is not necessary if partition is not existent.
                sourceStream.process(new HiveWriter(config, dataflowId, hiveSinkInfo)).uid(Constants.SINK_UID).name("Hive Sink").setParallelism(sinkParallelism);
            } else {
                sourceStream.process(new HiveWriter(config, dataflowId, hiveSinkInfo)).uid(Constants.SINK_UID).name("Hive Sink").setParallelism(sinkParallelism).addSink(new HiveCommitter(config, hiveSinkInfo)).name("Hive Committer").setParallelism(1);
            }
            break;
        case Constants.SINK_TYPE_ICEBERG:
            checkState(sinkInfo instanceof IcebergSinkInfo);
            IcebergSinkInfo icebergSinkInfo = (IcebergSinkInfo) sinkInfo;
            TableLoader tableLoader = TableLoader.fromHadoopTable(icebergSinkInfo.getTableLocation(), new org.apache.hadoop.conf.Configuration());
            FlinkSink.forRow(sourceStream, CommonUtils.getTableSchema(sinkInfo.getFields())).tableLoader(tableLoader).writeParallelism(sinkParallelism).build();
            break;
        case Constants.SINK_TYPE_KAFKA:
            checkState(sinkInfo instanceof KafkaSinkInfo);
            SerializationSchema<Row> schema = SerializationSchemaFactory.build(sinkInfo.getFields(), ((KafkaSinkInfo) sinkInfo).getSerializationInfo());
            sourceStream.addSink(buildKafkaSink((KafkaSinkInfo) sinkInfo, properties, schema, config)).uid(Constants.SINK_UID).name("Kafka Sink").setParallelism(sinkParallelism);
            break;
        default:
            throw new IllegalArgumentException("Unsupported sink type " + sinkType);
    }
}

Example 2 with KafkaSinkInfo

use of org.apache.inlong.sort.protocol.sink.KafkaSinkInfo in project incubator-inlong by apache.

the class KafkaSinkTestBase method testKafkaSink.

@Test(timeout = 3 * 60 * 1000)
public void testKafkaSink() throws Exception {
    TestingSource testingSource = createTestingSource();
    final ExecutorService executorService = Executors.newSingleThreadExecutor();
    CountDownLatch testFinishedCountDownLatch = new CountDownLatch(1);
    executorService.execute(() -> {
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        try {
            env.addSource(testingSource).addSink(buildKafkaSink(new KafkaSinkInfo(new FieldInfo[] {}, brokerConnStr, topic, null), new HashMap<>(), serializationSchema, new Configuration()));
            env.execute();
            testFinishedCountDownLatch.await();
        } catch (Exception e) {
            logger.error("Error occurred when executing flink test job: ", e);
        }
    });
    verify();
    testFinishedCountDownLatch.countDown();
}

Example 3 with KafkaSinkInfo

use of org.apache.inlong.sort.protocol.sink.KafkaSinkInfo in project incubator-inlong by apache.

the class KafkaSinkInfoTest method init.

@Override
public void init() {
    expectedObject = new KafkaSinkInfo(new FieldInfo[] { new FieldInfo("field1", new StringFormatInfo()) }, "testAddress", "testTopic", new JsonSerializationInfo());
    expectedJson = "{\n" + "    \"type\":\"kafka\",\n" + "    \"fields\":[\n" + "        {\n" + "            \"type\":\"base\",\n" + "            \"name\":\"field1\",\n" + "            \"format_info\":{\n" + "                \"type\":\"string\"\n" + "            }\n" + "        }\n" + "    ],\n" + "    \"address\":\"testAddress\",\n" + "    \"topic\":\"testTopic\",\n" + "    \"serialization_info\":{\n" + "        \"type\":\"json\"\n" + "    }\n" + "}";
    equalObj1 = expectedObject;
    equalObj2 = new KafkaSinkInfo(new FieldInfo[] { new FieldInfo("field1", new StringFormatInfo()) }, "testAddress", "testTopic", new JsonSerializationInfo());
    unequalObj = new KafkaSinkInfo(new FieldInfo[] { new FieldInfo("field1", new StringFormatInfo()) }, "testAddress", "testTopic2", new JsonSerializationInfo());
}

Example 4 with KafkaSinkInfo

use of org.apache.inlong.sort.protocol.sink.KafkaSinkInfo in project incubator-inlong by apache.

the class SinkInfoUtils method createKafkaSinkInfo.

private static KafkaSinkInfo createKafkaSinkInfo(SourceResponse sourceResponse, KafkaSinkResponse sinkResponse, List<FieldInfo> sinkFields) {
    String addressUrl = sinkResponse.getAddress();
    String topicName = sinkResponse.getTopicName();
    SerializationInfo serializationInfo = SerializationUtils.createSerialInfo(sourceResponse, sinkResponse);
    return new KafkaSinkInfo(sinkFields.toArray(new FieldInfo[0]), addressUrl, topicName, serializationInfo);
}