Examples with RecordPart com.hazelcast.jet.cdc.RecordPart used on opensource projects

Example 6 with RecordPart

use of com.hazelcast.jet.cdc.RecordPart in project hazelcast by hazelcast.

the class PostgresCdcListenBeforeExistsIntegrationTest method pipeline.

private Pipeline pipeline(StreamSource<ChangeRecord> source) {
    Pipeline pipeline = Pipeline.create();
    pipeline.readFrom(source).withNativeTimestamps(0).<ChangeRecord>customTransform("filter_timestamps", filterTimestampsProcessorSupplier()).setLocalParallelism(1).groupingKey(record -> (Integer) record.key().toMap().get("id")).mapStateful(LongAccumulator::new, (accumulator, rowId, record) -> {
        long count = accumulator.get();
        accumulator.add(1);
        Operation operation = record.operation();
        RecordPart value = record.value();
        TableRow row = value.toObject(TableRow.class);
        return entry(rowId + "/" + count, operation + ":" + row);
    }).setLocalParallelism(1).peek().writeTo(Sinks.map(SINK_MAP_NAME));
    return pipeline;
}

Example 7 with RecordPart

use of com.hazelcast.jet.cdc.RecordPart in project hazelcast by hazelcast.

the class PostgresCdcIntegrationTest method ordersPipeline.

@Nonnull
private Pipeline ordersPipeline() {
    Pipeline pipeline = Pipeline.create();
    pipeline.readFrom(source("orders")).withoutTimestamps().groupingKey(PostgresCdcIntegrationTest::getOrderNumber).mapStateful(LongAccumulator::new, (accumulator, orderId, record) -> {
        long count = accumulator.get();
        accumulator.add(1);
        Operation operation = record.operation();
        RecordPart value = record.value();
        Order order = value.toObject(Order.class);
        return entry(orderId + "/" + count, operation + ":" + order);
    }).setLocalParallelism(1).writeTo(Sinks.map("results"));
    return pipeline;
}

Example 8 with RecordPart

use of com.hazelcast.jet.cdc.RecordPart in project hazelcast by hazelcast.

the class MySqlCdcListenBeforeExistIntegrationTest method pipeline.

private Pipeline pipeline(StreamSource<ChangeRecord> source) {
    Pipeline pipeline = Pipeline.create();
    pipeline.readFrom(source).withNativeTimestamps(0).<ChangeRecord>customTransform("filter_timestamps", filterTimestampsProcessorSupplier()).setLocalParallelism(1).groupingKey(record -> (Integer) record.key().toMap().get("id")).mapStateful(LongAccumulator::new, (accumulator, rowId, record) -> {
        long count = accumulator.get();
        accumulator.add(1);
        Operation operation = record.operation();
        RecordPart value = record.value();
        TableRow row = value.toObject(TableRow.class);
        return entry(rowId + "/" + count, operation + ":" + row);
    }).setLocalParallelism(1).peek().writeTo(Sinks.map(SINK_MAP_NAME));
    return pipeline;
}

Example 9 with RecordPart

use of com.hazelcast.jet.cdc.RecordPart in project hazelcast by hazelcast.

the class MySqlCdcWhiteBlackListIntegrationTest method pipeline.

private Pipeline pipeline(StreamSource<ChangeRecord> source) {
    Pipeline pipeline = Pipeline.create();
    pipeline.readFrom(source).withNativeTimestamps(0).filter(t -> t.database().startsWith(DB_PREFIX)).setLocalParallelism(1).<ChangeRecord>customTransform("filter_timestamps", filterTimestampsProcessorSupplier()).setLocalParallelism(1).groupingKey(record -> (Integer) record.key().toMap().get("id")).mapStateful(LongAccumulator::new, (accumulator, rowId, record) -> {
        long count = accumulator.get();
        accumulator.add(1);
        Operation operation = record.operation();
        RecordPart value = record.value();
        TableRow row = value.toObject(TableRow.class);
        return entry(rowId + "/" + count, operation + ":" + row);
    }).setLocalParallelism(1).writeTo(Sinks.map(SINK_MAP_NAME));
    return pipeline;
}