Examples with RowKind org.apache.flink.types.RowKind used on opensource projects

Example 6 with RowKind

use of org.apache.flink.types.RowKind in project flink by apache.

the class DataStreamJavaITCase method testMaterializedResult.

private static void testMaterializedResult(DataStream<Row> dataStream, int primaryKeyPos, Row... expectedResult) throws Exception {
    try (CloseableIterator<Row> iterator = dataStream.executeAndCollect()) {
        final List<Row> materializedResult = new ArrayList<>();
        iterator.forEachRemaining(row -> {
            final RowKind kind = row.getKind();
            row.setKind(RowKind.INSERT);
            switch(kind) {
                case UPDATE_AFTER:
                    final Object primaryKeyValue = row.getField(primaryKeyPos);
                    assert primaryKeyValue != null;
                    materializedResult.removeIf(r -> primaryKeyValue.equals(r.getField(primaryKeyPos)));
                // fall through
                case INSERT:
                    materializedResult.add(row);
                    break;
                case UPDATE_BEFORE:
                case DELETE:
                    materializedResult.remove(row);
                    break;
            }
        });
        assertThat(materializedResult, containsInAnyOrder(expectedResult));
    }
}

Example 7 with RowKind

use of org.apache.flink.types.RowKind in project flink by apache.

the class ChangelogModeJsonDeserializer method deserialize.

@Override
public ChangelogMode deserialize(JsonParser jsonParser, DeserializationContext deserializationContext) throws IOException {
    ChangelogMode.Builder builder = ChangelogMode.newBuilder();
    JsonNode rowKindsNode = jsonParser.readValueAsTree();
    for (JsonNode rowKindNode : rowKindsNode) {
        RowKind rowKind = RowKind.valueOf(rowKindNode.asText().toUpperCase());
        builder.addContainedKind(rowKind);
    }
    return builder.build();
}

Example 8 with RowKind

use of org.apache.flink.types.RowKind in project flink by apache.

the class ChangelogModeJsonSerializer method serialize.

@Override
public void serialize(ChangelogMode changelogMode, JsonGenerator jsonGenerator, SerializerProvider serializerProvider) throws IOException {
    jsonGenerator.writeStartArray();
    for (RowKind rowKind : changelogMode.getContainedKinds()) {
        jsonGenerator.writeString(rowKind.name());
    }
    jsonGenerator.writeEndArray();
}

Example 9 with RowKind

use of org.apache.flink.types.RowKind in project flink by apache.

the class DeduplicateFunctionHelper method processLastRowOnChangelog.

/**
 * Processes element to deduplicate on keys, sends current element as last row, retracts
 * previous element if needed.
 *
 * <p>Note: we don't support stateless mode yet. Because this is not safe for Kafka tombstone
 * messages which doesn't contain full content. This can be a future improvement if the
 * downstream (e.g. sink) doesn't require full content for DELETE messages.
 *
 * @param currentRow latest row received by deduplicate function
 * @param generateUpdateBefore whether need to send UPDATE_BEFORE message for updates
 * @param state state of function
 * @param out underlying collector
 */
static void processLastRowOnChangelog(RowData currentRow, boolean generateUpdateBefore, ValueState<RowData> state, Collector<RowData> out, boolean isStateTtlEnabled, RecordEqualiser equaliser) throws Exception {
    RowData preRow = state.value();
    RowKind currentKind = currentRow.getRowKind();
    if (currentKind == RowKind.INSERT || currentKind == RowKind.UPDATE_AFTER) {
        if (preRow == null) {
            // the first row, send INSERT message
            currentRow.setRowKind(RowKind.INSERT);
            out.collect(currentRow);
        } else {
            if (!isStateTtlEnabled && equaliser.equals(preRow, currentRow)) {
                // state eviction of downstream operators.
                return;
            } else {
                if (generateUpdateBefore) {
                    preRow.setRowKind(RowKind.UPDATE_BEFORE);
                    out.collect(preRow);
                }
                currentRow.setRowKind(RowKind.UPDATE_AFTER);
                out.collect(currentRow);
            }
        }
        // normalize row kind
        currentRow.setRowKind(RowKind.INSERT);
        // save to state
        state.update(currentRow);
    } else {
        // DELETE or UPDATER_BEFORE
        if (preRow != null) {
            // always set to DELETE because this row has been removed
            // even the input is UPDATE_BEFORE, there may no UPDATE_AFTER after it.
            preRow.setRowKind(RowKind.DELETE);
            // output the preRow instead of currentRow,
            // because preRow always contains the full content.
            // currentRow may only contain key parts (e.g. Kafka tombstone records).
            out.collect(preRow);
            // clear state as the row has been removed
            state.clear();
        }
    // nothing to do if removing a non-existed row
    }
}

Example 10 with RowKind

use of org.apache.flink.types.RowKind in project flink by apache.

the class DeduplicateFunctionHelper method updateDeduplicateResult.

/**
 * Collect the updated result for duplicate row.
 *
 * @param generateUpdateBefore flag to generate UPDATE_BEFORE message or not
 * @param generateInsert flag to generate INSERT message or not
 * @param preRow previous row under the key
 * @param currentRow current row under the key which is the duplicate row
 * @param out underlying collector
 */
static void updateDeduplicateResult(boolean generateUpdateBefore, boolean generateInsert, RowData preRow, RowData currentRow, Collector<RowData> out) {
    if (generateUpdateBefore || generateInsert) {
        if (preRow == null) {
            // the first row, send INSERT message
            currentRow.setRowKind(RowKind.INSERT);
            out.collect(currentRow);
        } else {
            if (generateUpdateBefore) {
                final RowKind preRowKind = preRow.getRowKind();
                preRow.setRowKind(RowKind.UPDATE_BEFORE);
                out.collect(preRow);
                preRow.setRowKind(preRowKind);
            }
            currentRow.setRowKind(RowKind.UPDATE_AFTER);
            out.collect(currentRow);
        }
    } else {
        currentRow.setRowKind(RowKind.UPDATE_AFTER);
        out.collect(currentRow);
    }
}