Examples with RowKind org.apache.flink.types.RowKind used on opensource projects

Example 11 with RowKind

use of org.apache.flink.types.RowKind in project flink by apache.

the class RowSerializer method deserialize.

@Override
public Row deserialize(DataInputView source) throws IOException {
    final int length = fieldSerializers.length;
    // read bitmask
    readIntoMask(source, mask);
    // read row kind
    final RowKind kind;
    if (!supportsRowKind) {
        kind = RowKind.INSERT;
    } else {
        kind = readKindFromMask(mask);
    }
    // deserialize fields
    final Object[] fieldByPosition = new Object[length];
    for (int fieldPos = 0; fieldPos < length; fieldPos++) {
        if (!mask[rowKindOffset + fieldPos]) {
            fieldByPosition[fieldPos] = fieldSerializers[fieldPos].deserialize(source);
        }
    }
    return RowUtils.createRowWithNamedPositions(kind, fieldByPosition, positionByName);
}

Example 12 with RowKind

use of org.apache.flink.types.RowKind in project flink by apache.

the class InputConversionOperator method processElement.

@Override
public void processElement(StreamRecord<E> element) throws Exception {
    final E externalRecord = element.getValue();
    final Object internalRecord;
    try {
        internalRecord = converter.toInternal(externalRecord);
    } catch (Exception e) {
        throw new FlinkRuntimeException(String.format("Error during input conversion from external DataStream API to " + "internal Table API data structures. Make sure that the " + "provided data types that configure the converters are " + "correctly declared in the schema. Affected record:\n%s", externalRecord), e);
    }
    final RowData payloadRowData;
    if (requiresWrapping) {
        final GenericRowData wrapped = new GenericRowData(RowKind.INSERT, 1);
        wrapped.setField(0, internalRecord);
        payloadRowData = wrapped;
    } else {
        // top-level records must not be null and will be skipped
        if (internalRecord == null) {
            return;
        }
        payloadRowData = (RowData) internalRecord;
    }
    final RowKind kind = payloadRowData.getRowKind();
    if (isInsertOnly && kind != RowKind.INSERT) {
        throw new FlinkRuntimeException(String.format("Error during input conversion. Conversion expects insert-only " + "records but DataStream API record contains: %s", kind));
    }
    if (!produceRowtimeMetadata) {
        output.collect(outRecord.replace(payloadRowData));
        return;
    }
    if (!element.hasTimestamp()) {
        throw new FlinkRuntimeException("Could not find timestamp in DataStream API record. " + "Make sure that timestamps have been assigned before and " + "the event-time characteristic is enabled.");
    }
    final GenericRowData rowtimeRowData = new GenericRowData(1);
    rowtimeRowData.setField(0, TimestampData.fromEpochMillis(element.getTimestamp()));
    final JoinedRowData joinedRowData = new JoinedRowData(kind, payloadRowData, rowtimeRowData);
    output.collect(outRecord.replace(joinedRowData));
}

Example 13 with RowKind

use of org.apache.flink.types.RowKind in project flink by apache.

the class StreamingSemiAntiJoinOperator method processElement2.

/**
 * Process an input element and output incremental joined records, retraction messages will be
 * sent in some scenarios.
 *
 * <p>Following is the pseudo code to describe the core logic of this method.
 *
 * <p>Note: "+I" represents "INSERT", "-D" represents "DELETE", "+U" represents "UPDATE_AFTER",
 * "-U" represents "UPDATE_BEFORE".
 *
 * <pre>
 * if input record is accumulate
 * | state.add(record)
 * | if there is no matched rows on the other side, skip
 * | if there are matched rows on the other side
 * | | if the matched num in the matched rows == 0
 * | |   if anti join, send -D[other]s
 * | |   if semi join, send +I/+U[other]s (using input RowKind)
 * | | if the matched num in the matched rows > 0, skip
 * | | otherState.update(other, old+1)
 * | endif
 * endif
 * if input record is retract
 * | state.retract(record)
 * | if there is no matched rows on the other side, skip
 * | if there are matched rows on the other side
 * | | if the matched num in the matched rows == 0, this should never happen!
 * | | if the matched num in the matched rows == 1
 * | |   if semi join, send -D/-U[other] (using input RowKind)
 * | |   if anti join, send +I[other]
 * | | if the matched num in the matched rows > 1, skip
 * | | otherState.update(other, old-1)
 * | endif
 * endif
 * </pre>
 */
@Override
public void processElement2(StreamRecord<RowData> element) throws Exception {
    RowData input = element.getValue();
    boolean isAccumulateMsg = RowDataUtil.isAccumulateMsg(input);
    RowKind inputRowKind = input.getRowKind();
    // erase RowKind for later state updating
    input.setRowKind(RowKind.INSERT);
    AssociatedRecords associatedRecords = AssociatedRecords.of(input, false, leftRecordStateView, joinCondition);
    if (isAccumulateMsg) {
        // record is accumulate
        rightRecordStateView.addRecord(input);
        if (!associatedRecords.isEmpty()) {
            // there are matched rows on the other side
            for (OuterRecord outerRecord : associatedRecords.getOuterRecords()) {
                RowData other = outerRecord.record;
                if (outerRecord.numOfAssociations == 0) {
                    if (isAntiJoin) {
                        // send -D[other]
                        other.setRowKind(RowKind.DELETE);
                    } else {
                        // send +I/+U[other] (using input RowKind)
                        other.setRowKind(inputRowKind);
                    }
                    collector.collect(other);
                    // set header back to INSERT, because we will update the other row to state
                    other.setRowKind(RowKind.INSERT);
                }
                // ignore when number > 0
                leftRecordStateView.updateNumOfAssociations(other, outerRecord.numOfAssociations + 1);
            }
        }
    // ignore when associated number == 0
    } else {
        // retract input
        rightRecordStateView.retractRecord(input);
        if (!associatedRecords.isEmpty()) {
            // there are matched rows on the other side
            for (OuterRecord outerRecord : associatedRecords.getOuterRecords()) {
                RowData other = outerRecord.record;
                if (outerRecord.numOfAssociations == 1) {
                    if (!isAntiJoin) {
                        // send -D/-U[other] (using input RowKind)
                        other.setRowKind(inputRowKind);
                    } else {
                        // send +I[other]
                        other.setRowKind(RowKind.INSERT);
                    }
                    collector.collect(other);
                    // set RowKind back, because we will update the other row to state
                    other.setRowKind(RowKind.INSERT);
                }
                // ignore when number > 0
                leftRecordStateView.updateNumOfAssociations(other, outerRecord.numOfAssociations - 1);
            }
        }
    // ignore when associated number == 0
    }
}

Example 14 with RowKind

use of org.apache.flink.types.RowKind in project flink by apache.

the class BuiltInAggregateFunctionTestBase method materializeResult.

private static List<Row> materializeResult(TableResult tableResult) {
    try (final CloseableIterator<Row> iterator = tableResult.collect()) {
        final List<Row> actualRows = new ArrayList<>();
        iterator.forEachRemaining(row -> {
            final RowKind kind = row.getKind();
            switch(kind) {
                case INSERT:
                case UPDATE_AFTER:
                    row.setKind(RowKind.INSERT);
                    actualRows.add(row);
                    break;
                case UPDATE_BEFORE:
                case DELETE:
                    row.setKind(RowKind.INSERT);
                    actualRows.remove(row);
                    break;
            }
        });
        return actualRows;
    } catch (Exception e) {
        throw new RuntimeException("Could not collect results", e);
    }
}