Examples with GenericRowData org.apache.flink.table.data.GenericRowData used on opensource projects

Example 41 with GenericRowData

use of org.apache.flink.table.data.GenericRowData in project flink by apache.

the class SortMergeJoinOperator method open.

@Override
public void open() throws Exception {
    super.open();
    Configuration conf = getContainingTask().getJobConfiguration();
    isFinished = new boolean[] { false, false };
    collector = new StreamRecordCollector<>(output);
    ClassLoader cl = getUserCodeClassloader();
    AbstractRowDataSerializer inputSerializer1 = (AbstractRowDataSerializer) getOperatorConfig().getTypeSerializerIn1(cl);
    this.serializer1 = new BinaryRowDataSerializer(inputSerializer1.getArity());
    AbstractRowDataSerializer inputSerializer2 = (AbstractRowDataSerializer) getOperatorConfig().getTypeSerializerIn2(cl);
    this.serializer2 = new BinaryRowDataSerializer(inputSerializer2.getArity());
    this.memManager = this.getContainingTask().getEnvironment().getMemoryManager();
    this.ioManager = this.getContainingTask().getEnvironment().getIOManager();
    long totalMemory = computeMemorySize();
    externalBufferMemory = (long) (totalMemory * externalBufferMemRatio);
    externalBufferMemory = Math.max(externalBufferMemory, ResettableExternalBuffer.MIN_NUM_MEMORY);
    long totalSortMem = totalMemory - (type.equals(FlinkJoinType.FULL) ? externalBufferMemory * 2 : externalBufferMemory);
    if (totalSortMem < 0) {
        throw new TableException("Memory size is too small: " + totalMemory + ", please increase manage memory of task manager.");
    }
    // sorter1
    this.sorter1 = new BinaryExternalSorter(this.getContainingTask(), memManager, totalSortMem / 2, ioManager, inputSerializer1, serializer1, computer1.newInstance(cl), comparator1.newInstance(cl), conf);
    this.sorter1.startThreads();
    // sorter2
    this.sorter2 = new BinaryExternalSorter(this.getContainingTask(), memManager, totalSortMem / 2, ioManager, inputSerializer2, serializer2, computer2.newInstance(cl), comparator2.newInstance(cl), conf);
    this.sorter2.startThreads();
    keyComparator = genKeyComparator.newInstance(cl);
    this.condFunc = condFuncCode.newInstance(cl);
    condFunc.setRuntimeContext(getRuntimeContext());
    condFunc.open(new Configuration());
    projection1 = projectionCode1.newInstance(cl);
    projection2 = projectionCode2.newInstance(cl);
    this.leftNullRow = new GenericRowData(serializer1.getArity());
    this.rightNullRow = new GenericRowData(serializer2.getArity());
    this.joinedRow = new JoinedRowData();
    condFuncCode = null;
    computer1 = null;
    comparator1 = null;
    computer2 = null;
    comparator2 = null;
    projectionCode1 = null;
    projectionCode2 = null;
    genKeyComparator = null;
    getMetricGroup().gauge("memoryUsedSizeInBytes", (Gauge<Long>) () -> sorter1.getUsedMemoryInBytes() + sorter2.getUsedMemoryInBytes());
    getMetricGroup().gauge("numSpillFiles", (Gauge<Long>) () -> sorter1.getNumSpillFiles() + sorter2.getNumSpillFiles());
    getMetricGroup().gauge("spillInBytes", (Gauge<Long>) () -> sorter1.getSpillInBytes() + sorter2.getSpillInBytes());
}

Example 42 with GenericRowData

use of org.apache.flink.table.data.GenericRowData in project flink by apache.

the class AvroToRowDataConverters method createRowConverter.

// -------------------------------------------------------------------------------------
// Runtime Converters
// -------------------------------------------------------------------------------------
public static AvroToRowDataConverter createRowConverter(RowType rowType) {
    final AvroToRowDataConverter[] fieldConverters = rowType.getFields().stream().map(RowType.RowField::getType).map(AvroToRowDataConverters::createNullableConverter).toArray(AvroToRowDataConverter[]::new);
    final int arity = rowType.getFieldCount();
    return avroObject -> {
        IndexedRecord record = (IndexedRecord) avroObject;
        GenericRowData row = new GenericRowData(arity);
        for (int i = 0; i < arity; ++i) {
            // avro always deserialize successfully even though the type isn't matched
            // so no need to throw exception about which field can't be deserialized
            row.setField(i, fieldConverters[i].convert(record.get(i)));
        }
        return row;
    };
}

Example 43 with GenericRowData

use of org.apache.flink.table.data.GenericRowData in project flink by apache.

the class RegistryAvroRowDataSeDeSchemaTest method address2RowData.

private static RowData address2RowData(Address address) {
    GenericRowData rowData = new GenericRowData(5);
    rowData.setField(0, address.getNum());
    rowData.setField(1, new BinaryStringData(address.getStreet().toString()));
    rowData.setField(2, new BinaryStringData(address.getCity().toString()));
    rowData.setField(3, new BinaryStringData(address.getState().toString()));
    rowData.setField(4, new BinaryStringData(address.getZip().toString()));
    return rowData;
}

Example 44 with GenericRowData

use of org.apache.flink.table.data.GenericRowData in project flink by apache.

the class StreamArrowPythonGroupWindowAggregateFunctionOperator method open.

@Override
public void open() throws Exception {
    super.open();
    windowSerializer = windowAssigner.getWindowSerializer(new ExecutionConfig());
    internalTimerService = getInternalTimerService("window-timers", windowSerializer, this);
    triggerContext = new TriggerContext();
    triggerContext.open();
    StateDescriptor<ListState<RowData>, List<RowData>> windowStateDescriptor = new ListStateDescriptor<>("window-input", new RowDataSerializer(inputType));
    StateDescriptor<ListState<RowData>, List<RowData>> dataRetractStateDescriptor = new ListStateDescriptor<>("data-retract", new RowDataSerializer(inputType));
    this.windowAccumulateData = (InternalListState<K, W, RowData>) getOrCreateKeyedState(windowSerializer, windowStateDescriptor);
    this.windowRetractData = (InternalListState<K, W, RowData>) getOrCreateKeyedState(windowSerializer, dataRetractStateDescriptor);
    inputKeyAndWindow = new LinkedList<>();
    windowProperty = new GenericRowData(namedProperties.length);
    windowAggResult = new JoinedRowData();
    WindowContext windowContext = new WindowContext();
    windowAssigner.open(windowContext);
}

Example 45 with GenericRowData

use of org.apache.flink.table.data.GenericRowData in project flink by apache.

the class PythonStreamGroupWindowAggregateOperator method emitResult.

@Override
public void emitResult(Tuple2<byte[], Integer> resultTuple) throws Exception {
    byte[] rawUdfResult = resultTuple.f0;
    int length = resultTuple.f1;
    bais.setBuffer(rawUdfResult, 0, length);
    RowData udfResult = udfOutputTypeSerializer.deserialize(baisWrapper);
    byte recordType = udfResult.getByte(0);
    if (recordType == NORMAL_RECORD) {
        GenericRowData aggResult = (GenericRowData) udfResult.getRow(1, outputType.getFieldCount());
        int fieldCount = outputType.getFieldCount();
        for (int i = fieldCount - namedProperties.length; i < fieldCount; i++) {
            FlinkFnApi.GroupWindow.WindowProperty namedProperty = namedProperties[i - (fieldCount - namedProperties.length)];
            if (namedProperty == WINDOW_START || namedProperty == WINDOW_END) {
                aggResult.setField(i, TimestampData.fromEpochMillis(aggResult.getLong(i)));
            } else {
                aggResult.setField(i, TimestampData.fromEpochMillis(getShiftEpochMills(aggResult.getLong(i))));
            }
        }
        rowDataWrapper.collect(aggResult);
    } else {
        RowData timerData = udfResult.getRow(2, timerDataLength);
        byte timerOperandType = timerData.getByte(0);
        RowData key = timerData.getRow(1, keyLength);
        long timestamp = timerData.getLong(2);
        W window;
        byte[] encodedNamespace = timerData.getBinary(3);
        bais.setBuffer(encodedNamespace, 0, encodedNamespace.length);
        window = windowSerializer.deserialize(baisWrapper);
        BinaryRowData rowKey = keySerializer.toBinaryRow(key).copy();
        synchronized (getKeyedStateBackend()) {
            setCurrentKey(rowKey);
            if (timerOperandType == REGISTER_EVENT_TIMER) {
                internalTimerService.registerEventTimeTimer(window, toEpochMillsForTimer(timestamp, shiftTimeZone));
            } else if (timerOperandType == REGISTER_PROCESSING_TIMER) {
                internalTimerService.registerProcessingTimeTimer(window, toEpochMillsForTimer(timestamp, shiftTimeZone));
            } else if (timerOperandType == DELETE_EVENT_TIMER) {
                internalTimerService.deleteEventTimeTimer(window, toEpochMillsForTimer(timestamp, shiftTimeZone));
            } else if (timerOperandType == DELETE_PROCESSING_TIMER) {
                internalTimerService.deleteProcessingTimeTimer(window, toEpochMillsForTimer(timestamp, shiftTimeZone));
            } else {
                throw new RuntimeException(String.format("Unsupported timerOperandType %s.", timerOperandType));
            }
        }
    }
}