Examples with DataInputViewStreamWrapper org.apache.flink.core.memory.DataInputViewStreamWrapper used on opensource projects

Example 21 with DataInputViewStreamWrapper

use of org.apache.flink.core.memory.DataInputViewStreamWrapper in project flink by apache.

the class ContinuousFileReaderOperator method restoreState.

// ------------------------------------------------------------------------
//  Restoring / Migrating from an older Flink version.
// ------------------------------------------------------------------------
@Override
public void restoreState(FSDataInputStream in) throws Exception {
    LOG.info("{} (taskIdx={}) restoring state from an older Flink version.", getClass().getSimpleName(), getRuntimeContext().getIndexOfThisSubtask());
    // this is just to read the byte indicating if we have udf state or not
    int hasUdfState = in.read();
    Preconditions.checkArgument(hasUdfState == 0);
    final ObjectInputStream ois = new ObjectInputStream(in);
    final DataInputViewStreamWrapper div = new DataInputViewStreamWrapper(in);
    // read the split that was being read
    FileInputSplit currSplit = (FileInputSplit) ois.readObject();
    // read the pending splits list
    List<FileInputSplit> pendingSplits = new LinkedList<>();
    int noOfSplits = div.readInt();
    for (int i = 0; i < noOfSplits; i++) {
        FileInputSplit split = (FileInputSplit) ois.readObject();
        pendingSplits.add(split);
    }
    // read the state of the format
    Serializable formatState = (Serializable) ois.readObject();
    div.close();
    if (restoredReaderState == null) {
        restoredReaderState = new ArrayList<>();
    }
    // we do not know the modification time of the retrieved splits, so we assign them
    // artificial ones, with the only constraint that they respect the relative order of the
    // retrieved splits, because modification time is going to be used to sort the splits within
    // the "pending splits" priority queue.
    long now = getProcessingTimeService().getCurrentProcessingTime();
    long runningModTime = Math.max(now, noOfSplits + 1);
    TimestampedFileInputSplit currentSplit = createTimestampedFileSplit(currSplit, --runningModTime, formatState);
    restoredReaderState.add(currentSplit);
    for (FileInputSplit split : pendingSplits) {
        TimestampedFileInputSplit timestampedSplit = createTimestampedFileSplit(split, --runningModTime);
        restoredReaderState.add(timestampedSplit);
    }
    if (LOG.isDebugEnabled()) {
        if (LOG.isDebugEnabled()) {
            LOG.debug("{} (taskIdx={}) restored {} splits from legacy: {}.", getClass().getSimpleName(), getRuntimeContext().getIndexOfThisSubtask(), restoredReaderState.size(), restoredReaderState);
        }
    }
}

Example 22 with DataInputViewStreamWrapper

use of org.apache.flink.core.memory.DataInputViewStreamWrapper in project flink by apache.

the class FromElementsFunction method run.

@Override
public void run(SourceContext<T> ctx) throws Exception {
    ByteArrayInputStream bais = new ByteArrayInputStream(elementsSerialized);
    final DataInputView input = new DataInputViewStreamWrapper(bais);
    // if we are restored from a checkpoint and need to skip elements, skip them now.
    int toSkip = numElementsToSkip;
    if (toSkip > 0) {
        try {
            while (toSkip > 0) {
                serializer.deserialize(input);
                toSkip--;
            }
        } catch (Exception e) {
            throw new IOException("Failed to deserialize an element from the source. " + "If you are using user-defined serialization (Value and Writable types), check the " + "serialization functions.\nSerializer is " + serializer);
        }
        this.numElementsEmitted = this.numElementsToSkip;
    }
    final Object lock = ctx.getCheckpointLock();
    while (isRunning && numElementsEmitted < numElements) {
        T next;
        try {
            next = serializer.deserialize(input);
        } catch (Exception e) {
            throw new IOException("Failed to deserialize an element from the source. " + "If you are using user-defined serialization (Value and Writable types), check the " + "serialization functions.\nSerializer is " + serializer);
        }
        synchronized (lock) {
            ctx.collect(next);
            numElementsEmitted++;
        }
    }
}

Example 23 with DataInputViewStreamWrapper

use of org.apache.flink.core.memory.DataInputViewStreamWrapper in project flink by apache.

the class FoldApplyAllWindowFunction method open.

@Override
public void open(Configuration configuration) throws Exception {
    super.open(configuration);
    if (accSerializer == null) {
        throw new RuntimeException("No serializer set for the fold accumulator type. " + "Probably the setOutputType method was not called.");
    }
    if (serializedInitialValue == null) {
        throw new RuntimeException("No initial value was serialized for the fold " + "window function. Probably the setOutputType method was not called.");
    }
    ByteArrayInputStream bais = new ByteArrayInputStream(serializedInitialValue);
    DataInputViewStreamWrapper in = new DataInputViewStreamWrapper(bais);
    initialValue = accSerializer.deserialize(in);
}

Example 24 with DataInputViewStreamWrapper

use of org.apache.flink.core.memory.DataInputViewStreamWrapper in project flink by apache.

the class FoldApplyProcessAllWindowFunction method open.

@Override
public void open(Configuration configuration) throws Exception {
    FunctionUtils.openFunction(this.windowFunction, configuration);
    if (serializedInitialValue == null) {
        throw new RuntimeException("No initial value was serialized for the fold " + "window function. Probably the setOutputType method was not called.");
    }
    ByteArrayInputStream bais = new ByteArrayInputStream(serializedInitialValue);
    DataInputViewStreamWrapper in = new DataInputViewStreamWrapper(bais);
    initialValue = accSerializer.deserialize(in);
}

Example 25 with DataInputViewStreamWrapper

use of org.apache.flink.core.memory.DataInputViewStreamWrapper in project flink by apache.

the class FoldApplyWindowFunction method open.

@Override
public void open(Configuration configuration) throws Exception {
    super.open(configuration);
    if (accSerializer == null) {
        throw new RuntimeException("No serializer set for the fold accumulator type. " + "Probably the setOutputType method was not called.");
    }
    if (serializedInitialValue == null) {
        throw new RuntimeException("No initial value was serialized for the fold " + "window function. Probably the setOutputType method was not called.");
    }
    ByteArrayInputStream bais = new ByteArrayInputStream(serializedInitialValue);
    DataInputViewStreamWrapper in = new DataInputViewStreamWrapper(bais);
    initialValue = accSerializer.deserialize(in);
}