Examples with DataInputViewStreamWrapper org.apache.flink.core.memory.DataInputViewStreamWrapper used on opensource projects

Example 66 with DataInputViewStreamWrapper

use of org.apache.flink.core.memory.DataInputViewStreamWrapper in project flink by apache.

the class StreamGroupedFold method open.

@Override
public void open() throws Exception {
    super.open();
    if (serializedInitialValue == null) {
        throw new RuntimeException("No initial value was serialized for the fold " + "operator. Probably the setOutputType method was not called.");
    }
    try (ByteArrayInputStream bais = new ByteArrayInputStream(serializedInitialValue);
        DataInputViewStreamWrapper in = new DataInputViewStreamWrapper(bais)) {
        initialValue = outTypeSerializer.deserialize(in);
    }
    ValueStateDescriptor<OUT> stateId = new ValueStateDescriptor<>(STATE_NAME, outTypeSerializer);
    values = getPartitionedState(stateId);
}

Example 67 with DataInputViewStreamWrapper

use of org.apache.flink.core.memory.DataInputViewStreamWrapper in project flink by apache.

the class WindowOperator method restoreState.

@Override
public void restoreState(FSDataInputStream in) throws Exception {
    super.restoreState(in);
    LOG.info("{} (taskIdx={}) restoring {} state from an older Flink version.", getClass().getSimpleName(), legacyWindowOperatorType, getRuntimeContext().getIndexOfThisSubtask());
    DataInputViewStreamWrapper streamWrapper = new DataInputViewStreamWrapper(in);
    switch(legacyWindowOperatorType) {
        case NONE:
            restoreFromLegacyWindowOperator(streamWrapper);
            break;
        case FAST_ACCUMULATING:
        case FAST_AGGREGATING:
            restoreFromLegacyAlignedWindowOperator(streamWrapper);
            break;
    }
}

Example 68 with DataInputViewStreamWrapper

use of org.apache.flink.core.memory.DataInputViewStreamWrapper in project flink by apache.

the class GenericWriteAheadSink method notifyOfCompletedCheckpoint.

@Override
public void notifyOfCompletedCheckpoint(long checkpointId) throws Exception {
    super.notifyOfCompletedCheckpoint(checkpointId);
    synchronized (pendingCheckpoints) {
        Iterator<PendingCheckpoint> pendingCheckpointIt = pendingCheckpoints.iterator();
        while (pendingCheckpointIt.hasNext()) {
            PendingCheckpoint pendingCheckpoint = pendingCheckpointIt.next();
            long pastCheckpointId = pendingCheckpoint.checkpointId;
            int subtaskId = pendingCheckpoint.subtaskId;
            long timestamp = pendingCheckpoint.timestamp;
            StreamStateHandle streamHandle = pendingCheckpoint.stateHandle;
            if (pastCheckpointId <= checkpointId) {
                try {
                    if (!committer.isCheckpointCommitted(subtaskId, pastCheckpointId)) {
                        try (FSDataInputStream in = streamHandle.openInputStream()) {
                            boolean success = sendValues(new ReusingMutableToRegularIteratorWrapper<>(new InputViewIterator<>(new DataInputViewStreamWrapper(in), serializer), serializer), timestamp);
                            if (success) {
                                // in case the checkpoint was successfully committed,
                                // discard its state from the backend and mark it for removal
                                // in case it failed, we retry on the next checkpoint
                                committer.commitCheckpoint(subtaskId, pastCheckpointId);
                                streamHandle.discardState();
                                pendingCheckpointIt.remove();
                            }
                        }
                    } else {
                        streamHandle.discardState();
                        pendingCheckpointIt.remove();
                    }
                } catch (Exception e) {
                    // we have to break here to prevent a new (later) checkpoint
                    // from being committed before this one
                    LOG.error("Could not commit checkpoint.", e);
                    break;
                }
            }
        }
    }
}

Example 69 with DataInputViewStreamWrapper

use of org.apache.flink.core.memory.DataInputViewStreamWrapper in project flink by apache.

the class StateInitializationContextImplTest method getKeyedStateStreams.

@Test
public void getKeyedStateStreams() throws Exception {
    int readKeyGroupCount = 0;
    for (KeyGroupStatePartitionStreamProvider stateStreamProvider : initializationContext.getRawKeyedStateInputs()) {
        Assert.assertNotNull(stateStreamProvider);
        try (InputStream is = stateStreamProvider.getStream()) {
            DataInputView div = new DataInputViewStreamWrapper(is);
            int val = div.readInt();
            ++readKeyGroupCount;
            Assert.assertEquals(stateStreamProvider.getKeyGroupId(), val);
        }
    }
    Assert.assertEquals(writtenKeyGroups, readKeyGroupCount);
}

Example 70 with DataInputViewStreamWrapper

use of org.apache.flink.core.memory.DataInputViewStreamWrapper in project flink by apache.

the class HeapInternalTimerServiceTest method restoreTimerService.

private static HeapInternalTimerService<Integer, String> restoreTimerService(Map<Integer, byte[]> state, Triggerable<Integer, String> triggerable, KeyContext keyContext, ProcessingTimeService processingTimeService, KeyGroupsList keyGroupsList, int maxParallelism) throws Exception {
    // create an empty service
    HeapInternalTimerService<Integer, String> service = new HeapInternalTimerService<>(maxParallelism, keyGroupsList, keyContext, processingTimeService);
    // restore the timers
    for (Integer keyGroupIndex : keyGroupsList) {
        if (state.containsKey(keyGroupIndex)) {
            service.restoreTimersForKeyGroup(new DataInputViewStreamWrapper(new ByteArrayInputStream(state.get(keyGroupIndex))), keyGroupIndex, HeapInternalTimerServiceTest.class.getClassLoader());
        }
    }
    // initialize the service
    service.startTimerService(IntSerializer.INSTANCE, StringSerializer.INSTANCE, triggerable);
    return service;
}