Examples with DataOutputViewStreamWrapper org.apache.flink.core.memory.DataOutputViewStreamWrapper used on opensource projects

Example 71 with DataOutputViewStreamWrapper

use of org.apache.flink.core.memory.DataOutputViewStreamWrapper in project flink by apache.

the class SocketStreamIteratorTest method testIterator.

@Test
public void testIterator() throws Exception {
    final AtomicReference<Throwable> error = new AtomicReference<>();
    final long seed = new Random().nextLong();
    final int numElements = 1000;
    final SocketStreamIterator<Long> iterator = new SocketStreamIterator<>(LongSerializer.INSTANCE);
    Thread writer = new Thread() {

        @Override
        public void run() {
            try {
                try (Socket sock = new Socket(iterator.getBindAddress(), iterator.getPort());
                    DataOutputViewStreamWrapper out = new DataOutputViewStreamWrapper(sock.getOutputStream())) {
                    final TypeSerializer<Long> serializer = LongSerializer.INSTANCE;
                    final Random rnd = new Random(seed);
                    for (int i = 0; i < numElements; i++) {
                        serializer.serialize(rnd.nextLong(), out);
                    }
                }
            } catch (Throwable t) {
                error.set(t);
            }
        }
    };
    writer.start();
    final Random validator = new Random(seed);
    for (int i = 0; i < numElements; i++) {
        assertTrue(iterator.hasNext());
        assertTrue(iterator.hasNext());
        assertEquals(validator.nextLong(), iterator.next().longValue());
    }
    assertFalse(iterator.hasNext());
    writer.join();
    assertFalse(iterator.hasNext());
}

Example 72 with DataOutputViewStreamWrapper

use of org.apache.flink.core.memory.DataOutputViewStreamWrapper in project flink by apache.

the class DefaultOperatorStateBackend method snapshot.

@Override
public RunnableFuture<OperatorStateHandle> snapshot(long checkpointId, long timestamp, CheckpointStreamFactory streamFactory, CheckpointOptions checkpointOptions) throws Exception {
    if (registeredStates.isEmpty()) {
        return DoneFuture.nullValue();
    }
    List<OperatorBackendSerializationProxy.StateMetaInfo<?>> metaInfoList = new ArrayList<>(registeredStates.size());
    for (Map.Entry<String, PartitionableListState<?>> entry : registeredStates.entrySet()) {
        PartitionableListState<?> state = entry.getValue();
        OperatorBackendSerializationProxy.StateMetaInfo<?> metaInfo = new OperatorBackendSerializationProxy.StateMetaInfo<>(state.getName(), state.getPartitionStateSerializer(), state.getAssignmentMode());
        metaInfoList.add(metaInfo);
    }
    Map<String, OperatorStateHandle.StateMetaInfo> writtenStatesMetaData = new HashMap<>(registeredStates.size());
    CheckpointStreamFactory.CheckpointStateOutputStream out = streamFactory.createCheckpointStateOutputStream(checkpointId, timestamp);
    try {
        closeStreamOnCancelRegistry.registerClosable(out);
        DataOutputView dov = new DataOutputViewStreamWrapper(out);
        OperatorBackendSerializationProxy backendSerializationProxy = new OperatorBackendSerializationProxy(metaInfoList);
        backendSerializationProxy.write(dov);
        dov.writeInt(registeredStates.size());
        for (Map.Entry<String, PartitionableListState<?>> entry : registeredStates.entrySet()) {
            PartitionableListState<?> value = entry.getValue();
            long[] partitionOffsets = value.write(out);
            OperatorStateHandle.Mode mode = value.getAssignmentMode();
            writtenStatesMetaData.put(entry.getKey(), new OperatorStateHandle.StateMetaInfo(partitionOffsets, mode));
        }
        OperatorStateHandle handle = new OperatorStateHandle(writtenStatesMetaData, out.closeAndGetHandle());
        return new DoneFuture<>(handle);
    } finally {
        closeStreamOnCancelRegistry.unregisterClosable(out);
        out.close();
    }
}

Example 73 with DataOutputViewStreamWrapper

use of org.apache.flink.core.memory.DataOutputViewStreamWrapper in project flink by apache.

the class StateTableSnapshotCompatibilityTest method restoreStateTableFromSnapshot.

private static <K, N, S> void restoreStateTableFromSnapshot(StateTable<K, N, S> stateTable, StateTableSnapshot snapshot, KeyGroupRange keyGroupRange) throws IOException {
    final ByteArrayOutputStreamWithPos out = new ByteArrayOutputStreamWithPos(1024 * 1024);
    final DataOutputViewStreamWrapper dov = new DataOutputViewStreamWrapper(out);
    for (Integer keyGroup : keyGroupRange) {
        snapshot.writeMappingsInKeyGroup(dov, keyGroup);
    }
    final ByteArrayInputStreamWithPos in = new ByteArrayInputStreamWithPos(out.getBuf());
    final DataInputViewStreamWrapper div = new DataInputViewStreamWrapper(in);
    final StateTableByKeyGroupReader keyGroupReader = StateTableByKeyGroupReaders.readerForVersion(stateTable, KeyedBackendSerializationProxy.VERSION);
    for (Integer keyGroup : keyGroupRange) {
        keyGroupReader.readMappingsInKeyGroup(div, keyGroup);
    }
}

Example 74 with DataOutputViewStreamWrapper

use of org.apache.flink.core.memory.DataOutputViewStreamWrapper in project flink by apache.

the class AbstractAlignedProcessingTimeWindowOperator method snapshotState.

// ------------------------------------------------------------------------
//  Checkpointing
// ------------------------------------------------------------------------
@Override
public void snapshotState(FSDataOutputStream out, long checkpointId, long timestamp) throws Exception {
    super.snapshotState(out, checkpointId, timestamp);
    // we write the panes with the key/value maps into the stream, as well as when this state
    // should have triggered and slided
    DataOutputViewStreamWrapper outView = new DataOutputViewStreamWrapper(out);
    outView.writeLong(nextEvaluationTime);
    outView.writeLong(nextSlideTime);
    panes.writeToOutput(outView, keySerializer, stateTypeSerializer);
    outView.flush();
}

Example 75 with DataOutputViewStreamWrapper

use of org.apache.flink.core.memory.DataOutputViewStreamWrapper in project flink by apache.

the class HeapInternalTimerServiceTest method testSnapshotAndRestore.

@Test
public void testSnapshotAndRestore() throws Exception {
    @SuppressWarnings("unchecked") Triggerable<Integer, String> mockTriggerable = mock(Triggerable.class);
    TestKeyContext keyContext = new TestKeyContext();
    TestProcessingTimeService processingTimeService = new TestProcessingTimeService();
    HeapInternalTimerService<Integer, String> timerService = createTimerService(mockTriggerable, keyContext, processingTimeService, testKeyGroupRange, maxParallelism);
    // get two different keys
    int key1 = getKeyInKeyGroupRange(testKeyGroupRange, maxParallelism);
    int key2 = getKeyInKeyGroupRange(testKeyGroupRange, maxParallelism);
    while (key2 == key1) {
        key2 = getKeyInKeyGroupRange(testKeyGroupRange, maxParallelism);
    }
    keyContext.setCurrentKey(key1);
    timerService.registerProcessingTimeTimer("ciao", 10);
    timerService.registerEventTimeTimer("hello", 10);
    keyContext.setCurrentKey(key2);
    timerService.registerEventTimeTimer("ciao", 10);
    timerService.registerProcessingTimeTimer("hello", 10);
    assertEquals(2, timerService.numProcessingTimeTimers());
    assertEquals(1, timerService.numProcessingTimeTimers("hello"));
    assertEquals(1, timerService.numProcessingTimeTimers("ciao"));
    assertEquals(2, timerService.numEventTimeTimers());
    assertEquals(1, timerService.numEventTimeTimers("hello"));
    assertEquals(1, timerService.numEventTimeTimers("ciao"));
    Map<Integer, byte[]> snapshot = new HashMap<>();
    for (Integer keyGroupIndex : testKeyGroupRange) {
        ByteArrayOutputStream outStream = new ByteArrayOutputStream();
        timerService.snapshotTimersForKeyGroup(new DataOutputViewStreamWrapper(outStream), keyGroupIndex);
        outStream.close();
        snapshot.put(keyGroupIndex, outStream.toByteArray());
    }
    @SuppressWarnings("unchecked") Triggerable<Integer, String> mockTriggerable2 = mock(Triggerable.class);
    keyContext = new TestKeyContext();
    processingTimeService = new TestProcessingTimeService();
    timerService = restoreTimerService(snapshot, mockTriggerable2, keyContext, processingTimeService, testKeyGroupRange, maxParallelism);
    processingTimeService.setCurrentTime(10);
    timerService.advanceWatermark(10);
    verify(mockTriggerable2, times(2)).onProcessingTime(anyInternalTimer());
    verify(mockTriggerable2, times(1)).onProcessingTime(eq(new InternalTimer<>(10, key1, "ciao")));
    verify(mockTriggerable2, times(1)).onProcessingTime(eq(new InternalTimer<>(10, key2, "hello")));
    verify(mockTriggerable2, times(2)).onEventTime(anyInternalTimer());
    verify(mockTriggerable2, times(1)).onEventTime(eq(new InternalTimer<>(10, key1, "hello")));
    verify(mockTriggerable2, times(1)).onEventTime(eq(new InternalTimer<>(10, key2, "ciao")));
    assertEquals(0, timerService.numEventTimeTimers());
}