Examples with DataOutputViewStreamWrapper org.apache.flink.core.memory.DataOutputViewStreamWrapper used on opensource projects

Example 46 with DataOutputViewStreamWrapper

use of org.apache.flink.core.memory.DataOutputViewStreamWrapper in project flink by apache.

the class OutputEmitterTest method testWrongKeyClass.

@Test
public void testWrongKeyClass() throws Exception {
    // Test for IntValue
    final TypeComparator<Record> doubleComp = new RecordComparatorFactory(new int[] { 0 }, new Class[] { DoubleValue.class }).createComparator();
    final ChannelSelector<SerializationDelegate<Record>> selector = createChannelSelector(ShipStrategyType.PARTITION_HASH, doubleComp, 100);
    final SerializationDelegate<Record> delegate = new SerializationDelegate<>(new RecordSerializerFactory().getSerializer());
    PipedInputStream pipedInput = new PipedInputStream(1024 * 1024);
    DataInputView in = new DataInputViewStreamWrapper(pipedInput);
    DataOutputView out = new DataOutputViewStreamWrapper(new PipedOutputStream(pipedInput));
    Record record = new Record(1);
    record.setField(0, new IntValue());
    record.write(out);
    record = new Record();
    record.read(in);
    try {
        delegate.setInstance(record);
        selector.selectChannel(delegate);
    } catch (DeserializationException re) {
        return;
    }
    Assert.fail("Expected a NullKeyFieldException.");
}

Example 47 with DataOutputViewStreamWrapper

use of org.apache.flink.core.memory.DataOutputViewStreamWrapper in project flink by apache.

the class CopyOnWriteSkipListStateMapTestUtils method verifySnapshotWithoutTransform.

static <K, N, S> void verifySnapshotWithoutTransform(Map<N, Map<K, S>> referenceStates, @Nonnull CopyOnWriteSkipListStateMapSnapshot<K, N, S> snapshot, TypeSerializer<K> keySerializer, TypeSerializer<N> namespaceSerializer, TypeSerializer<S> stateSerializer, SnapshotVerificationMode verificationMode) throws IOException {
    ByteArrayOutputStreamWithPos outputStream = new ByteArrayOutputStreamWithPos();
    DataOutputView outputView = new DataOutputViewStreamWrapper(outputStream);
    if (verificationMode == SnapshotVerificationMode.ITERATOR) {
        Iterator<StateEntry<K, N, S>> iterator = snapshot.getIterator(keySerializer, namespaceSerializer, stateSerializer, null);
        assertThat(() -> iterator, containsInAnyOrder(toMatchers(referenceStates)));
    } else {
        snapshot.writeState(keySerializer, namespaceSerializer, stateSerializer, outputView, null);
        Map<N, Map<K, S>> actualStates = readStateFromSnapshot(outputStream.toByteArray(), keySerializer, namespaceSerializer, stateSerializer);
        assertEquals(referenceStates, actualStates);
    }
}

Example 48 with DataOutputViewStreamWrapper

use of org.apache.flink.core.memory.DataOutputViewStreamWrapper in project flink by apache.

the class CopyOnWriteSkipListStateMapTestUtils method verifySnapshotWithTransform.

static <K, N, S> void verifySnapshotWithTransform(@Nonnull Map<N, Map<K, S>> referenceStates, @Nonnull CopyOnWriteSkipListStateMapSnapshot<K, N, S> snapshot, StateSnapshotTransformer<S> transformer, TypeSerializer<K> keySerializer, TypeSerializer<N> namespaceSerializer, TypeSerializer<S> stateSerializer, SnapshotVerificationMode verificationMode) throws IOException {
    ByteArrayOutputStreamWithPos outputStream = new ByteArrayOutputStreamWithPos();
    DataOutputView outputView = new DataOutputViewStreamWrapper(outputStream);
    Map<N, Map<K, S>> transformedStates = new HashMap<>();
    for (Map.Entry<N, Map<K, S>> namespaceEntry : referenceStates.entrySet()) {
        for (Map.Entry<K, S> keyEntry : namespaceEntry.getValue().entrySet()) {
            S state = transformer.filterOrTransform(keyEntry.getValue());
            if (state != null) {
                transformedStates.computeIfAbsent(namespaceEntry.getKey(), (none) -> new HashMap<>()).put(keyEntry.getKey(), state);
            }
        }
    }
    if (verificationMode == SnapshotVerificationMode.SERIALIZED) {
        snapshot.writeState(keySerializer, namespaceSerializer, stateSerializer, outputView, transformer);
        Map<N, Map<K, S>> actualStates = readStateFromSnapshot(outputStream.toByteArray(), keySerializer, namespaceSerializer, stateSerializer);
        assertEquals(transformedStates, actualStates);
    } else {
        Iterator<StateEntry<K, N, S>> iterator = snapshot.getIterator(keySerializer, namespaceSerializer, stateSerializer, null);
        assertThat(() -> iterator, containsInAnyOrder(toMatchers(referenceStates)));
    }
}

Example 49 with DataOutputViewStreamWrapper

use of org.apache.flink.core.memory.DataOutputViewStreamWrapper in project flink by apache.

the class SocketStreamIteratorTest method testIterator.

@Test
public void testIterator() throws Exception {
    final AtomicReference<Throwable> error = new AtomicReference<>();
    final long seed = new Random().nextLong();
    final int numElements = 1000;
    final SocketStreamIterator<Long> iterator = new SocketStreamIterator<>(LongSerializer.INSTANCE);
    Thread writer = new Thread() {

        @Override
        public void run() {
            try {
                try (Socket sock = new Socket(iterator.getBindAddress(), iterator.getPort());
                    DataOutputViewStreamWrapper out = new DataOutputViewStreamWrapper(sock.getOutputStream())) {
                    final TypeSerializer<Long> serializer = LongSerializer.INSTANCE;
                    final Random rnd = new Random(seed);
                    for (int i = 0; i < numElements; i++) {
                        serializer.serialize(rnd.nextLong(), out);
                    }
                }
            } catch (Throwable t) {
                error.set(t);
            }
        }
    };
    writer.start();
    final Random validator = new Random(seed);
    for (int i = 0; i < numElements; i++) {
        assertTrue(iterator.hasNext());
        assertTrue(iterator.hasNext());
        assertEquals(validator.nextLong(), iterator.next().longValue());
    }
    assertFalse(iterator.hasNext());
    writer.join();
    assertFalse(iterator.hasNext());
}

Example 50 with DataOutputViewStreamWrapper

use of org.apache.flink.core.memory.DataOutputViewStreamWrapper in project flink by apache.

the class CollectResultBufferTest method createSerializedResults.

private List<byte[]> createSerializedResults(List<Integer> values) throws Exception {
    List<byte[]> serializedResults = new ArrayList<>();
    for (int value : values) {
        ByteArrayOutputStream baos = new ByteArrayOutputStream();
        DataOutputView wrapper = new DataOutputViewStreamWrapper(baos);
        serializer.serialize(value, wrapper);
        serializedResults.add(baos.toByteArray());
    }
    return serializedResults;
}