Examples with DataOutputViewStreamWrapper org.apache.flink.core.memory.DataOutputViewStreamWrapper used on opensource projects

Example 56 with DataOutputViewStreamWrapper

use of org.apache.flink.core.memory.DataOutputViewStreamWrapper in project flink by apache.

the class CopyOnWriteStateTableTest method testSerializerDuplicationInSnapshot.

/**
 * This tests that serializers used for snapshots are duplicates of the ones used in processing
 * to avoid race conditions in stateful serializers.
 */
@Test
public void testSerializerDuplicationInSnapshot() throws IOException {
    final TestDuplicateSerializer namespaceSerializer = new TestDuplicateSerializer();
    final TestDuplicateSerializer stateSerializer = new TestDuplicateSerializer();
    final TestDuplicateSerializer keySerializer = new TestDuplicateSerializer();
    RegisteredKeyValueStateBackendMetaInfo<Integer, Integer> metaInfo = new RegisteredKeyValueStateBackendMetaInfo<>(StateDescriptor.Type.VALUE, "test", namespaceSerializer, stateSerializer);
    InternalKeyContext<Integer> mockKeyContext = new MockInternalKeyContext<>();
    CopyOnWriteStateTable<Integer, Integer, Integer> table = new CopyOnWriteStateTable<>(mockKeyContext, metaInfo, keySerializer);
    table.put(0, 0, 0, 0);
    table.put(1, 0, 0, 1);
    table.put(2, 0, 1, 2);
    final CopyOnWriteStateTableSnapshot<Integer, Integer, Integer> snapshot = table.stateSnapshot();
    final StateSnapshot.StateKeyGroupWriter partitionedSnapshot = snapshot.getKeyGroupWriter();
    namespaceSerializer.disable();
    keySerializer.disable();
    stateSerializer.disable();
    partitionedSnapshot.writeStateInKeyGroup(new DataOutputViewStreamWrapper(new ByteArrayOutputStreamWithPos(1024)), 0);
}

Example 57 with DataOutputViewStreamWrapper

use of org.apache.flink.core.memory.DataOutputViewStreamWrapper in project flink by apache.

the class CopyOnWriteStateTableTest method testReleaseForSuccessfulSnapshot.

/**
 * This tests that resource can be released for a successful snapshot.
 */
@Test
public void testReleaseForSuccessfulSnapshot() throws IOException {
    int numberOfKeyGroups = 10;
    CopyOnWriteStateTable<Integer, Integer, Float> table = createStateTableForSnapshotRelease(numberOfKeyGroups);
    ByteArrayOutputStreamWithPos byteArrayOutputStreamWithPos = new ByteArrayOutputStreamWithPos();
    DataOutputView dataOutputView = new DataOutputViewStreamWrapper(byteArrayOutputStreamWithPos);
    CopyOnWriteStateTableSnapshot<Integer, Integer, Float> snapshot = table.stateSnapshot();
    for (int group = 0; group < numberOfKeyGroups; group++) {
        snapshot.writeStateInKeyGroup(dataOutputView, group);
        // resource used by one key group should be released after the snapshot is successful
        Assert.assertTrue(isResourceReleasedForKeyGroup(table, group));
    }
    snapshot.release();
    verifyResourceIsReleasedForAllKeyGroup(table, 1);
}

Example 58 with DataOutputViewStreamWrapper

use of org.apache.flink.core.memory.DataOutputViewStreamWrapper in project flink by apache.

the class CollectSinkFunction method serializeAccumulatorResult.

@VisibleForTesting
public static byte[] serializeAccumulatorResult(long offset, String version, long lastCheckpointedOffset, List<byte[]> buffer) throws IOException {
    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    DataOutputViewStreamWrapper wrapper = new DataOutputViewStreamWrapper(baos);
    wrapper.writeLong(offset);
    CollectCoordinationResponse finalResponse = new CollectCoordinationResponse(version, lastCheckpointedOffset, buffer);
    finalResponse.serialize(wrapper);
    return baos.toByteArray();
}

Example 59 with DataOutputViewStreamWrapper

use of org.apache.flink.core.memory.DataOutputViewStreamWrapper in project flink by apache.

the class SerializerTestUtil method snapshotAndReconfigure.

/**
 * Snapshot and restore the given serializer. Returns the restored serializer.
 */
public static <T> TypeSerializer<T> snapshotAndReconfigure(TypeSerializer<T> serializer, SerializerGetter<T> serializerGetter) throws IOException {
    TypeSerializerSnapshot<T> configSnapshot = serializer.snapshotConfiguration();
    byte[] serializedConfig;
    try (ByteArrayOutputStream out = new ByteArrayOutputStream()) {
        TypeSerializerSnapshotSerializationUtil.writeSerializerSnapshot(new DataOutputViewStreamWrapper(out), configSnapshot, serializer);
        serializedConfig = out.toByteArray();
    }
    TypeSerializerSnapshot<T> restoredConfig;
    try (ByteArrayInputStream in = new ByteArrayInputStream(serializedConfig)) {
        restoredConfig = TypeSerializerSnapshotSerializationUtil.readSerializerSnapshot(new DataInputViewStreamWrapper(in), Thread.currentThread().getContextClassLoader(), serializerGetter.getSerializer());
    }
    TypeSerializerSchemaCompatibility<T> strategy = restoredConfig.resolveSchemaCompatibility(serializerGetter.getSerializer());
    final TypeSerializer<T> restoredSerializer;
    if (strategy.isCompatibleAsIs()) {
        restoredSerializer = restoredConfig.restoreSerializer();
    } else if (strategy.isCompatibleWithReconfiguredSerializer()) {
        restoredSerializer = strategy.getReconfiguredSerializer();
    } else {
        throw new AssertionError("Unable to restore serializer with " + strategy);
    }
    assertEquals(serializer.getClass(), restoredSerializer.getClass());
    return restoredSerializer;
}

Example 60 with DataOutputViewStreamWrapper

use of org.apache.flink.core.memory.DataOutputViewStreamWrapper in project flink by apache.

the class KeyedStateCheckpointOutputStreamTest method writeAllTestKeyGroups.

private KeyGroupsStateHandle writeAllTestKeyGroups(KeyedStateCheckpointOutputStream stream, KeyGroupRange keyRange) throws Exception {
    DataOutputView dov = new DataOutputViewStreamWrapper(stream);
    for (int kg : keyRange) {
        stream.startNewKeyGroup(kg);
        dov.writeInt(kg);
    }
    return stream.closeAndGetHandle();
}