Examples with DataOutputViewStreamWrapper org.apache.flink.core.memory.DataOutputViewStreamWrapper used on opensource projects

Example 91 with DataOutputViewStreamWrapper

use of org.apache.flink.core.memory.DataOutputViewStreamWrapper in project flink by apache.

the class RecordTest method setUp.

@Before
public void setUp() throws Exception {
    PipedInputStream pipeIn = new PipedInputStream(1024 * 1024);
    PipedOutputStream pipeOut = new PipedOutputStream(pipeIn);
    this.in = new DataInputViewStreamWrapper(pipeIn);
    this.out = new DataOutputViewStreamWrapper(pipeOut);
}

Example 92 with DataOutputViewStreamWrapper

use of org.apache.flink.core.memory.DataOutputViewStreamWrapper in project flink by apache.

the class NFAStateSerializer method copy.

@Override
public NFAState copy(NFAState from) {
    try {
        ByteArrayOutputStream baos = new ByteArrayOutputStream();
        serialize(from, new DataOutputViewStreamWrapper(baos));
        baos.close();
        byte[] data = baos.toByteArray();
        ByteArrayInputStream bais = new ByteArrayInputStream(data);
        NFAState copy = deserialize(new DataInputViewStreamWrapper(bais));
        bais.close();
        return copy;
    } catch (IOException e) {
        throw new RuntimeException("Could not copy NFA.", e);
    }
}

Example 93 with DataOutputViewStreamWrapper

use of org.apache.flink.core.memory.DataOutputViewStreamWrapper in project flink by apache.

the class PojoSerializerTest method testReconfigureRepopulateNonregisteredSubclassSerializerCache.

/**
 * Tests that reconfiguration repopulates previously cached subclass serializers.
 */
@Test
public void testReconfigureRepopulateNonregisteredSubclassSerializerCache() throws Exception {
    // don't register any subclasses
    PojoSerializer<TestUserClass> pojoSerializer = (PojoSerializer<TestUserClass>) type.createSerializer(new ExecutionConfig());
    // create cached serializers for SubTestUserClassA and SubTestUserClassB
    pojoSerializer.getSubclassSerializer(SubTestUserClassA.class);
    pojoSerializer.getSubclassSerializer(SubTestUserClassB.class);
    assertEquals(2, pojoSerializer.getSubclassSerializerCache().size());
    assertTrue(pojoSerializer.getSubclassSerializerCache().containsKey(SubTestUserClassA.class));
    assertTrue(pojoSerializer.getSubclassSerializerCache().containsKey(SubTestUserClassB.class));
    // snapshot configuration and serialize to bytes
    TypeSerializerSnapshot pojoSerializerConfigSnapshot = pojoSerializer.snapshotConfiguration();
    byte[] serializedConfig;
    try (ByteArrayOutputStream out = new ByteArrayOutputStream()) {
        TypeSerializerSnapshotSerializationUtil.writeSerializerSnapshot(new DataOutputViewStreamWrapper(out), pojoSerializerConfigSnapshot, pojoSerializer);
        serializedConfig = out.toByteArray();
    }
    // instantiate new PojoSerializer
    pojoSerializer = (PojoSerializer<TestUserClass>) type.createSerializer(new ExecutionConfig());
    // read configuration from bytes
    try (ByteArrayInputStream in = new ByteArrayInputStream(serializedConfig)) {
        pojoSerializerConfigSnapshot = TypeSerializerSnapshotSerializationUtil.readSerializerSnapshot(new DataInputViewStreamWrapper(in), Thread.currentThread().getContextClassLoader(), pojoSerializer);
    }
    // reconfigure - check reconfiguration result and that subclass serializer cache is
    // repopulated
    @SuppressWarnings("unchecked") TypeSerializerSchemaCompatibility<TestUserClass> compatResult = pojoSerializerConfigSnapshot.resolveSchemaCompatibility(pojoSerializer);
    assertTrue(compatResult.isCompatibleWithReconfiguredSerializer());
    assertThat(compatResult.getReconfiguredSerializer(), instanceOf(PojoSerializer.class));
    PojoSerializer<TestUserClass> reconfiguredPojoSerializer = (PojoSerializer<TestUserClass>) compatResult.getReconfiguredSerializer();
    assertEquals(2, reconfiguredPojoSerializer.getSubclassSerializerCache().size());
    assertTrue(reconfiguredPojoSerializer.getSubclassSerializerCache().containsKey(SubTestUserClassA.class));
    assertTrue(reconfiguredPojoSerializer.getSubclassSerializerCache().containsKey(SubTestUserClassB.class));
}

Example 94 with DataOutputViewStreamWrapper

use of org.apache.flink.core.memory.DataOutputViewStreamWrapper in project flink by apache.

the class PojoSerializerTest method testReconfigureWithDifferentPojoType.

// --------------------------------------------------------------------------------------------
// Configuration snapshotting & reconfiguring tests
// --------------------------------------------------------------------------------------------
/**
 * Verifies that reconfiguring with a config snapshot of a preceding POJO serializer with
 * different POJO type will result in INCOMPATIBLE.
 */
@Test
public void testReconfigureWithDifferentPojoType() throws Exception {
    PojoSerializer<SubTestUserClassB> pojoSerializer1 = (PojoSerializer<SubTestUserClassB>) TypeExtractor.getForClass(SubTestUserClassB.class).createSerializer(new ExecutionConfig());
    // snapshot configuration and serialize to bytes
    TypeSerializerSnapshot pojoSerializerConfigSnapshot = pojoSerializer1.snapshotConfiguration();
    byte[] serializedConfig;
    try (ByteArrayOutputStream out = new ByteArrayOutputStream()) {
        TypeSerializerSnapshotSerializationUtil.writeSerializerSnapshot(new DataOutputViewStreamWrapper(out), pojoSerializerConfigSnapshot, pojoSerializer1);
        serializedConfig = out.toByteArray();
    }
    PojoSerializer<SubTestUserClassA> pojoSerializer2 = (PojoSerializer<SubTestUserClassA>) TypeExtractor.getForClass(SubTestUserClassA.class).createSerializer(new ExecutionConfig());
    // read configuration again from bytes
    try (ByteArrayInputStream in = new ByteArrayInputStream(serializedConfig)) {
        pojoSerializerConfigSnapshot = TypeSerializerSnapshotSerializationUtil.readSerializerSnapshot(new DataInputViewStreamWrapper(in), Thread.currentThread().getContextClassLoader(), pojoSerializer2);
    }
    @SuppressWarnings("unchecked") TypeSerializerSchemaCompatibility<SubTestUserClassA> compatResult = pojoSerializerConfigSnapshot.resolveSchemaCompatibility(pojoSerializer2);
    assertTrue(compatResult.isIncompatible());
}

Example 95 with DataOutputViewStreamWrapper

use of org.apache.flink.core.memory.DataOutputViewStreamWrapper in project flink by apache.

the class PojoSerializerTest method testReconfigureDifferentSubclassRegistrationOrder.

/**
 * Tests that reconfiguration correctly reorders subclass registrations to their previous order.
 */
@Test
public void testReconfigureDifferentSubclassRegistrationOrder() throws Exception {
    ExecutionConfig executionConfig = new ExecutionConfig();
    executionConfig.registerPojoType(SubTestUserClassA.class);
    executionConfig.registerPojoType(SubTestUserClassB.class);
    PojoSerializer<TestUserClass> pojoSerializer = (PojoSerializer<TestUserClass>) type.createSerializer(executionConfig);
    // get original registration ids
    int subClassATag = pojoSerializer.getRegisteredClasses().get(SubTestUserClassA.class);
    int subClassBTag = pojoSerializer.getRegisteredClasses().get(SubTestUserClassB.class);
    // snapshot configuration and serialize to bytes
    TypeSerializerSnapshot pojoSerializerConfigSnapshot = pojoSerializer.snapshotConfiguration();
    byte[] serializedConfig;
    try (ByteArrayOutputStream out = new ByteArrayOutputStream()) {
        TypeSerializerSnapshotSerializationUtil.writeSerializerSnapshot(new DataOutputViewStreamWrapper(out), pojoSerializerConfigSnapshot, pojoSerializer);
        serializedConfig = out.toByteArray();
    }
    // use new config and instantiate new PojoSerializer
    executionConfig = new ExecutionConfig();
    executionConfig.registerPojoType(// test with B registered before A
    SubTestUserClassB.class);
    executionConfig.registerPojoType(SubTestUserClassA.class);
    pojoSerializer = (PojoSerializer<TestUserClass>) type.createSerializer(executionConfig);
    // read configuration from bytes
    try (ByteArrayInputStream in = new ByteArrayInputStream(serializedConfig)) {
        pojoSerializerConfigSnapshot = TypeSerializerSnapshotSerializationUtil.readSerializerSnapshot(new DataInputViewStreamWrapper(in), Thread.currentThread().getContextClassLoader(), pojoSerializer);
    }
    @SuppressWarnings("unchecked") TypeSerializerSchemaCompatibility<TestUserClass> compatResult = pojoSerializerConfigSnapshot.resolveSchemaCompatibility(pojoSerializer);
    assertTrue(compatResult.isCompatibleWithReconfiguredSerializer());
    assertThat(compatResult.getReconfiguredSerializer(), instanceOf(PojoSerializer.class));
    // reconfigure - check reconfiguration result and that registration ids remains the same
    // assertEquals(ReconfigureResult.COMPATIBLE,
    // pojoSerializer.reconfigure(pojoSerializerConfigSnapshot));
    PojoSerializer<TestUserClass> reconfiguredPojoSerializer = (PojoSerializer<TestUserClass>) compatResult.getReconfiguredSerializer();
    assertEquals(subClassATag, reconfiguredPojoSerializer.getRegisteredClasses().get(SubTestUserClassA.class).intValue());
    assertEquals(subClassBTag, reconfiguredPojoSerializer.getRegisteredClasses().get(SubTestUserClassB.class).intValue());
}