Examples with DataOutputViewStreamWrapper org.apache.flink.core.memory.DataOutputViewStreamWrapper used on opensource projects

Example 16 with DataOutputViewStreamWrapper

use of org.apache.flink.core.memory.DataOutputViewStreamWrapper in project flink by apache.

the class ContinuousHivePendingSplitsCheckpointSerializer method serialize.

@Override
public byte[] serialize(PendingSplitsCheckpoint<HiveSourceSplit> checkpoint) throws IOException {
    Preconditions.checkArgument(checkpoint.getClass() == ContinuousHivePendingSplitsCheckpoint.class, "Only supports %s", ContinuousHivePendingSplitsCheckpoint.class.getName());
    ContinuousHivePendingSplitsCheckpoint hiveCheckpoint = (ContinuousHivePendingSplitsCheckpoint) checkpoint;
    PendingSplitsCheckpoint<HiveSourceSplit> superCP = PendingSplitsCheckpoint.fromCollectionSnapshot(hiveCheckpoint.getSplits(), hiveCheckpoint.getAlreadyProcessedPaths());
    byte[] superBytes = superSerDe.serialize(superCP);
    ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
    try (DataOutputViewStreamWrapper outputWrapper = new DataOutputViewStreamWrapper(byteArrayOutputStream)) {
        outputWrapper.writeInt(superBytes.length);
        outputWrapper.write(superBytes);
        readOffsetSerDe.serialize(hiveCheckpoint.getCurrentReadOffset(), outputWrapper);
        outputWrapper.writeInt(hiveCheckpoint.getSeenPartitionsSinceOffset().size());
        for (List<String> partition : hiveCheckpoint.getSeenPartitionsSinceOffset()) {
            partitionSerDe.serialize(partition, outputWrapper);
        }
    }
    return byteArrayOutputStream.toByteArray();
}

Example 17 with DataOutputViewStreamWrapper

use of org.apache.flink.core.memory.DataOutputViewStreamWrapper in project flink by apache.

the class InstantiationUtil method serializeToByteArray.

public static <T> byte[] serializeToByteArray(TypeSerializer<T> serializer, T record) throws IOException {
    if (record == null) {
        throw new NullPointerException("Record to serialize to byte array must not be null.");
    }
    ByteArrayOutputStream bos = new ByteArrayOutputStream(64);
    DataOutputViewStreamWrapper outputViewWrapper = new DataOutputViewStreamWrapper(bos);
    serializer.serialize(record, outputViewWrapper);
    return bos.toByteArray();
}

Example 18 with DataOutputViewStreamWrapper

use of org.apache.flink.core.memory.DataOutputViewStreamWrapper in project flink by apache.

the class VersionedIOWriteableTest method testReadMismatchVersion.

@Test
public void testReadMismatchVersion() throws Exception {
    String payload = "test";
    TestWriteable testWriteable = new TestWriteable(1, payload);
    byte[] serialized;
    try (ByteArrayOutputStreamWithPos out = new ByteArrayOutputStreamWithPos()) {
        testWriteable.write(new DataOutputViewStreamWrapper(out));
        serialized = out.toByteArray();
    }
    testWriteable = new TestWriteable(2);
    try (ByteArrayInputStreamWithPos in = new ByteArrayInputStreamWithPos(serialized)) {
        testWriteable.read(new DataInputViewStreamWrapper(in));
        Assert.fail("Version mismatch expected.");
    } catch (VersionMismatchException ignored) {
    }
    Assert.assertEquals(null, testWriteable.getData());
}

Example 19 with DataOutputViewStreamWrapper

use of org.apache.flink.core.memory.DataOutputViewStreamWrapper in project flink by apache.

the class VersionedIOWriteableTest method testReadSameVersion.

@Test
public void testReadSameVersion() throws Exception {
    String payload = "test";
    TestWriteable testWriteable = new TestWriteable(1, payload);
    byte[] serialized;
    try (ByteArrayOutputStreamWithPos out = new ByteArrayOutputStreamWithPos()) {
        testWriteable.write(new DataOutputViewStreamWrapper(out));
        serialized = out.toByteArray();
    }
    testWriteable = new TestWriteable(1);
    try (ByteArrayInputStreamWithPos in = new ByteArrayInputStreamWithPos(serialized)) {
        testWriteable.read(new DataInputViewStreamWrapper(in));
    }
    Assert.assertEquals(payload, testWriteable.getData());
}

Example 20 with DataOutputViewStreamWrapper

use of org.apache.flink.core.memory.DataOutputViewStreamWrapper in project flink by apache.

the class KryoSerializerCompatibilityTest method testMigrationStrategyForDifferentRegistrationOrder.

/**
 * Tests that after reconfiguration, registration ids are reconfigured to remain the same as the
 * preceding KryoSerializer.
 */
@Test
public void testMigrationStrategyForDifferentRegistrationOrder() throws Exception {
    ExecutionConfig executionConfig = new ExecutionConfig();
    executionConfig.registerKryoType(TestClassA.class);
    executionConfig.registerKryoType(TestClassB.class);
    KryoSerializer<TestClass> kryoSerializer = new KryoSerializer<>(TestClass.class, executionConfig);
    // get original registration ids
    int testClassId = kryoSerializer.getKryo().getRegistration(TestClass.class).getId();
    int testClassAId = kryoSerializer.getKryo().getRegistration(TestClassA.class).getId();
    int testClassBId = kryoSerializer.getKryo().getRegistration(TestClassB.class).getId();
    // snapshot configuration and serialize to bytes
    TypeSerializerSnapshot kryoSerializerConfigSnapshot = kryoSerializer.snapshotConfiguration();
    byte[] serializedConfig;
    try (ByteArrayOutputStream out = new ByteArrayOutputStream()) {
        TypeSerializerSnapshotSerializationUtil.writeSerializerSnapshot(new DataOutputViewStreamWrapper(out), kryoSerializerConfigSnapshot, kryoSerializer);
        serializedConfig = out.toByteArray();
    }
    // use new config and instantiate new KryoSerializer
    executionConfig = new ExecutionConfig();
    // test with B registered before A
    executionConfig.registerKryoType(TestClassB.class);
    executionConfig.registerKryoType(TestClassA.class);
    kryoSerializer = new KryoSerializer<>(TestClass.class, executionConfig);
    // read configuration from bytes
    try (ByteArrayInputStream in = new ByteArrayInputStream(serializedConfig)) {
        kryoSerializerConfigSnapshot = TypeSerializerSnapshotSerializationUtil.readSerializerSnapshot(new DataInputViewStreamWrapper(in), Thread.currentThread().getContextClassLoader(), kryoSerializer);
    }
    // reconfigure - check reconfiguration result and that registration id remains the same
    @SuppressWarnings("unchecked") TypeSerializerSchemaCompatibility<TestClass> compatResult = kryoSerializerConfigSnapshot.resolveSchemaCompatibility(kryoSerializer);
    assertTrue(compatResult.isCompatibleWithReconfiguredSerializer());
    kryoSerializer = (KryoSerializer<TestClass>) compatResult.getReconfiguredSerializer();
    assertEquals(testClassId, kryoSerializer.getKryo().getRegistration(TestClass.class).getId());
    assertEquals(testClassAId, kryoSerializer.getKryo().getRegistration(TestClassA.class).getId());
    assertEquals(testClassBId, kryoSerializer.getKryo().getRegistration(TestClassB.class).getId());
}