Examples with DataOutputViewStreamWrapper org.apache.flink.core.memory.DataOutputViewStreamWrapper used on opensource projects

Example 21 with DataOutputViewStreamWrapper

use of org.apache.flink.core.memory.DataOutputViewStreamWrapper in project flink by apache.

the class HeapSnapshotStrategy method asyncSnapshot.

@Override
public SnapshotResultSupplier<KeyedStateHandle> asyncSnapshot(HeapSnapshotResources<K> syncPartResource, long checkpointId, long timestamp, @Nonnull CheckpointStreamFactory streamFactory, @Nonnull CheckpointOptions checkpointOptions) {
    List<StateMetaInfoSnapshot> metaInfoSnapshots = syncPartResource.getMetaInfoSnapshots();
    if (metaInfoSnapshots.isEmpty()) {
        return snapshotCloseableRegistry -> SnapshotResult.empty();
    }
    final KeyedBackendSerializationProxy<K> serializationProxy = new KeyedBackendSerializationProxy<>(// get a serialized form already at state registration time in the future
    syncPartResource.getKeySerializer(), metaInfoSnapshots, !Objects.equals(UncompressedStreamCompressionDecorator.INSTANCE, keyGroupCompressionDecorator));
    final SupplierWithException<CheckpointStreamWithResultProvider, Exception> checkpointStreamSupplier = localRecoveryConfig.isLocalRecoveryEnabled() && !checkpointOptions.getCheckpointType().isSavepoint() ? () -> createDuplicatingStream(checkpointId, CheckpointedStateScope.EXCLUSIVE, streamFactory, localRecoveryConfig.getLocalStateDirectoryProvider().orElseThrow(LocalRecoveryConfig.localRecoveryNotEnabled())) : () -> createSimpleStream(CheckpointedStateScope.EXCLUSIVE, streamFactory);
    return (snapshotCloseableRegistry) -> {
        final Map<StateUID, Integer> stateNamesToId = syncPartResource.getStateNamesToId();
        final Map<StateUID, StateSnapshot> cowStateStableSnapshots = syncPartResource.getCowStateStableSnapshots();
        final CheckpointStreamWithResultProvider streamWithResultProvider = checkpointStreamSupplier.get();
        snapshotCloseableRegistry.registerCloseable(streamWithResultProvider);
        final CheckpointStateOutputStream localStream = streamWithResultProvider.getCheckpointOutputStream();
        final DataOutputViewStreamWrapper outView = new DataOutputViewStreamWrapper(localStream);
        serializationProxy.write(outView);
        final long[] keyGroupRangeOffsets = new long[keyGroupRange.getNumberOfKeyGroups()];
        for (int keyGroupPos = 0; keyGroupPos < keyGroupRange.getNumberOfKeyGroups(); ++keyGroupPos) {
            int keyGroupId = keyGroupRange.getKeyGroupId(keyGroupPos);
            keyGroupRangeOffsets[keyGroupPos] = localStream.getPos();
            outView.writeInt(keyGroupId);
            for (Map.Entry<StateUID, StateSnapshot> stateSnapshot : cowStateStableSnapshots.entrySet()) {
                StateSnapshot.StateKeyGroupWriter partitionedSnapshot = stateSnapshot.getValue().getKeyGroupWriter();
                try (OutputStream kgCompressionOut = keyGroupCompressionDecorator.decorateWithCompression(localStream)) {
                    DataOutputViewStreamWrapper kgCompressionView = new DataOutputViewStreamWrapper(kgCompressionOut);
                    kgCompressionView.writeShort(stateNamesToId.get(stateSnapshot.getKey()));
                    partitionedSnapshot.writeStateInKeyGroup(kgCompressionView, keyGroupId);
                }
            // this will just close the outer compression stream
            }
        }
        if (snapshotCloseableRegistry.unregisterCloseable(streamWithResultProvider)) {
            KeyGroupRangeOffsets kgOffs = new KeyGroupRangeOffsets(keyGroupRange, keyGroupRangeOffsets);
            SnapshotResult<StreamStateHandle> result = streamWithResultProvider.closeAndFinalizeCheckpointStreamResult();
            return toKeyedStateHandleSnapshotResult(result, kgOffs, KeyGroupsStateHandle::new);
        } else {
            throw new IOException("Stream already unregistered.");
        }
    };
}

Example 22 with DataOutputViewStreamWrapper

use of org.apache.flink.core.memory.DataOutputViewStreamWrapper in project flink by apache.

the class HeapListState method getSerializedValue.

@Override
public byte[] getSerializedValue(final byte[] serializedKeyAndNamespace, final TypeSerializer<K> safeKeySerializer, final TypeSerializer<N> safeNamespaceSerializer, final TypeSerializer<List<V>> safeValueSerializer) throws Exception {
    Preconditions.checkNotNull(serializedKeyAndNamespace);
    Preconditions.checkNotNull(safeKeySerializer);
    Preconditions.checkNotNull(safeNamespaceSerializer);
    Preconditions.checkNotNull(safeValueSerializer);
    Tuple2<K, N> keyAndNamespace = KvStateSerializer.deserializeKeyAndNamespace(serializedKeyAndNamespace, safeKeySerializer, safeNamespaceSerializer);
    List<V> result = stateTable.get(keyAndNamespace.f0, keyAndNamespace.f1);
    if (result == null) {
        return null;
    }
    final TypeSerializer<V> dupSerializer = ((ListSerializer<V>) safeValueSerializer).getElementSerializer();
    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    DataOutputViewStreamWrapper view = new DataOutputViewStreamWrapper(baos);
    // write the same as RocksDB writes lists, with one ',' separator
    for (int i = 0; i < result.size(); i++) {
        dupSerializer.serialize(result.get(i), view);
        if (i < result.size() - 1) {
            view.writeByte(',');
        }
    }
    view.flush();
    return baos.toByteArray();
}

Example 23 with DataOutputViewStreamWrapper

use of org.apache.flink.core.memory.DataOutputViewStreamWrapper in project flink by apache.

the class MetadataV3SerializerTest method testCheckpointSerialization.

/**
 * Test checkpoint metadata (de)serialization.
 *
 * @param checkpointId The given checkpointId will write into the metadata.
 * @param operatorStates the given states for all the operators.
 * @param masterStates the masterStates of the given checkpoint/savepoint.
 */
private void testCheckpointSerialization(long checkpointId, Collection<OperatorState> operatorStates, Collection<MasterState> masterStates, @Nullable String basePath) throws IOException {
    MetadataV3Serializer serializer = MetadataV3Serializer.INSTANCE;
    ByteArrayOutputStreamWithPos baos = new ByteArrayOutputStreamWithPos();
    DataOutputStream out = new DataOutputViewStreamWrapper(baos);
    CheckpointMetadata metadata = new CheckpointMetadata(checkpointId, operatorStates, masterStates);
    MetadataV3Serializer.serialize(metadata, out);
    out.close();
    // even if it makes the tests a a tad bit more clumsy
    if (basePath != null) {
        final Path metaPath = new Path(basePath, "_metadata");
        // this is in the temp folder, so it will get automatically deleted
        FileSystem.getLocalFileSystem().create(metaPath, FileSystem.WriteMode.OVERWRITE).close();
    }
    byte[] bytes = baos.toByteArray();
    DataInputStream in = new DataInputViewStreamWrapper(new ByteArrayInputStreamWithPos(bytes));
    CheckpointMetadata deserialized = serializer.deserialize(in, getClass().getClassLoader(), basePath);
    assertEquals(checkpointId, deserialized.getCheckpointId());
    assertEquals(operatorStates, deserialized.getOperatorStates());
    assertEquals(operatorStates.stream().map(OperatorState::isFullyFinished).collect(Collectors.toList()), deserialized.getOperatorStates().stream().map(OperatorState::isFullyFinished).collect(Collectors.toList()));
    assertEquals(masterStates.size(), deserialized.getMasterStates().size());
    for (Iterator<MasterState> a = masterStates.iterator(), b = deserialized.getMasterStates().iterator(); a.hasNext(); ) {
        CheckpointTestUtils.assertMasterStateEquality(a.next(), b.next());
    }
}

Example 24 with DataOutputViewStreamWrapper

use of org.apache.flink.core.memory.DataOutputViewStreamWrapper in project flink by apache.

the class AvroRecordInputFormatTest method testDeserializeToSpecificType.

/**
 * This test validates proper serialization with specific (generated POJO) types.
 */
@Test
public void testDeserializeToSpecificType() throws IOException {
    DatumReader<User> datumReader = new SpecificDatumReader<>(userSchema);
    try (FileReader<User> dataFileReader = DataFileReader.openReader(testFile, datumReader)) {
        User rec = dataFileReader.next();
        // check if record has been read correctly
        assertNotNull(rec);
        assertEquals("name not equal", TEST_NAME, rec.get("name").toString());
        assertEquals("enum not equal", TEST_ENUM_COLOR.toString(), rec.get("type_enum").toString());
        // now serialize it with our framework:
        ExecutionConfig ec = new ExecutionConfig();
        TypeInformation<User> te = TypeExtractor.createTypeInfo(User.class);
        assertEquals(AvroTypeInfo.class, te.getClass());
        TypeSerializer<User> tser = te.createSerializer(ec);
        ByteArrayOutputStream out = new ByteArrayOutputStream();
        try (DataOutputViewStreamWrapper outView = new DataOutputViewStreamWrapper(out)) {
            tser.serialize(rec, outView);
        }
        User newRec;
        try (DataInputViewStreamWrapper inView = new DataInputViewStreamWrapper(new ByteArrayInputStream(out.toByteArray()))) {
            newRec = tser.deserialize(inView);
        }
        // check if it is still the same
        assertNotNull(newRec);
        assertEquals("name not equal", TEST_NAME, newRec.getName().toString());
        assertEquals("enum not equal", TEST_ENUM_COLOR.toString(), newRec.getTypeEnum().toString());
    }
}

Example 25 with DataOutputViewStreamWrapper

use of org.apache.flink.core.memory.DataOutputViewStreamWrapper in project flink by apache.

the class TypeSerializerSerializationUtilTest method testAnonymousSerializerClassWithChangedSerialVersionUID.

/**
 * Verifies that serializers of anonymous classes can be deserialized, even if serialVersionUID
 * changes.
 */
@Test
public void testAnonymousSerializerClassWithChangedSerialVersionUID() throws Exception {
    TypeSerializer anonymousClassSerializer = new AbstractIntSerializer() {

        @Override
        public TypeSerializerSnapshot<Integer> snapshotConfiguration() {
            return null;
        }
    };
    // assert that our assumption holds
    Assert.assertTrue(anonymousClassSerializer.getClass().isAnonymousClass());
    byte[] anonymousSerializerBytes;
    try (ByteArrayOutputStream out = new ByteArrayOutputStream()) {
        TypeSerializerSerializationUtil.writeSerializer(new DataOutputViewStreamWrapper(out), anonymousClassSerializer);
        anonymousSerializerBytes = out.toByteArray();
    }
    long newSerialVersionUID = 1234567L;
    // assert that we're actually modifying to a different serialVersionUID
    Assert.assertNotEquals(ObjectStreamClass.lookup(anonymousClassSerializer.getClass()).getSerialVersionUID(), newSerialVersionUID);
    modifySerialVersionUID(anonymousSerializerBytes, anonymousClassSerializer.getClass().getName(), newSerialVersionUID);
    try (ByteArrayInputStream in = new ByteArrayInputStream(anonymousSerializerBytes)) {
        anonymousClassSerializer = TypeSerializerSerializationUtil.tryReadSerializer(new DataInputViewStreamWrapper(in), Thread.currentThread().getContextClassLoader());
    }
    // serializer should have been deserialized despite serialVersionUID mismatch
    Assert.assertNotNull(anonymousClassSerializer);
    Assert.assertTrue(anonymousClassSerializer.getClass().isAnonymousClass());
}