Examples with DataOutputViewStreamWrapper org.apache.flink.core.memory.DataOutputViewStreamWrapper used on opensource projects

Example 41 with DataOutputViewStreamWrapper

use of org.apache.flink.core.memory.DataOutputViewStreamWrapper in project flink by apache.

the class StateChangeFormat method write.

Map<StateChangeSet, Long> write(OutputStreamWithPos os, Collection<StateChangeSet> changeSets) throws IOException {
    List<StateChangeSet> sorted = new ArrayList<>(changeSets);
    // using sorting instead of bucketing for simplicity
    sorted.sort(comparing(StateChangeSet::getLogId).thenComparing(StateChangeSet::getSequenceNumber));
    DataOutputViewStreamWrapper dataOutput = new DataOutputViewStreamWrapper(os);
    Map<StateChangeSet, Long> pendingResults = new HashMap<>();
    for (StateChangeSet changeSet : sorted) {
        pendingResults.put(changeSet, os.getPos());
        writeChangeSet(dataOutput, changeSet.getChanges());
    }
    return pendingResults;
}

Example 42 with DataOutputViewStreamWrapper

use of org.apache.flink.core.memory.DataOutputViewStreamWrapper in project flink by apache.

the class BinaryOutputFormat method open.

@Override
public void open(int taskNumber, int numTasks) throws IOException {
    super.open(taskNumber, numTasks);
    final long blockSize = this.blockSize == NATIVE_BLOCK_SIZE ? this.outputFilePath.getFileSystem().getDefaultBlockSize() : this.blockSize;
    this.blockBasedOutput = new BlockBasedOutput(this.stream, (int) blockSize);
    this.outView = new DataOutputViewStreamWrapper(this.blockBasedOutput);
}

Example 43 with DataOutputViewStreamWrapper

use of org.apache.flink.core.memory.DataOutputViewStreamWrapper in project flink by apache.

the class PojoSerializerTest method testReconfigureWithPreviouslyNonregisteredSubclasses.

/**
 * Tests that: - Previous Pojo serializer did not have registrations, and created cached
 * serializers for subclasses - On restore, it had those subclasses registered
 *
 * <p>In this case, after reconfiguration, the cache should be repopulated, and registrations
 * should also exist for the subclasses.
 *
 * <p>Note: the cache still needs to be repopulated because previous data of those subclasses
 * were written with the cached serializers. In this case, the repopulated cache has
 * reconfigured serializers for the subclasses so that previous written data can be read, but
 * the registered serializers for the subclasses do not necessarily need to be reconfigured
 * since they will only be used to write new data.
 */
@Test
public void testReconfigureWithPreviouslyNonregisteredSubclasses() throws Exception {
    // don't register any subclasses at first
    PojoSerializer<TestUserClass> pojoSerializer = (PojoSerializer<TestUserClass>) type.createSerializer(new ExecutionConfig());
    // create cached serializers for SubTestUserClassA and SubTestUserClassB
    pojoSerializer.getSubclassSerializer(SubTestUserClassA.class);
    pojoSerializer.getSubclassSerializer(SubTestUserClassB.class);
    // make sure serializers are in cache
    assertEquals(2, pojoSerializer.getSubclassSerializerCache().size());
    assertTrue(pojoSerializer.getSubclassSerializerCache().containsKey(SubTestUserClassA.class));
    assertTrue(pojoSerializer.getSubclassSerializerCache().containsKey(SubTestUserClassB.class));
    // make sure that registrations are empty
    assertTrue(pojoSerializer.getRegisteredClasses().isEmpty());
    assertEquals(0, pojoSerializer.getRegisteredSerializers().length);
    // snapshot configuration and serialize to bytes
    TypeSerializerSnapshot pojoSerializerConfigSnapshot = pojoSerializer.snapshotConfiguration();
    byte[] serializedConfig;
    try (ByteArrayOutputStream out = new ByteArrayOutputStream()) {
        TypeSerializerSnapshotSerializationUtil.writeSerializerSnapshot(new DataOutputViewStreamWrapper(out), pojoSerializerConfigSnapshot, pojoSerializer);
        serializedConfig = out.toByteArray();
    }
    // instantiate new PojoSerializer, with new execution config that has the subclass
    // registrations
    ExecutionConfig newExecutionConfig = new ExecutionConfig();
    newExecutionConfig.registerPojoType(SubTestUserClassA.class);
    newExecutionConfig.registerPojoType(SubTestUserClassB.class);
    pojoSerializer = (PojoSerializer<TestUserClass>) type.createSerializer(newExecutionConfig);
    // read configuration from bytes
    try (ByteArrayInputStream in = new ByteArrayInputStream(serializedConfig)) {
        pojoSerializerConfigSnapshot = TypeSerializerSnapshotSerializationUtil.readSerializerSnapshot(new DataInputViewStreamWrapper(in), Thread.currentThread().getContextClassLoader(), pojoSerializer);
    }
    // reconfigure - check reconfiguration result and that
    // 1) subclass serializer cache is repopulated
    // 2) registrations also contain the now registered subclasses
    @SuppressWarnings("unchecked") TypeSerializerSchemaCompatibility<TestUserClass> compatResult = pojoSerializerConfigSnapshot.resolveSchemaCompatibility(pojoSerializer);
    assertTrue(compatResult.isCompatibleWithReconfiguredSerializer());
    assertThat(compatResult.getReconfiguredSerializer(), instanceOf(PojoSerializer.class));
    PojoSerializer<TestUserClass> reconfiguredPojoSerializer = (PojoSerializer<TestUserClass>) compatResult.getReconfiguredSerializer();
    assertEquals(2, reconfiguredPojoSerializer.getSubclassSerializerCache().size());
    assertTrue(reconfiguredPojoSerializer.getSubclassSerializerCache().containsKey(SubTestUserClassA.class));
    assertTrue(reconfiguredPojoSerializer.getSubclassSerializerCache().containsKey(SubTestUserClassB.class));
    assertEquals(2, reconfiguredPojoSerializer.getRegisteredClasses().size());
    assertTrue(reconfiguredPojoSerializer.getRegisteredClasses().containsKey(SubTestUserClassA.class));
    assertTrue(reconfiguredPojoSerializer.getRegisteredClasses().containsKey(SubTestUserClassB.class));
}

Example 44 with DataOutputViewStreamWrapper

use of org.apache.flink.core.memory.DataOutputViewStreamWrapper in project flink by apache.

the class DefaultOperatorStateBackendSnapshotStrategy method asyncSnapshot.

@Override
public SnapshotResultSupplier<OperatorStateHandle> asyncSnapshot(DefaultOperatorStateBackendSnapshotResources syncPartResource, long checkpointId, long timestamp, @Nonnull CheckpointStreamFactory streamFactory, @Nonnull CheckpointOptions checkpointOptions) {
    Map<String, PartitionableListState<?>> registeredOperatorStatesDeepCopies = syncPartResource.getRegisteredOperatorStatesDeepCopies();
    Map<String, BackendWritableBroadcastState<?, ?>> registeredBroadcastStatesDeepCopies = syncPartResource.getRegisteredBroadcastStatesDeepCopies();
    if (registeredBroadcastStatesDeepCopies.isEmpty() && registeredOperatorStatesDeepCopies.isEmpty()) {
        return snapshotCloseableRegistry -> SnapshotResult.empty();
    }
    return (snapshotCloseableRegistry) -> {
        CheckpointStateOutputStream localOut = streamFactory.createCheckpointStateOutputStream(CheckpointedStateScope.EXCLUSIVE);
        snapshotCloseableRegistry.registerCloseable(localOut);
        // get the registered operator state infos ...
        List<StateMetaInfoSnapshot> operatorMetaInfoSnapshots = new ArrayList<>(registeredOperatorStatesDeepCopies.size());
        for (Map.Entry<String, PartitionableListState<?>> entry : registeredOperatorStatesDeepCopies.entrySet()) {
            operatorMetaInfoSnapshots.add(entry.getValue().getStateMetaInfo().snapshot());
        }
        // ... get the registered broadcast operator state infos ...
        List<StateMetaInfoSnapshot> broadcastMetaInfoSnapshots = new ArrayList<>(registeredBroadcastStatesDeepCopies.size());
        for (Map.Entry<String, BackendWritableBroadcastState<?, ?>> entry : registeredBroadcastStatesDeepCopies.entrySet()) {
            broadcastMetaInfoSnapshots.add(entry.getValue().getStateMetaInfo().snapshot());
        }
        // ... write them all in the checkpoint stream ...
        DataOutputView dov = new DataOutputViewStreamWrapper(localOut);
        OperatorBackendSerializationProxy backendSerializationProxy = new OperatorBackendSerializationProxy(operatorMetaInfoSnapshots, broadcastMetaInfoSnapshots);
        backendSerializationProxy.write(dov);
        // ... and then go for the states ...
        // we put BOTH normal and broadcast state metadata here
        int initialMapCapacity = registeredOperatorStatesDeepCopies.size() + registeredBroadcastStatesDeepCopies.size();
        final Map<String, OperatorStateHandle.StateMetaInfo> writtenStatesMetaData = new HashMap<>(initialMapCapacity);
        for (Map.Entry<String, PartitionableListState<?>> entry : registeredOperatorStatesDeepCopies.entrySet()) {
            PartitionableListState<?> value = entry.getValue();
            long[] partitionOffsets = value.write(localOut);
            OperatorStateHandle.Mode mode = value.getStateMetaInfo().getAssignmentMode();
            writtenStatesMetaData.put(entry.getKey(), new OperatorStateHandle.StateMetaInfo(partitionOffsets, mode));
        }
        // ... and the broadcast states themselves ...
        for (Map.Entry<String, BackendWritableBroadcastState<?, ?>> entry : registeredBroadcastStatesDeepCopies.entrySet()) {
            BackendWritableBroadcastState<?, ?> value = entry.getValue();
            long[] partitionOffsets = { value.write(localOut) };
            OperatorStateHandle.Mode mode = value.getStateMetaInfo().getAssignmentMode();
            writtenStatesMetaData.put(entry.getKey(), new OperatorStateHandle.StateMetaInfo(partitionOffsets, mode));
        }
        // ... and, finally, create the state handle.
        OperatorStateHandle retValue = null;
        if (snapshotCloseableRegistry.unregisterCloseable(localOut)) {
            StreamStateHandle stateHandle = localOut.closeAndGetHandle();
            if (stateHandle != null) {
                retValue = new OperatorStreamStateHandle(writtenStatesMetaData, stateHandle);
            }
            return SnapshotResult.of(retValue);
        } else {
            throw new IOException("Stream was already unregistered.");
        }
    };
}

Example 45 with DataOutputViewStreamWrapper

use of org.apache.flink.core.memory.DataOutputViewStreamWrapper in project flink by apache.

the class HeapBroadcastState method write.

@Override
public long write(FSDataOutputStream out) throws IOException {
    long partitionOffset = out.getPos();
    DataOutputView dov = new DataOutputViewStreamWrapper(out);
    dov.writeInt(backingMap.size());
    for (Map.Entry<K, V> entry : backingMap.entrySet()) {
        getStateMetaInfo().getKeySerializer().serialize(entry.getKey(), dov);
        getStateMetaInfo().getValueSerializer().serialize(entry.getValue(), dov);
    }
    return partitionOffset;
}