Examples with FSDataInputStream org.apache.flink.core.fs.FSDataInputStream used on opensource projects

Example 1 with FSDataInputStream

use of org.apache.flink.core.fs.FSDataInputStream in project flink by apache.

the class DefaultOperatorStateBackend method restore.

@Override
public void restore(Collection<OperatorStateHandle> restoreSnapshots) throws Exception {
    if (null == restoreSnapshots) {
        return;
    }
    for (OperatorStateHandle stateHandle : restoreSnapshots) {
        if (stateHandle == null) {
            continue;
        }
        FSDataInputStream in = stateHandle.openInputStream();
        closeStreamOnCancelRegistry.registerClosable(in);
        ClassLoader restoreClassLoader = Thread.currentThread().getContextClassLoader();
        try {
            Thread.currentThread().setContextClassLoader(userClassloader);
            OperatorBackendSerializationProxy backendSerializationProxy = new OperatorBackendSerializationProxy(userClassloader);
            backendSerializationProxy.read(new DataInputViewStreamWrapper(in));
            List<OperatorBackendSerializationProxy.StateMetaInfo<?>> metaInfoList = backendSerializationProxy.getNamedStateSerializationProxies();
            // Recreate all PartitionableListStates from the meta info
            for (OperatorBackendSerializationProxy.StateMetaInfo<?> stateMetaInfo : metaInfoList) {
                PartitionableListState<?> listState = registeredStates.get(stateMetaInfo.getName());
                if (null == listState) {
                    listState = new PartitionableListState<>(stateMetaInfo.getName(), stateMetaInfo.getStateSerializer(), stateMetaInfo.getMode());
                    registeredStates.put(listState.getName(), listState);
                } else {
                    Preconditions.checkState(listState.getPartitionStateSerializer().isCompatibleWith(stateMetaInfo.getStateSerializer()), "Incompatible state serializers found: " + listState.getPartitionStateSerializer() + " is not compatible with " + stateMetaInfo.getStateSerializer());
                }
            }
            // Restore all the state in PartitionableListStates
            for (Map.Entry<String, OperatorStateHandle.StateMetaInfo> nameToOffsets : stateHandle.getStateNameToPartitionOffsets().entrySet()) {
                PartitionableListState<?> stateListForName = registeredStates.get(nameToOffsets.getKey());
                Preconditions.checkState(null != stateListForName, "Found state without " + "corresponding meta info: " + nameToOffsets.getKey());
                deserializeStateValues(stateListForName, in, nameToOffsets.getValue());
            }
        } finally {
            Thread.currentThread().setContextClassLoader(restoreClassLoader);
            closeStreamOnCancelRegistry.unregisterClosable(in);
            IOUtils.closeQuietly(in);
        }
    }
}

Example 2 with FSDataInputStream

use of org.apache.flink.core.fs.FSDataInputStream in project flink by apache.

the class HeapKeyedStateBackend method restoreOldSavepointKeyedState.

@SuppressWarnings({ "unchecked", "rawtypes", "DeprecatedIsStillUsed" })
@Deprecated
private void restoreOldSavepointKeyedState(Collection<KeyGroupsStateHandle> stateHandles) throws IOException, ClassNotFoundException {
    if (stateHandles.isEmpty()) {
        return;
    }
    Preconditions.checkState(1 == stateHandles.size(), "Only one element expected here.");
    HashMap<String, KvStateSnapshot<K, ?, ?, ?>> namedStates;
    try (FSDataInputStream inputStream = stateHandles.iterator().next().openInputStream()) {
        namedStates = InstantiationUtil.deserializeObject(inputStream, userCodeClassLoader);
    }
    for (Map.Entry<String, KvStateSnapshot<K, ?, ?, ?>> nameToState : namedStates.entrySet()) {
        final String stateName = nameToState.getKey();
        final KvStateSnapshot<K, ?, ?, ?> genericSnapshot = nameToState.getValue();
        if (genericSnapshot instanceof MigrationRestoreSnapshot) {
            MigrationRestoreSnapshot<K, ?, ?> stateSnapshot = (MigrationRestoreSnapshot<K, ?, ?>) genericSnapshot;
            final StateTable rawResultMap = stateSnapshot.deserialize(stateName, this);
            // add named state to the backend
            stateTables.put(stateName, rawResultMap);
        } else {
            throw new IllegalStateException("Unknown state: " + genericSnapshot);
        }
    }
}

Example 3 with FSDataInputStream

use of org.apache.flink.core.fs.FSDataInputStream in project flink by apache.

the class RocksDBKeyedStateBackend method restoreOldSavepointKeyedState.

/**
	 * For backwards compatibility, remove again later!
	 */
@Deprecated
private void restoreOldSavepointKeyedState(Collection<KeyGroupsStateHandle> restoreState) throws Exception {
    if (restoreState.isEmpty()) {
        return;
    }
    Preconditions.checkState(1 == restoreState.size(), "Only one element expected here.");
    HashMap<String, RocksDBStateBackend.FinalFullyAsyncSnapshot> namedStates;
    try (FSDataInputStream inputStream = restoreState.iterator().next().openInputStream()) {
        namedStates = InstantiationUtil.deserializeObject(inputStream, userCodeClassLoader);
    }
    Preconditions.checkState(1 == namedStates.size(), "Only one element expected here.");
    DataInputView inputView = namedStates.values().iterator().next().stateHandle.getState(userCodeClassLoader);
    // clear k/v state information before filling it
    kvStateInformation.clear();
    // first get the column family mapping
    int numColumns = inputView.readInt();
    Map<Byte, StateDescriptor<?, ?>> columnFamilyMapping = new HashMap<>(numColumns);
    for (int i = 0; i < numColumns; i++) {
        byte mappingByte = inputView.readByte();
        ObjectInputStream ooIn = new InstantiationUtil.ClassLoaderObjectInputStream(new DataInputViewStream(inputView), userCodeClassLoader);
        StateDescriptor stateDescriptor = (StateDescriptor) ooIn.readObject();
        columnFamilyMapping.put(mappingByte, stateDescriptor);
        // this will fill in the k/v state information
        getColumnFamily(stateDescriptor, MigrationNamespaceSerializerProxy.INSTANCE);
    }
    // try and read until EOF
    try {
        // the EOFException will get us out of this...
        while (true) {
            byte mappingByte = inputView.readByte();
            ColumnFamilyHandle handle = getColumnFamily(columnFamilyMapping.get(mappingByte), MigrationNamespaceSerializerProxy.INSTANCE);
            byte[] keyAndNamespace = BytePrimitiveArraySerializer.INSTANCE.deserialize(inputView);
            ByteArrayInputStreamWithPos bis = new ByteArrayInputStreamWithPos(keyAndNamespace);
            K reconstructedKey = keySerializer.deserialize(new DataInputViewStreamWrapper(bis));
            int len = bis.getPosition();
            int keyGroup = (byte) KeyGroupRangeAssignment.assignToKeyGroup(reconstructedKey, numberOfKeyGroups);
            if (keyGroupPrefixBytes == 1) {
                // copy and override one byte (42) between key and namespace
                System.arraycopy(keyAndNamespace, 0, keyAndNamespace, 1, len);
                keyAndNamespace[0] = (byte) keyGroup;
            } else {
                byte[] largerKey = new byte[1 + keyAndNamespace.length];
                // write key-group
                largerKey[0] = (byte) ((keyGroup >> 8) & 0xFF);
                largerKey[1] = (byte) (keyGroup & 0xFF);
                // write key
                System.arraycopy(keyAndNamespace, 0, largerKey, 2, len);
                //skip one byte (42), write namespace
                System.arraycopy(keyAndNamespace, 1 + len, largerKey, 2 + len, keyAndNamespace.length - len - 1);
                keyAndNamespace = largerKey;
            }
            byte[] value = BytePrimitiveArraySerializer.INSTANCE.deserialize(inputView);
            db.put(handle, keyAndNamespace, value);
        }
    } catch (EOFException e) {
    // expected
    }
}

Example 4 with FSDataInputStream

use of org.apache.flink.core.fs.FSDataInputStream in project flink by apache.

the class FileSerializableStateHandle method getState.

@Override
@SuppressWarnings("unchecked")
public T getState(ClassLoader classLoader) throws Exception {
    ensureNotClosed();
    try (FSDataInputStream inStream = getFileSystem().open(getFilePath())) {
        // make sure any deserialization can be aborted
        registerCloseable(inStream);
        ObjectInputStream ois = new MigrationInstantiationUtil.ClassLoaderObjectInputStream(inStream, classLoader);
        return (T) ois.readObject();
    }
}

Example 5 with FSDataInputStream

use of org.apache.flink.core.fs.FSDataInputStream in project flink by apache.

the class CheckpointCoordinatorTest method collectResult.

private static void collectResult(int opIdx, OperatorStateHandle operatorStateHandle, List<String> resultCollector) throws Exception {
    try (FSDataInputStream in = operatorStateHandle.openInputStream()) {
        for (Map.Entry<String, OperatorStateHandle.StateMetaInfo> entry : operatorStateHandle.getStateNameToPartitionOffsets().entrySet()) {
            for (long offset : entry.getValue().getOffsets()) {
                in.seek(offset);
                Integer state = InstantiationUtil.deserializeObject(in, Thread.currentThread().getContextClassLoader());
                resultCollector.add(opIdx + " : " + entry.getKey() + " : " + state);
            }
        }
    }
}