Examples with FlinkRuntimeException org.apache.flink.util.FlinkRuntimeException used on opensource projects

Example 56 with FlinkRuntimeException

use of org.apache.flink.util.FlinkRuntimeException in project flink by apache.

the class RocksDBKeyedStateBackend method getKeys.

@SuppressWarnings("unchecked")
@Override
public <N> Stream<K> getKeys(String state, N namespace) {
    RocksDbKvStateInfo columnInfo = kvStateInformation.get(state);
    if (columnInfo == null || !(columnInfo.metaInfo instanceof RegisteredKeyValueStateBackendMetaInfo)) {
        return Stream.empty();
    }
    RegisteredKeyValueStateBackendMetaInfo<N, ?> registeredKeyValueStateBackendMetaInfo = (RegisteredKeyValueStateBackendMetaInfo<N, ?>) columnInfo.metaInfo;
    final TypeSerializer<N> namespaceSerializer = registeredKeyValueStateBackendMetaInfo.getNamespaceSerializer();
    final DataOutputSerializer namespaceOutputView = new DataOutputSerializer(8);
    boolean ambiguousKeyPossible = CompositeKeySerializationUtils.isAmbiguousKeyPossible(getKeySerializer(), namespaceSerializer);
    final byte[] nameSpaceBytes;
    try {
        CompositeKeySerializationUtils.writeNameSpace(namespace, namespaceSerializer, namespaceOutputView, ambiguousKeyPossible);
        nameSpaceBytes = namespaceOutputView.getCopyOfBuffer();
    } catch (IOException ex) {
        throw new FlinkRuntimeException("Failed to get keys from RocksDB state backend.", ex);
    }
    RocksIteratorWrapper iterator = RocksDBOperationUtils.getRocksIterator(db, columnInfo.columnFamilyHandle, readOptions);
    iterator.seekToFirst();
    final RocksStateKeysIterator<K> iteratorWrapper = new RocksStateKeysIterator<>(iterator, state, getKeySerializer(), keyGroupPrefixBytes, ambiguousKeyPossible, nameSpaceBytes);
    Stream<K> targetStream = StreamSupport.stream(Spliterators.spliteratorUnknownSize(iteratorWrapper, Spliterator.ORDERED), false);
    return targetStream.onClose(iteratorWrapper::close);
}

Example 57 with FlinkRuntimeException

use of org.apache.flink.util.FlinkRuntimeException in project flink by apache.

the class RocksDBKeyedStateBackend method migrateStateValues.

/**
 * Migrate only the state value, that is the "value" that is stored in RocksDB. We don't migrate
 * the key here, which is made up of key group, key, namespace and map key (in case of
 * MapState).
 */
@SuppressWarnings("unchecked")
private <N, S extends State, SV> void migrateStateValues(StateDescriptor<S, SV> stateDesc, Tuple2<ColumnFamilyHandle, RegisteredKeyValueStateBackendMetaInfo<N, SV>> stateMetaInfo) throws Exception {
    if (stateDesc.getType() == StateDescriptor.Type.MAP) {
        TypeSerializerSnapshot<SV> previousSerializerSnapshot = stateMetaInfo.f1.getPreviousStateSerializerSnapshot();
        checkState(previousSerializerSnapshot != null, "the previous serializer snapshot should exist.");
        checkState(previousSerializerSnapshot instanceof MapSerializerSnapshot, "previous serializer snapshot should be a MapSerializerSnapshot.");
        TypeSerializer<SV> newSerializer = stateMetaInfo.f1.getStateSerializer();
        checkState(newSerializer instanceof MapSerializer, "new serializer should be a MapSerializer.");
        MapSerializer<?, ?> mapSerializer = (MapSerializer<?, ?>) newSerializer;
        MapSerializerSnapshot<?, ?> mapSerializerSnapshot = (MapSerializerSnapshot<?, ?>) previousSerializerSnapshot;
        if (!checkMapStateKeySchemaCompatibility(mapSerializerSnapshot, mapSerializer)) {
            throw new StateMigrationException("The new serializer for a MapState requires state migration in order for the job to proceed, since the key schema has changed. However, migration for MapState currently only allows value schema evolutions.");
        }
    }
    LOG.info("Performing state migration for state {} because the state serializer's schema, i.e. serialization format, has changed.", stateDesc);
    // we need to get an actual state instance because migration is different
    // for different state types. For example, ListState needs to deal with
    // individual elements
    StateFactory stateFactory = STATE_FACTORIES.get(stateDesc.getType());
    if (stateFactory == null) {
        String message = String.format("State %s is not supported by %s", stateDesc.getClass(), this.getClass());
        throw new FlinkRuntimeException(message);
    }
    State state = stateFactory.createState(stateDesc, stateMetaInfo, RocksDBKeyedStateBackend.this);
    if (!(state instanceof AbstractRocksDBState)) {
        throw new FlinkRuntimeException("State should be an AbstractRocksDBState but is " + state);
    }
    @SuppressWarnings("unchecked") AbstractRocksDBState<?, ?, SV> rocksDBState = (AbstractRocksDBState<?, ?, SV>) state;
    Snapshot rocksDBSnapshot = db.getSnapshot();
    try (RocksIteratorWrapper iterator = RocksDBOperationUtils.getRocksIterator(db, stateMetaInfo.f0, readOptions);
        RocksDBWriteBatchWrapper batchWriter = new RocksDBWriteBatchWrapper(db, getWriteOptions(), getWriteBatchSize())) {
        iterator.seekToFirst();
        DataInputDeserializer serializedValueInput = new DataInputDeserializer();
        DataOutputSerializer migratedSerializedValueOutput = new DataOutputSerializer(512);
        while (iterator.isValid()) {
            serializedValueInput.setBuffer(iterator.value());
            rocksDBState.migrateSerializedValue(serializedValueInput, migratedSerializedValueOutput, stateMetaInfo.f1.getPreviousStateSerializer(), stateMetaInfo.f1.getStateSerializer());
            batchWriter.put(stateMetaInfo.f0, iterator.key(), migratedSerializedValueOutput.getCopyOfBuffer());
            migratedSerializedValueOutput.clear();
            iterator.next();
        }
    } finally {
        db.releaseSnapshot(rocksDBSnapshot);
        rocksDBSnapshot.close();
    }
}

Example 58 with FlinkRuntimeException

use of org.apache.flink.util.FlinkRuntimeException in project flink by apache.

the class RocksDBListState method mergeNamespaces.

@Override
public void mergeNamespaces(N target, Collection<N> sources) {
    if (sources == null || sources.isEmpty()) {
        return;
    }
    try {
        // create the target full-binary-key
        setCurrentNamespace(target);
        final byte[] targetKey = serializeCurrentKeyWithGroupAndNamespace();
        // merge the sources to the target
        for (N source : sources) {
            if (source != null) {
                setCurrentNamespace(source);
                final byte[] sourceKey = serializeCurrentKeyWithGroupAndNamespace();
                byte[] valueBytes = backend.db.get(columnFamily, sourceKey);
                if (valueBytes != null) {
                    backend.db.delete(columnFamily, writeOptions, sourceKey);
                    backend.db.merge(columnFamily, writeOptions, targetKey, valueBytes);
                }
            }
        }
    } catch (Exception e) {
        throw new FlinkRuntimeException("Error while merging state in RocksDB", e);
    }
}

Example 59 with FlinkRuntimeException

use of org.apache.flink.util.FlinkRuntimeException in project flink by apache.

the class RocksDBListState method getInternal.

@Override
public List<V> getInternal() {
    try {
        byte[] key = serializeCurrentKeyWithGroupAndNamespace();
        byte[] valueBytes = backend.db.get(columnFamily, key);
        return listSerializer.deserializeList(valueBytes, elementSerializer);
    } catch (RocksDBException e) {
        throw new FlinkRuntimeException("Error while retrieving data from RocksDB", e);
    }
}

Example 60 with FlinkRuntimeException

use of org.apache.flink.util.FlinkRuntimeException in project flink by apache.

the class AbstractRocksDBState method getValueBytes.

byte[] getValueBytes(V value) {
    try {
        dataOutputView.clear();
        valueSerializer.serialize(value, dataOutputView);
        return dataOutputView.getCopyOfBuffer();
    } catch (IOException e) {
        throw new FlinkRuntimeException("Error while serializing value", e);
    }
}