Examples with RocksDBIncrementalRestoreOperation org.apache.flink.contrib.streaming.state.restore.RocksDBIncrementalRestoreOperation used on opensource projects

Example 1 with RocksDBIncrementalRestoreOperation

use of org.apache.flink.contrib.streaming.state.restore.RocksDBIncrementalRestoreOperation in project flink by apache.

the class RocksDBKeyedStateBackendBuilder method build.

@Override
public RocksDBKeyedStateBackend<K> build() throws BackendBuildingException {
    RocksDBWriteBatchWrapper writeBatchWrapper = null;
    ColumnFamilyHandle defaultColumnFamilyHandle = null;
    RocksDBNativeMetricMonitor nativeMetricMonitor = null;
    CloseableRegistry cancelStreamRegistryForBackend = new CloseableRegistry();
    LinkedHashMap<String, RocksDBKeyedStateBackend.RocksDbKvStateInfo> kvStateInformation = new LinkedHashMap<>();
    LinkedHashMap<String, HeapPriorityQueueSnapshotRestoreWrapper<?>> registeredPQStates = new LinkedHashMap<>();
    RocksDB db = null;
    RocksDBRestoreOperation restoreOperation = null;
    RocksDbTtlCompactFiltersManager ttlCompactFiltersManager = new RocksDbTtlCompactFiltersManager(ttlTimeProvider);
    ResourceGuard rocksDBResourceGuard = new ResourceGuard();
    RocksDBSnapshotStrategyBase<K, ?> checkpointStrategy = null;
    PriorityQueueSetFactory priorityQueueFactory;
    SerializedCompositeKeyBuilder<K> sharedRocksKeyBuilder;
    // Number of bytes required to prefix the key groups.
    int keyGroupPrefixBytes = CompositeKeySerializationUtils.computeRequiredBytesInKeyGroupPrefix(numberOfKeyGroups);
    try {
        // Variables for snapshot strategy when incremental checkpoint is enabled
        UUID backendUID = UUID.randomUUID();
        SortedMap<Long, Map<StateHandleID, StreamStateHandle>> materializedSstFiles = new TreeMap<>();
        long lastCompletedCheckpointId = -1L;
        if (injectedTestDB != null) {
            db = injectedTestDB;
            defaultColumnFamilyHandle = injectedDefaultColumnFamilyHandle;
            nativeMetricMonitor = nativeMetricOptions.isEnabled() ? new RocksDBNativeMetricMonitor(nativeMetricOptions, metricGroup, db) : null;
        } else {
            prepareDirectories();
            restoreOperation = getRocksDBRestoreOperation(keyGroupPrefixBytes, cancelStreamRegistry, kvStateInformation, registeredPQStates, ttlCompactFiltersManager);
            RocksDBRestoreResult restoreResult = restoreOperation.restore();
            db = restoreResult.getDb();
            defaultColumnFamilyHandle = restoreResult.getDefaultColumnFamilyHandle();
            nativeMetricMonitor = restoreResult.getNativeMetricMonitor();
            if (restoreOperation instanceof RocksDBIncrementalRestoreOperation) {
                backendUID = restoreResult.getBackendUID();
                materializedSstFiles = restoreResult.getRestoredSstFiles();
                lastCompletedCheckpointId = restoreResult.getLastCompletedCheckpointId();
            }
        }
        writeBatchWrapper = new RocksDBWriteBatchWrapper(db, optionsContainer.getWriteOptions(), writeBatchSize);
        // it is important that we only create the key builder after the restore, and not
        // before;
        // restore operations may reconfigure the key serializer, so accessing the key
        // serializer
        // only now we can be certain that the key serializer used in the builder is final.
        sharedRocksKeyBuilder = new SerializedCompositeKeyBuilder<>(keySerializerProvider.currentSchemaSerializer(), keyGroupPrefixBytes, 32);
        // init snapshot strategy after db is assured to be initialized
        checkpointStrategy = initializeSavepointAndCheckpointStrategies(cancelStreamRegistryForBackend, rocksDBResourceGuard, kvStateInformation, registeredPQStates, keyGroupPrefixBytes, db, backendUID, materializedSstFiles, lastCompletedCheckpointId);
        // init priority queue factory
        priorityQueueFactory = initPriorityQueueFactory(keyGroupPrefixBytes, kvStateInformation, db, writeBatchWrapper, nativeMetricMonitor);
    } catch (Throwable e) {
        // Do clean up
        List<ColumnFamilyOptions> columnFamilyOptions = new ArrayList<>(kvStateInformation.values().size());
        IOUtils.closeQuietly(cancelStreamRegistryForBackend);
        IOUtils.closeQuietly(writeBatchWrapper);
        RocksDBOperationUtils.addColumnFamilyOptionsToCloseLater(columnFamilyOptions, defaultColumnFamilyHandle);
        IOUtils.closeQuietly(defaultColumnFamilyHandle);
        IOUtils.closeQuietly(nativeMetricMonitor);
        for (RocksDBKeyedStateBackend.RocksDbKvStateInfo kvStateInfo : kvStateInformation.values()) {
            RocksDBOperationUtils.addColumnFamilyOptionsToCloseLater(columnFamilyOptions, kvStateInfo.columnFamilyHandle);
            IOUtils.closeQuietly(kvStateInfo.columnFamilyHandle);
        }
        IOUtils.closeQuietly(db);
        // it's possible that db has been initialized but later restore steps failed
        IOUtils.closeQuietly(restoreOperation);
        IOUtils.closeAllQuietly(columnFamilyOptions);
        IOUtils.closeQuietly(optionsContainer);
        ttlCompactFiltersManager.disposeAndClearRegisteredCompactionFactories();
        kvStateInformation.clear();
        IOUtils.closeQuietly(checkpointStrategy);
        try {
            FileUtils.deleteDirectory(instanceBasePath);
        } catch (Exception ex) {
            logger.warn("Failed to delete base path for RocksDB: " + instanceBasePath, ex);
        }
        // Log and rethrow
        if (e instanceof BackendBuildingException) {
            throw (BackendBuildingException) e;
        } else {
            String errMsg = "Caught unexpected exception.";
            logger.error(errMsg, e);
            throw new BackendBuildingException(errMsg, e);
        }
    }
    InternalKeyContext<K> keyContext = new InternalKeyContextImpl<>(keyGroupRange, numberOfKeyGroups);
    logger.info("Finished building RocksDB keyed state-backend at {}.", instanceBasePath);
    return new RocksDBKeyedStateBackend<>(this.userCodeClassLoader, this.instanceBasePath, this.optionsContainer, columnFamilyOptionsFactory, this.kvStateRegistry, this.keySerializerProvider.currentSchemaSerializer(), this.executionConfig, this.ttlTimeProvider, latencyTrackingStateConfig, db, kvStateInformation, registeredPQStates, keyGroupPrefixBytes, cancelStreamRegistryForBackend, this.keyGroupCompressionDecorator, rocksDBResourceGuard, checkpointStrategy, writeBatchWrapper, defaultColumnFamilyHandle, nativeMetricMonitor, sharedRocksKeyBuilder, priorityQueueFactory, ttlCompactFiltersManager, keyContext, writeBatchSize);
}

Example 2 with RocksDBIncrementalRestoreOperation

use of org.apache.flink.contrib.streaming.state.restore.RocksDBIncrementalRestoreOperation in project flink by apache.

the class RocksDBKeyedStateBackendBuilder method getRocksDBRestoreOperation.

private RocksDBRestoreOperation getRocksDBRestoreOperation(int keyGroupPrefixBytes, CloseableRegistry cancelStreamRegistry, LinkedHashMap<String, RocksDBKeyedStateBackend.RocksDbKvStateInfo> kvStateInformation, LinkedHashMap<String, HeapPriorityQueueSnapshotRestoreWrapper<?>> registeredPQStates, RocksDbTtlCompactFiltersManager ttlCompactFiltersManager) {
    DBOptions dbOptions = optionsContainer.getDbOptions();
    if (restoreStateHandles.isEmpty()) {
        return new RocksDBNoneRestoreOperation<>(kvStateInformation, instanceRocksDBPath, dbOptions, columnFamilyOptionsFactory, nativeMetricOptions, metricGroup, ttlCompactFiltersManager, optionsContainer.getWriteBufferManagerCapacity());
    }
    KeyedStateHandle firstStateHandle = restoreStateHandles.iterator().next();
    if (firstStateHandle instanceof IncrementalKeyedStateHandle) {
        return new RocksDBIncrementalRestoreOperation<>(operatorIdentifier, keyGroupRange, keyGroupPrefixBytes, numberOfTransferingThreads, cancelStreamRegistry, userCodeClassLoader, kvStateInformation, keySerializerProvider, instanceBasePath, instanceRocksDBPath, dbOptions, columnFamilyOptionsFactory, nativeMetricOptions, metricGroup, restoreStateHandles, ttlCompactFiltersManager, writeBatchSize, optionsContainer.getWriteBufferManagerCapacity());
    } else if (priorityQueueStateType == EmbeddedRocksDBStateBackend.PriorityQueueStateType.HEAP) {
        return new RocksDBHeapTimersFullRestoreOperation<>(keyGroupRange, numberOfKeyGroups, userCodeClassLoader, kvStateInformation, registeredPQStates, createHeapQueueFactory(), keySerializerProvider, instanceRocksDBPath, dbOptions, columnFamilyOptionsFactory, nativeMetricOptions, metricGroup, restoreStateHandles, ttlCompactFiltersManager, writeBatchSize, optionsContainer.getWriteBufferManagerCapacity());
    } else {
        return new RocksDBFullRestoreOperation<>(keyGroupRange, userCodeClassLoader, kvStateInformation, keySerializerProvider, instanceRocksDBPath, dbOptions, columnFamilyOptionsFactory, nativeMetricOptions, metricGroup, restoreStateHandles, ttlCompactFiltersManager, writeBatchSize, optionsContainer.getWriteBufferManagerCapacity());
    }
}