Examples with MaterializationRunnable org.apache.flink.state.changelog.PeriodicMaterializationManager.MaterializationRunnable used on opensource projects

Example 1 with MaterializationRunnable

use of org.apache.flink.state.changelog.PeriodicMaterializationManager.MaterializationRunnable in project flink by apache.

the class ChangelogKeyedStateBackend method initMaterialization.

/**
 * Initialize state materialization so that materialized data can be persisted durably and
 * included into the checkpoint.
 *
 * <p>This method is not thread safe. It should be called either under a lock or through task
 * mailbox executor.
 *
 * @return a tuple of - future snapshot result from the underlying state backend - a {@link
 *     SequenceNumber} identifying the latest change in the changelog
 */
public Optional<MaterializationRunnable> initMaterialization() throws Exception {
    SequenceNumber upTo = stateChangelogWriter.nextSequenceNumber();
    SequenceNumber lastMaterializedTo = changelogSnapshotState.lastMaterializedTo();
    LOG.info("Initialize Materialization. Current changelog writers last append to sequence number {}", upTo);
    if (upTo.compareTo(lastMaterializedTo) > 0) {
        LOG.info("Starting materialization from {} : {}", lastMaterializedTo, upTo);
        // This ID is not needed for materialization; But since we are re-using the
        // streamFactory that is designed for state backend snapshot, which requires unique
        // checkpoint ID. A faked materialized Id is provided here.
        long materializationID = materializedId++;
        MaterializationRunnable materializationRunnable = new MaterializationRunnable(keyedStateBackend.snapshot(materializationID, System.currentTimeMillis(), // TODO: implement its own streamFactory.
        streamFactory, CHECKPOINT_OPTIONS), materializationID, upTo);
        // log metadata after materialization is triggered
        for (ChangelogState changelogState : changelogStates.values()) {
            changelogState.resetWritingMetaFlag();
        }
        for (ChangelogKeyGroupedPriorityQueue<?> priorityQueueState : priorityQueueStatesByName.values()) {
            priorityQueueState.resetWritingMetaFlag();
        }
        return Optional.of(materializationRunnable);
    } else {
        LOG.debug("Skip materialization, last materialized to {} : last log to {}", lastMaterializedTo, upTo);
        return Optional.empty();
    }
}