Examples with ProcessingTimeService org.apache.flink.streaming.runtime.tasks.ProcessingTimeService used on opensource projects

Example 6 with ProcessingTimeService

use of org.apache.flink.streaming.runtime.tasks.ProcessingTimeService in project beam by apache.

the class ExecutableStageDoFnOperator method finishBundleCallback.

@SuppressWarnings("FutureReturnValueIgnored")
private void finishBundleCallback() {
    minEventTimeTimerTimestampInLastBundle = minEventTimeTimerTimestampInCurrentBundle;
    minEventTimeTimerTimestampInCurrentBundle = Long.MAX_VALUE;
    try {
        if (!closed && minEventTimeTimerTimestampInLastBundle < Long.MAX_VALUE && minEventTimeTimerTimestampInLastBundle <= getEffectiveInputWatermark()) {
            ProcessingTimeService processingTimeService = getProcessingTimeService();
            // We are scheduling a timer for advancing the watermark, to not delay finishing the bundle
            // and temporarily release the checkpoint lock. Otherwise, we could potentially loop when a
            // timer keeps scheduling a timer for the same timestamp.
            processingTimeService.registerTimer(processingTimeService.getCurrentProcessingTime(), ts -> processWatermark1(new Watermark(getEffectiveInputWatermark())));
        } else {
            processWatermark1(new Watermark(getEffectiveInputWatermark()));
        }
    } catch (Exception e) {
        throw new RuntimeException("Failed to progress watermark to " + getEffectiveInputWatermark(), e);
    }
}

Example 7 with ProcessingTimeService

use of org.apache.flink.streaming.runtime.tasks.ProcessingTimeService in project flink by apache.

the class StreamTaskTimerTest method checkScheduledTimestampe.

@Test
public void checkScheduledTimestampe() {
    try {
        final OneInputStreamTask<String, String> mapTask = new OneInputStreamTask<>();
        final OneInputStreamTaskTestHarness<String, String> testHarness = new OneInputStreamTaskTestHarness<>(mapTask, BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO);
        testHarness.setupOutputForSingletonOperatorChain();
        StreamConfig streamConfig = testHarness.getStreamConfig();
        StreamMap<String, String> mapOperator = new StreamMap<>(new DummyMapFunction<String>());
        streamConfig.setStreamOperator(mapOperator);
        testHarness.invoke();
        testHarness.waitForTaskRunning();
        final AtomicReference<Throwable> errorRef = new AtomicReference<>();
        final long t1 = System.currentTimeMillis();
        final long t2 = System.currentTimeMillis() - 200;
        final long t3 = System.currentTimeMillis() + 100;
        final long t4 = System.currentTimeMillis() + 200;
        ProcessingTimeService timeService = mapTask.getProcessingTimeService();
        timeService.registerTimer(t1, new ValidatingProcessingTimeCallback(errorRef, t1, 0));
        timeService.registerTimer(t2, new ValidatingProcessingTimeCallback(errorRef, t2, 1));
        timeService.registerTimer(t3, new ValidatingProcessingTimeCallback(errorRef, t3, 2));
        timeService.registerTimer(t4, new ValidatingProcessingTimeCallback(errorRef, t4, 3));
        long deadline = System.currentTimeMillis() + 20000;
        while (errorRef.get() == null && ValidatingProcessingTimeCallback.numInSequence < 4 && System.currentTimeMillis() < deadline) {
            Thread.sleep(100);
        }
        // handle errors
        if (errorRef.get() != null) {
            errorRef.get().printStackTrace();
            fail(errorRef.get().getMessage());
        }
        assertEquals(4, ValidatingProcessingTimeCallback.numInSequence);
        testHarness.endInput();
        testHarness.waitForTaskCompletion();
        // wait until the trigger thread is shut down. otherwise, the other tests may become unstable
        deadline = System.currentTimeMillis() + 4000;
        while (StreamTask.TRIGGER_THREAD_GROUP.activeCount() > 0 && System.currentTimeMillis() < deadline) {
            Thread.sleep(10);
        }
        assertEquals("Trigger timer thread did not properly shut down", 0, StreamTask.TRIGGER_THREAD_GROUP.activeCount());
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 8 with ProcessingTimeService

use of org.apache.flink.streaming.runtime.tasks.ProcessingTimeService in project flink by apache.

the class KinesisDataFetcher method runFetcher.

/**
 * Starts the fetcher. After starting the fetcher, it can only be stopped by calling {@link
 * KinesisDataFetcher#shutdownFetcher()}.
 *
 * @throws Exception the first error or exception thrown by the fetcher or any of the threads
 *     created by the fetcher.
 */
public void runFetcher() throws Exception {
    // check that we are running before proceeding
    if (!running) {
        return;
    }
    // ------------------------------------------------------------------------
    // Procedures before starting the infinite while loop:
    // ------------------------------------------------------------------------
    // 1. check that there is at least one shard in the subscribed streams to consume from (can
    // be done by
    // checking if at least one value in subscribedStreamsToLastDiscoveredShardIds is not
    // null)
    boolean hasShards = false;
    StringBuilder streamsWithNoShardsFound = new StringBuilder();
    for (Map.Entry<String, String> streamToLastDiscoveredShardEntry : subscribedStreamsToLastDiscoveredShardIds.entrySet()) {
        if (streamToLastDiscoveredShardEntry.getValue() != null) {
            hasShards = true;
        } else {
            streamsWithNoShardsFound.append(streamToLastDiscoveredShardEntry.getKey()).append(", ");
        }
    }
    if (streamsWithNoShardsFound.length() != 0 && LOG.isWarnEnabled()) {
        LOG.warn("Subtask {} has failed to find any shards for the following subscribed streams: {}", indexOfThisConsumerSubtask, streamsWithNoShardsFound.toString());
    }
    if (!hasShards) {
        throw new RuntimeException("No shards can be found for all subscribed streams: " + streams);
    }
    // consumer using a restored state checkpoint
    for (int seededStateIndex = 0; seededStateIndex < subscribedShardsState.size(); seededStateIndex++) {
        KinesisStreamShardState seededShardState = subscribedShardsState.get(seededStateIndex);
        // read already
        if (!seededShardState.getLastProcessedSequenceNum().equals(SentinelSequenceNumber.SENTINEL_SHARD_ENDING_SEQUENCE_NUM.get())) {
            if (LOG.isInfoEnabled()) {
                LOG.info("Subtask {} will start consuming seeded shard {} from sequence number {} with ShardConsumer {}", indexOfThisConsumerSubtask, seededShardState.getStreamShardHandle().toString(), seededShardState.getLastProcessedSequenceNum(), seededStateIndex);
            }
            StreamShardHandle streamShardHandle = subscribedShardsState.get(seededStateIndex).getStreamShardHandle();
            KinesisDeserializationSchema<T> shardDeserializationSchema = getClonedDeserializationSchema();
            shardDeserializationSchema.open(RuntimeContextInitializationContextAdapters.deserializationAdapter(runtimeContext, // ignore the provided metric group
            metricGroup -> consumerMetricGroup.addGroup("subtaskId", String.valueOf(indexOfThisConsumerSubtask)).addGroup("shardId", streamShardHandle.getShard().getShardId()).addGroup("user")));
            shardConsumersExecutor.submit(createShardConsumer(seededStateIndex, streamShardHandle, subscribedShardsState.get(seededStateIndex).getLastProcessedSequenceNum(), registerShardMetricGroup(consumerMetricGroup, subscribedShardsState.get(seededStateIndex)), shardDeserializationSchema));
        }
    }
    // start periodic watermark emitter, if a watermark assigner was configured
    if (periodicWatermarkAssigner != null) {
        long periodicWatermarkIntervalMillis = runtimeContext.getExecutionConfig().getAutoWatermarkInterval();
        if (periodicWatermarkIntervalMillis > 0) {
            ProcessingTimeService timerService = ((StreamingRuntimeContext) runtimeContext).getProcessingTimeService();
            LOG.info("Starting periodic watermark emitter with interval {}", periodicWatermarkIntervalMillis);
            new PeriodicWatermarkEmitter(timerService, periodicWatermarkIntervalMillis).start();
            if (watermarkTracker != null) {
                // setup global watermark tracking
                long watermarkSyncMillis = Long.parseLong(getConsumerConfiguration().getProperty(ConsumerConfigConstants.WATERMARK_SYNC_MILLIS, Long.toString(ConsumerConfigConstants.DEFAULT_WATERMARK_SYNC_MILLIS)));
                watermarkTracker.setUpdateTimeoutMillis(// synchronization latency
                watermarkSyncMillis * 3);
                watermarkTracker.open(runtimeContext);
                new WatermarkSyncCallback(timerService, watermarkSyncMillis).start();
                // emit records ahead of watermark to offset synchronization latency
                long lookaheadMillis = Long.parseLong(getConsumerConfiguration().getProperty(ConsumerConfigConstants.WATERMARK_LOOKAHEAD_MILLIS, Long.toString(0)));
                recordEmitter.setMaxLookaheadMillis(Math.max(lookaheadMillis, watermarkSyncMillis * 3));
                // record emitter depends on periodic watermark
                // it runs in a separate thread since main thread is used for discovery
                Runnable recordEmitterRunnable = new Runnable() {

                    @Override
                    public void run() {
                        try {
                            recordEmitter.run();
                        } catch (Throwable error) {
                            // report the error that terminated the emitter loop to
                            // source thread
                            stopWithError(error);
                        }
                    }
                };
                Thread thread = new Thread(recordEmitterRunnable);
                thread.setName("recordEmitter-" + runtimeContext.getTaskNameWithSubtasks());
                thread.setDaemon(true);
                thread.start();
            }
        }
        this.shardIdleIntervalMillis = Long.parseLong(getConsumerConfiguration().getProperty(ConsumerConfigConstants.SHARD_IDLE_INTERVAL_MILLIS, Long.toString(ConsumerConfigConstants.DEFAULT_SHARD_IDLE_INTERVAL_MILLIS)));
    }
    // ------------------------------------------------------------------------
    // finally, start the infinite shard discovery and consumer launching loop;
    // we will escape from this loop only when shutdownFetcher() or stopWithError() is called
    // TODO: have this thread emit the records for tracking backpressure
    final long discoveryIntervalMillis = Long.parseLong(configProps.getProperty(ConsumerConfigConstants.SHARD_DISCOVERY_INTERVAL_MILLIS, Long.toString(ConsumerConfigConstants.DEFAULT_SHARD_DISCOVERY_INTERVAL_MILLIS)));
    if (this.numberOfActiveShards.get() == 0) {
        LOG.info("Subtask {} has no active shards to read on startup; marking the subtask as temporarily idle ...", indexOfThisConsumerSubtask);
        sourceContext.markAsTemporarilyIdle();
    }
    while (running) {
        if (LOG.isDebugEnabled()) {
            LOG.debug("Subtask {} is trying to discover new shards that were created due to resharding ...", indexOfThisConsumerSubtask);
        }
        List<StreamShardHandle> newShardsDueToResharding = discoverNewShardsToSubscribe();
        for (StreamShardHandle shard : newShardsDueToResharding) {
            // since there may be delay in discovering a new shard, all new shards due to
            // resharding should be read starting from the earliest record possible
            KinesisStreamShardState newShardState = new KinesisStreamShardState(convertToStreamShardMetadata(shard), shard, SentinelSequenceNumber.SENTINEL_EARLIEST_SEQUENCE_NUM.get());
            int newStateIndex = registerNewSubscribedShardState(newShardState);
            if (LOG.isInfoEnabled()) {
                LOG.info("Subtask {} has discovered a new shard {} due to resharding, and will start consuming " + "the shard from sequence number {} with ShardConsumer {}", indexOfThisConsumerSubtask, newShardState.getStreamShardHandle().toString(), newShardState.getLastProcessedSequenceNum(), newStateIndex);
            }
            StreamShardHandle streamShardHandle = newShardState.getStreamShardHandle();
            KinesisDeserializationSchema<T> shardDeserializationSchema = getClonedDeserializationSchema();
            shardDeserializationSchema.open(RuntimeContextInitializationContextAdapters.deserializationAdapter(runtimeContext, // ignore the provided metric group
            metricGroup -> consumerMetricGroup.addGroup("subtaskId", String.valueOf(indexOfThisConsumerSubtask)).addGroup("shardId", streamShardHandle.getShard().getShardId()).addGroup("user")));
            shardConsumersExecutor.submit(createShardConsumer(newStateIndex, newShardState.getStreamShardHandle(), newShardState.getLastProcessedSequenceNum(), registerShardMetricGroup(consumerMetricGroup, newShardState), shardDeserializationSchema));
        }
        // interval if the running flag was set to false during the middle of the while loop
        if (running && discoveryIntervalMillis != 0) {
            try {
                cancelFuture.get(discoveryIntervalMillis, TimeUnit.MILLISECONDS);
                LOG.debug("Cancelled discovery");
            } catch (TimeoutException iex) {
            // timeout is expected when fetcher is not cancelled
            }
        }
    }
    // make sure all resources have been terminated before leaving
    try {
        awaitTermination();
    } catch (InterruptedException ie) {
        // If there is an original exception, preserve it, since that's more important/useful.
        this.error.compareAndSet(null, ie);
    }
    // any error thrown in the shard consumer threads will be thrown to the main thread
    Throwable throwable = this.error.get();
    if (throwable != null) {
        if (throwable instanceof Exception) {
            throw (Exception) throwable;
        } else if (throwable instanceof Error) {
            throw (Error) throwable;
        } else {
            throw new Exception(throwable);
        }
    }
}

Example 9 with ProcessingTimeService

use of org.apache.flink.streaming.runtime.tasks.ProcessingTimeService in project flink by apache.

the class StreamOperatorFactoryUtil method createOperator.

/**
 * Creates a new operator using a factory and makes sure that all special factory traits are
 * properly handled.
 *
 * @param operatorFactory the operator factory.
 * @param containingTask the containing task.
 * @param configuration the configuration of the operator.
 * @param output the output of the operator.
 * @param operatorEventDispatcher the operator event dispatcher for communication between
 *     operator and coordinators.
 * @return a newly created and configured operator, and the {@link ProcessingTimeService}
 *     instance it can access.
 */
public static <OUT, OP extends StreamOperator<OUT>> Tuple2<OP, Optional<ProcessingTimeService>> createOperator(StreamOperatorFactory<OUT> operatorFactory, StreamTask<OUT, ?> containingTask, StreamConfig configuration, Output<StreamRecord<OUT>> output, OperatorEventDispatcher operatorEventDispatcher) {
    MailboxExecutor mailboxExecutor = containingTask.getMailboxExecutorFactory().createExecutor(configuration.getChainIndex());
    if (operatorFactory instanceof YieldingOperatorFactory) {
        ((YieldingOperatorFactory<?>) operatorFactory).setMailboxExecutor(mailboxExecutor);
    }
    final Supplier<ProcessingTimeService> processingTimeServiceFactory = () -> containingTask.getProcessingTimeServiceFactory().createProcessingTimeService(mailboxExecutor);
    final ProcessingTimeService processingTimeService;
    if (operatorFactory instanceof ProcessingTimeServiceAware) {
        processingTimeService = processingTimeServiceFactory.get();
        ((ProcessingTimeServiceAware) operatorFactory).setProcessingTimeService(processingTimeService);
    } else {
        processingTimeService = null;
    }
    // TODO: what to do with ProcessingTimeServiceAware?
    OP op = operatorFactory.createStreamOperator(new StreamOperatorParameters<>(containingTask, configuration, output, processingTimeService != null ? () -> processingTimeService : processingTimeServiceFactory, operatorEventDispatcher));
    return new Tuple2<>(op, Optional.ofNullable(processingTimeService));
}

Example 10 with ProcessingTimeService

use of org.apache.flink.streaming.runtime.tasks.ProcessingTimeService in project flink by apache.

the class StreamOperatorSnapshotRestoreTest method testOperatorStatesSnapshotRestoreInternal.

private void testOperatorStatesSnapshotRestoreInternal(final int mode) throws Exception {
    // -------------------------------------------------------------------------- snapshot
    StateBackend stateBackend;
    FsStateBackend fsstateBackend = createStateBackendInternal();
    switch(stateBackendEnum) {
        case FILE:
            stateBackend = fsstateBackend;
            break;
        case ROCKSDB_FULLY_ASYNC:
            stateBackend = new RocksDBStateBackend(fsstateBackend, TernaryBoolean.FALSE);
            break;
        case ROCKSDB_INCREMENTAL:
            stateBackend = new RocksDBStateBackend(fsstateBackend, TernaryBoolean.TRUE);
            break;
        default:
            throw new IllegalStateException(String.format("Do not support statebackend type %s", stateBackendEnum));
    }
    TestOneInputStreamOperator op = new TestOneInputStreamOperator(false);
    JobID jobID = new JobID();
    JobVertexID jobVertexID = new JobVertexID();
    int subtaskIdx = 0;
    LocalRecoveryDirectoryProvider directoryProvider = mode == ONLY_JM_RECOVERY ? null : new LocalRecoveryDirectoryProviderImpl(temporaryFolder.newFolder(), jobID, jobVertexID, subtaskIdx);
    LocalRecoveryConfig localRecoveryConfig = new LocalRecoveryConfig(directoryProvider);
    MockEnvironment mockEnvironment = new MockEnvironmentBuilder().setJobID(jobID).setJobVertexID(jobVertexID).setTaskName("test").setManagedMemorySize(1024L * 1024L).setInputSplitProvider(new MockInputSplitProvider()).setBufferSize(1024 * 1024).setTaskStateManager(new TestTaskStateManager(localRecoveryConfig)).setMaxParallelism(MAX_PARALLELISM).setSubtaskIndex(subtaskIdx).setUserCodeClassLoader(getClass().getClassLoader()).build();
    KeyedOneInputStreamOperatorTestHarness<Integer, Integer, Integer> testHarness = new KeyedOneInputStreamOperatorTestHarness<>(op, (KeySelector<Integer, Integer>) value -> value, TypeInformation.of(Integer.class), mockEnvironment);
    testHarness.setStateBackend(stateBackend);
    testHarness.open();
    for (int i = 0; i < 10; ++i) {
        testHarness.processElement(new StreamRecord<>(i));
    }
    OperatorSnapshotFinalizer snapshotWithLocalState = testHarness.snapshotWithLocalState(1L, 1L);
    testHarness.close();
    // -------------------------------------------------------------------------- restore
    op = new TestOneInputStreamOperator(true);
    testHarness = new KeyedOneInputStreamOperatorTestHarness<>(op, (KeySelector<Integer, Integer>) value -> value, TypeInformation.of(Integer.class), MAX_PARALLELISM, 1, /* num subtasks */
    0);
    testHarness.setTimeServiceManagerProvider(new InternalTimeServiceManager.Provider() {

        @Override
        public <K> InternalTimeServiceManager<K> create(CheckpointableKeyedStateBackend<K> keyedStatedBackend, ClassLoader userClassloader, KeyContext keyContext, ProcessingTimeService processingTimeService, Iterable<KeyGroupStatePartitionStreamProvider> rawKeyedStates) throws IOException {
            return null;
        }
    });
    testHarness.setStateBackend(stateBackend);
    OperatorSubtaskState jobManagerOwnedState = snapshotWithLocalState.getJobManagerOwnedState();
    OperatorSubtaskState taskLocalState = snapshotWithLocalState.getTaskLocalState();
    // We check if local state was created when we enabled local recovery
    Assert.assertTrue(mode > ONLY_JM_RECOVERY == (taskLocalState != null && taskLocalState.hasState()));
    if (mode == TM_REMOVE_JM_RECOVERY) {
        jobManagerOwnedState.getManagedKeyedState().discardState();
    } else if (mode == JM_REMOVE_TM_RECOVERY) {
        taskLocalState.getManagedKeyedState().discardState();
    }
    testHarness.initializeState(jobManagerOwnedState, taskLocalState);
    testHarness.open();
    for (int i = 0; i < 10; ++i) {
        testHarness.processElement(new StreamRecord<>(i));
    }
    testHarness.close();
}