Examples with OperatorID org.apache.flink.runtime.jobgraph.OperatorID used on opensource projects

Example 51 with OperatorID

use of org.apache.flink.runtime.jobgraph.OperatorID in project flink by apache.

the class StreamOperatorStateHandlerTest method testFailingBackendSnapshotMethod.

/**
 * Tests that a failing snapshot method call to the keyed state backend will trigger the closing
 * of the StateSnapshotContextSynchronousImpl and the cancellation of the
 * OperatorSnapshotResult. The latter is supposed to also cancel all assigned futures.
 */
@Test
public void testFailingBackendSnapshotMethod() throws Exception {
    final long checkpointId = 42L;
    final long timestamp = 1L;
    try (CloseableRegistry closeableRegistry = new CloseableRegistry()) {
        RunnableFuture<SnapshotResult<KeyedStateHandle>> keyedStateManagedFuture = new CancelableFuture<>();
        RunnableFuture<SnapshotResult<KeyedStateHandle>> keyedStateRawFuture = new CancelableFuture<>();
        RunnableFuture<SnapshotResult<OperatorStateHandle>> operatorStateManagedFuture = new CancelableFuture<>();
        RunnableFuture<SnapshotResult<OperatorStateHandle>> operatorStateRawFuture = new CancelableFuture<>();
        RunnableFuture<SnapshotResult<StateObjectCollection<InputChannelStateHandle>>> inputChannelStateFuture = new CancelableFuture<>();
        RunnableFuture<SnapshotResult<StateObjectCollection<ResultSubpartitionStateHandle>>> resultSubpartitionStateFuture = new CancelableFuture<>();
        OperatorSnapshotFutures operatorSnapshotResult = new OperatorSnapshotFutures(keyedStateManagedFuture, keyedStateRawFuture, operatorStateManagedFuture, operatorStateRawFuture, inputChannelStateFuture, resultSubpartitionStateFuture);
        StateSnapshotContextSynchronousImpl context = new TestStateSnapshotContextSynchronousImpl(checkpointId, timestamp, closeableRegistry);
        context.getRawKeyedOperatorStateOutput();
        context.getRawOperatorStateOutput();
        StreamTaskStateInitializerImpl stateInitializer = new StreamTaskStateInitializerImpl(new MockEnvironmentBuilder().build(), new MemoryStateBackend());
        StreamOperatorStateContext stateContext = stateInitializer.streamOperatorStateContext(new OperatorID(), "whatever", new TestProcessingTimeService(), new UnUsedKeyContext(), IntSerializer.INSTANCE, closeableRegistry, new InterceptingOperatorMetricGroup(), 1.0, false);
        StreamOperatorStateHandler stateHandler = new StreamOperatorStateHandler(stateContext, new ExecutionConfig(), closeableRegistry);
        final String keyedStateField = "keyedStateField";
        final String operatorStateField = "operatorStateField";
        CheckpointedStreamOperator checkpointedStreamOperator = new CheckpointedStreamOperator() {

            @Override
            public void initializeState(StateInitializationContext context) throws Exception {
                context.getKeyedStateStore().getState(new ValueStateDescriptor<>(keyedStateField, LongSerializer.INSTANCE)).update(42L);
                context.getOperatorStateStore().getListState(new ListStateDescriptor<>(operatorStateField, LongSerializer.INSTANCE)).add(42L);
            }

            @Override
            public void snapshotState(StateSnapshotContext context) throws Exception {
                throw new ExpectedTestException();
            }
        };
        stateHandler.setCurrentKey("44");
        stateHandler.initializeOperatorState(checkpointedStreamOperator);
        assertThat(stateContext.operatorStateBackend().getRegisteredStateNames(), is(not(empty())));
        assertThat(((AbstractKeyedStateBackend<?>) stateContext.keyedStateBackend()).numKeyValueStatesByName(), equalTo(1));
        try {
            stateHandler.snapshotState(checkpointedStreamOperator, Optional.of(stateContext.internalTimerServiceManager()), "42", 42, 42, CheckpointOptions.forCheckpointWithDefaultLocation(), new MemCheckpointStreamFactory(1024), operatorSnapshotResult, context, false);
            fail("Exception expected.");
        } catch (CheckpointException e) {
            // as CheckpointException is wrapping the cause with SerializedThrowable
            if (!ExceptionUtils.findThrowableWithMessage(e, ExpectedTestException.MESSAGE).isPresent()) {
                throw e;
            }
        }
        assertTrue(keyedStateManagedFuture.isCancelled());
        assertTrue(keyedStateRawFuture.isCancelled());
        assertTrue(context.getKeyedStateStreamFuture().isCancelled());
        assertTrue(operatorStateManagedFuture.isCancelled());
        assertTrue(operatorStateRawFuture.isCancelled());
        assertTrue(context.getOperatorStateStreamFuture().isCancelled());
        assertTrue(inputChannelStateFuture.isCancelled());
        assertTrue(resultSubpartitionStateFuture.isCancelled());
        stateHandler.dispose();
        assertThat(stateContext.operatorStateBackend().getRegisteredBroadcastStateNames(), is(empty()));
        assertThat(stateContext.operatorStateBackend().getRegisteredStateNames(), is(empty()));
        assertThat(((AbstractKeyedStateBackend<?>) stateContext.keyedStateBackend()).numKeyValueStatesByName(), equalTo(0));
    }
}

Example 52 with OperatorID

use of org.apache.flink.runtime.jobgraph.OperatorID in project flink by apache.

the class String2SortMergeJoinOperatorTest method buildSortMergeJoin.

private TwoInputStreamTaskTestHarness<BinaryRowData, BinaryRowData, JoinedRowData> buildSortMergeJoin(StreamOperator operator) throws Exception {
    final TwoInputStreamTaskTestHarness<BinaryRowData, BinaryRowData, JoinedRowData> testHarness = new TwoInputStreamTaskTestHarness<>(TwoInputStreamTask::new, 2, 2, new int[] { 1, 2 }, typeInfo, (TypeInformation) typeInfo, joinedInfo);
    testHarness.memorySize = 36 * 1024 * 1024;
    testHarness.setupOutputForSingletonOperatorChain();
    testHarness.getStreamConfig().setStreamOperator(operator);
    testHarness.getStreamConfig().setOperatorID(new OperatorID());
    testHarness.getStreamConfig().setManagedMemoryFractionOperatorOfUseCase(ManagedMemoryUseCase.OPERATOR, 0.99);
    long initialTime = 0L;
    testHarness.invoke();
    testHarness.waitForTaskRunning();
    testHarness.processElement(new StreamRecord<>(newRow("a", "0"), initialTime), 0, 0);
    testHarness.processElement(new StreamRecord<>(newRow("d", "0"), initialTime), 0, 0);
    testHarness.processElement(new StreamRecord<>(newRow("a", "2"), initialTime), 1, 1);
    testHarness.processElement(new StreamRecord<>(newRow("b", "1"), initialTime), 0, 1);
    testHarness.processElement(new StreamRecord<>(newRow("c", "2"), initialTime), 1, 1);
    testHarness.processElement(new StreamRecord<>(newRow("b", "4"), initialTime), 1, 0);
    testHarness.waitForInputProcessing();
    testHarness.endInput();
    return testHarness;
}

Example 53 with OperatorID

use of org.apache.flink.runtime.jobgraph.OperatorID in project flink by apache.

the class InputTest method setup.

@Before
public void setup() {
    element = new StreamRecord<>(GenericRowData.of(StringData.fromString("123")), 456);
    watermark = new Watermark(1223456789);
    latencyMarker = new LatencyMarker(122345678, new OperatorID(123, 456), 1);
}

Example 54 with OperatorID

use of org.apache.flink.runtime.jobgraph.OperatorID in project flink by apache.

the class DefaultOperatorCoordinatorHandlerTest method testRegisterAndStartNewCoordinators.

@Test
public void testRegisterAndStartNewCoordinators() throws Exception {
    final JobVertex[] jobVertices = createJobVertices(BLOCKING);
    OperatorID operatorId1 = OperatorID.fromJobVertexID(jobVertices[0].getID());
    OperatorID operatorId2 = OperatorID.fromJobVertexID(jobVertices[1].getID());
    ExecutionGraph executionGraph = createDynamicGraph(jobVertices);
    ExecutionJobVertex ejv1 = executionGraph.getJobVertex(jobVertices[0].getID());
    ExecutionJobVertex ejv2 = executionGraph.getJobVertex(jobVertices[1].getID());
    executionGraph.start(ComponentMainThreadExecutorServiceAdapter.forMainThread());
    executionGraph.initializeJobVertex(ejv1, 0L);
    DefaultOperatorCoordinatorHandler handler = new DefaultOperatorCoordinatorHandler(executionGraph, throwable -> {
    });
    assertThat(handler.getCoordinatorMap().keySet(), containsInAnyOrder(operatorId1));
    executionGraph.initializeJobVertex(ejv2, 0L);
    handler.registerAndStartNewCoordinators(ejv2.getOperatorCoordinators(), executionGraph.getJobMasterMainThreadExecutor());
    assertThat(handler.getCoordinatorMap().keySet(), containsInAnyOrder(operatorId1, operatorId2));
}

Example 55 with OperatorID

use of org.apache.flink.runtime.jobgraph.OperatorID in project flink by apache.

the class DefaultExecutionGraphFactoryTest method createJobGraphWithSavepoint.

@Nonnull
private JobGraph createJobGraphWithSavepoint(boolean allowNonRestoredState, long savepointId) throws IOException {
    // create savepoint data
    final OperatorID operatorID = new OperatorID();
    final File savepointFile = TestUtils.createSavepointWithOperatorState(TEMPORARY_FOLDER.newFile(), savepointId, operatorID);
    // set savepoint settings which don't allow non restored state
    final SavepointRestoreSettings savepointRestoreSettings = SavepointRestoreSettings.forPath(savepointFile.getAbsolutePath(), allowNonRestoredState);
    // create a new operator
    final JobVertex jobVertex = new JobVertex("New operator");
    jobVertex.setInvokableClass(NoOpInvokable.class);
    jobVertex.setParallelism(1);
    // a given OperatorID that does not match any operator of the newly created JobGraph
    return TestUtils.createJobGraphFromJobVerticesWithCheckpointing(savepointRestoreSettings, jobVertex);
}