Examples with JobVertexID org.apache.flink.runtime.jobgraph.JobVertexID used on opensource projects

Example 66 with JobVertexID

use of org.apache.flink.runtime.jobgraph.JobVertexID in project flink by apache.

the class CompletedCheckpointStatsSummaryTest method createCompletedCheckpoint.

private CompletedCheckpointStats createCompletedCheckpoint(long checkpointId, long triggerTimestamp, long ackTimestamp, long stateSize, long alignmentBuffered) {
    SubtaskStateStats latest = mock(SubtaskStateStats.class);
    when(latest.getAckTimestamp()).thenReturn(ackTimestamp);
    Map<JobVertexID, TaskStateStats> taskStats = new HashMap<>();
    JobVertexID jobVertexId = new JobVertexID();
    taskStats.put(jobVertexId, new TaskStateStats(jobVertexId, 1));
    return new CompletedCheckpointStats(checkpointId, triggerTimestamp, CheckpointProperties.forStandardCheckpoint(), 1, taskStats, 1, stateSize, alignmentBuffered, latest, null);
}

Example 67 with JobVertexID

use of org.apache.flink.runtime.jobgraph.JobVertexID in project flink by apache.

the class CompletedCheckpointStoreTest method createCheckpoint.

protected TestCompletedCheckpoint createCheckpoint(int id, int numberOfStates, CheckpointProperties props) throws IOException {
    JobVertexID jvid = new JobVertexID();
    Map<JobVertexID, TaskState> taskGroupStates = new HashMap<>();
    TaskState taskState = new TaskState(jvid, numberOfStates, numberOfStates, 1);
    taskGroupStates.put(jvid, taskState);
    for (int i = 0; i < numberOfStates; i++) {
        ChainedStateHandle<StreamStateHandle> stateHandle = CheckpointCoordinatorTest.generateChainedStateHandle(new CheckpointMessagesTest.MyHandle());
        taskState.putState(i, new SubtaskState(stateHandle, null, null, null, null));
    }
    return new TestCompletedCheckpoint(new JobID(), id, 0, taskGroupStates, props);
}

Example 68 with JobVertexID

use of org.apache.flink.runtime.jobgraph.JobVertexID in project flink by apache.

the class CompletedCheckpointTest method testCleanUpOnSubsume.

/**
	 * Tests that the garbage collection properties are respected when subsuming checkpoints.
	 */
@Test
public void testCleanUpOnSubsume() throws Exception {
    TaskState state = mock(TaskState.class);
    Map<JobVertexID, TaskState> taskStates = new HashMap<>();
    taskStates.put(new JobVertexID(), state);
    boolean discardSubsumed = true;
    CheckpointProperties props = new CheckpointProperties(false, false, discardSubsumed, true, true, true, true);
    CompletedCheckpoint checkpoint = new CompletedCheckpoint(new JobID(), 0, 0, 1, taskStates, props);
    // Subsume
    checkpoint.subsume();
    verify(state, times(1)).discardState();
}

Example 69 with JobVertexID

use of org.apache.flink.runtime.jobgraph.JobVertexID in project flink by apache.

the class KvStateServerHandlerTest method testFailureOnGetSerializedValue.

/**
	 * Tests the failure response on a failure on the {@link InternalKvState#getSerializedValue(byte[])}
	 * call.
	 */
@Test
public void testFailureOnGetSerializedValue() throws Exception {
    KvStateRegistry registry = new KvStateRegistry();
    AtomicKvStateRequestStats stats = new AtomicKvStateRequestStats();
    KvStateServerHandler handler = new KvStateServerHandler(registry, TEST_THREAD_POOL, stats);
    EmbeddedChannel channel = new EmbeddedChannel(getFrameDecoder(), handler);
    // Failing KvState
    InternalKvState<?> kvState = mock(InternalKvState.class);
    when(kvState.getSerializedValue(any(byte[].class))).thenThrow(new RuntimeException("Expected test Exception"));
    KvStateID kvStateId = registry.registerKvState(new JobID(), new JobVertexID(), new KeyGroupRange(0, 0), "vanilla", kvState);
    ByteBuf request = KvStateRequestSerializer.serializeKvStateRequest(channel.alloc(), 282872, kvStateId, new byte[0]);
    // Write the request and wait for the response
    channel.writeInbound(request);
    ByteBuf buf = (ByteBuf) readInboundBlocking(channel);
    // skip frame length
    buf.skipBytes(4);
    // Verify the response
    assertEquals(KvStateRequestType.REQUEST_FAILURE, KvStateRequestSerializer.deserializeHeader(buf));
    KvStateRequestFailure response = KvStateRequestSerializer.deserializeKvStateRequestFailure(buf);
    assertTrue(response.getCause().getMessage().contains("Expected test Exception"));
    assertEquals(1, stats.getNumRequests());
    assertEquals(1, stats.getNumFailed());
}

Example 70 with JobVertexID

use of org.apache.flink.runtime.jobgraph.JobVertexID in project flink by apache.

the class KvStateServerTest method testSimpleRequest.

/**
	 * Tests a simple successful query via a SocketChannel.
	 */
@Test
public void testSimpleRequest() throws Exception {
    KvStateServer server = null;
    Bootstrap bootstrap = null;
    try {
        KvStateRegistry registry = new KvStateRegistry();
        KvStateRequestStats stats = new AtomicKvStateRequestStats();
        server = new KvStateServer(InetAddress.getLocalHost(), 0, 1, 1, registry, stats);
        server.start();
        KvStateServerAddress serverAddress = server.getAddress();
        int numKeyGroups = 1;
        AbstractStateBackend abstractBackend = new MemoryStateBackend();
        DummyEnvironment dummyEnv = new DummyEnvironment("test", 1, 0);
        dummyEnv.setKvStateRegistry(registry);
        AbstractKeyedStateBackend<Integer> backend = abstractBackend.createKeyedStateBackend(dummyEnv, new JobID(), "test_op", IntSerializer.INSTANCE, numKeyGroups, new KeyGroupRange(0, 0), registry.createTaskRegistry(new JobID(), new JobVertexID()));
        final KvStateServerHandlerTest.TestRegistryListener registryListener = new KvStateServerHandlerTest.TestRegistryListener();
        registry.registerListener(registryListener);
        ValueStateDescriptor<Integer> desc = new ValueStateDescriptor<>("any", IntSerializer.INSTANCE);
        desc.setQueryable("vanilla");
        ValueState<Integer> state = backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, desc);
        // Update KvState
        int expectedValue = 712828289;
        int key = 99812822;
        backend.setCurrentKey(key);
        state.update(expectedValue);
        // Request
        byte[] serializedKeyAndNamespace = KvStateRequestSerializer.serializeKeyAndNamespace(key, IntSerializer.INSTANCE, VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE);
        // Connect to the server
        final BlockingQueue<ByteBuf> responses = new LinkedBlockingQueue<>();
        bootstrap = createBootstrap(new LengthFieldBasedFrameDecoder(Integer.MAX_VALUE, 0, 4, 0, 4), new ChannelInboundHandlerAdapter() {

            @Override
            public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception {
                responses.add((ByteBuf) msg);
            }
        });
        Channel channel = bootstrap.connect(serverAddress.getHost(), serverAddress.getPort()).sync().channel();
        long requestId = Integer.MAX_VALUE + 182828L;
        assertTrue(registryListener.registrationName.equals("vanilla"));
        ByteBuf request = KvStateRequestSerializer.serializeKvStateRequest(channel.alloc(), requestId, registryListener.kvStateId, serializedKeyAndNamespace);
        channel.writeAndFlush(request);
        ByteBuf buf = responses.poll(TIMEOUT_MILLIS, TimeUnit.MILLISECONDS);
        assertEquals(KvStateRequestType.REQUEST_RESULT, KvStateRequestSerializer.deserializeHeader(buf));
        KvStateRequestResult response = KvStateRequestSerializer.deserializeKvStateRequestResult(buf);
        assertEquals(requestId, response.getRequestId());
        int actualValue = KvStateRequestSerializer.deserializeValue(response.getSerializedResult(), IntSerializer.INSTANCE);
        assertEquals(expectedValue, actualValue);
    } finally {
        if (server != null) {
            server.shutDown();
        }
        if (bootstrap != null) {
            EventLoopGroup group = bootstrap.group();
            if (group != null) {
                group.shutdownGracefully();
            }
        }
    }
}