Examples with JobVertexID org.apache.flink.runtime.jobgraph.JobVertexID used on opensource projects

Example 71 with JobVertexID

use of org.apache.flink.runtime.jobgraph.JobVertexID in project flink by apache.

the class AkkaKvStateLocationLookupServiceTest method testLeaderSessionIdChange.

/**
	 * Tests that messages are properly decorated with the leader session ID.
	 */
@Test
public void testLeaderSessionIdChange() throws Exception {
    TestingLeaderRetrievalService leaderRetrievalService = new TestingLeaderRetrievalService();
    Queue<LookupKvStateLocation> received = new LinkedBlockingQueue<>();
    AkkaKvStateLocationLookupService lookupService = new AkkaKvStateLocationLookupService(leaderRetrievalService, testActorSystem, TIMEOUT, new AkkaKvStateLocationLookupService.DisabledLookupRetryStrategyFactory());
    lookupService.start();
    // Create test actors with random leader session IDs
    KvStateLocation expected1 = new KvStateLocation(new JobID(), new JobVertexID(), 8282, "salt");
    UUID leaderSessionId1 = UUID.randomUUID();
    ActorRef testActor1 = LookupResponseActor.create(received, leaderSessionId1, expected1);
    String testActorAddress1 = AkkaUtils.getAkkaURL(testActorSystem, testActor1);
    KvStateLocation expected2 = new KvStateLocation(new JobID(), new JobVertexID(), 22321, "pepper");
    UUID leaderSessionId2 = UUID.randomUUID();
    ActorRef testActor2 = LookupResponseActor.create(received, leaderSessionId1, expected2);
    String testActorAddress2 = AkkaUtils.getAkkaURL(testActorSystem, testActor2);
    JobID jobId = new JobID();
    //
    // Notify about first leader
    //
    leaderRetrievalService.notifyListener(testActorAddress1, leaderSessionId1);
    KvStateLocation location = Await.result(lookupService.getKvStateLookupInfo(jobId, "rock"), TIMEOUT);
    assertEquals(expected1, location);
    assertEquals(1, received.size());
    verifyLookupMsg(received.poll(), jobId, "rock");
    //
    // Notify about second leader
    //
    leaderRetrievalService.notifyListener(testActorAddress2, leaderSessionId2);
    location = Await.result(lookupService.getKvStateLookupInfo(jobId, "roll"), TIMEOUT);
    assertEquals(expected2, location);
    assertEquals(1, received.size());
    verifyLookupMsg(received.poll(), jobId, "roll");
}

Example 72 with JobVertexID

use of org.apache.flink.runtime.jobgraph.JobVertexID in project flink by apache.

the class KvStateLocationRegistryTest method testRegisterDuplicateName.

/**
	 * Tests that registrations with duplicate names throw an Exception.
	 */
@Test
public void testRegisterDuplicateName() throws Exception {
    ExecutionJobVertex[] vertices = new ExecutionJobVertex[] { createJobVertex(32), createJobVertex(13) };
    Map<JobVertexID, ExecutionJobVertex> vertexMap = createVertexMap(vertices);
    String registrationName = "duplicated-name";
    KvStateLocationRegistry registry = new KvStateLocationRegistry(new JobID(), vertexMap);
    // First operator registers
    registry.notifyKvStateRegistered(vertices[0].getJobVertexId(), new KeyGroupRange(0, 0), registrationName, new KvStateID(), new KvStateServerAddress(InetAddress.getLocalHost(), 12328));
    try {
        // Second operator registers same name
        registry.notifyKvStateRegistered(vertices[1].getJobVertexId(), new KeyGroupRange(0, 0), registrationName, new KvStateID(), new KvStateServerAddress(InetAddress.getLocalHost(), 12032));
        fail("Did not throw expected Exception after duplicated name");
    } catch (IllegalStateException ignored) {
    // Expected
    }
}

Example 73 with JobVertexID

use of org.apache.flink.runtime.jobgraph.JobVertexID in project flink by apache.

the class QueryableStateClientTest method testIntegrationWithKvStateServer.

/**
	 * Tests queries against multiple servers.
	 *
	 * <p>The servers are populated with different keys and the client queries
	 * all available keys from all servers.
	 */
@Test
public void testIntegrationWithKvStateServer() throws Exception {
    // Config
    int numServers = 2;
    int numKeys = 1024;
    int numKeyGroups = 1;
    JobID jobId = new JobID();
    JobVertexID jobVertexId = new JobVertexID();
    KvStateServer[] servers = new KvStateServer[numServers];
    AtomicKvStateRequestStats[] serverStats = new AtomicKvStateRequestStats[numServers];
    QueryableStateClient client = null;
    KvStateClient networkClient = null;
    AtomicKvStateRequestStats networkClientStats = new AtomicKvStateRequestStats();
    MemoryStateBackend backend = new MemoryStateBackend();
    DummyEnvironment dummyEnv = new DummyEnvironment("test", 1, 0);
    AbstractKeyedStateBackend<Integer> keyedStateBackend = backend.createKeyedStateBackend(dummyEnv, new JobID(), "test_op", IntSerializer.INSTANCE, numKeyGroups, new KeyGroupRange(0, 0), new KvStateRegistry().createTaskRegistry(new JobID(), new JobVertexID()));
    try {
        KvStateRegistry[] registries = new KvStateRegistry[numServers];
        KvStateID[] kvStateIds = new KvStateID[numServers];
        List<HeapValueState<Integer, VoidNamespace, Integer>> kvStates = new ArrayList<>();
        // Start the servers
        for (int i = 0; i < numServers; i++) {
            registries[i] = new KvStateRegistry();
            serverStats[i] = new AtomicKvStateRequestStats();
            servers[i] = new KvStateServer(InetAddress.getLocalHost(), 0, 1, 1, registries[i], serverStats[i]);
            servers[i].start();
            ValueStateDescriptor<Integer> descriptor = new ValueStateDescriptor<>("any", IntSerializer.INSTANCE);
            RegisteredBackendStateMetaInfo<VoidNamespace, Integer> registeredBackendStateMetaInfo = new RegisteredBackendStateMetaInfo<>(descriptor.getType(), descriptor.getName(), VoidNamespaceSerializer.INSTANCE, IntSerializer.INSTANCE);
            // Register state
            HeapValueState<Integer, VoidNamespace, Integer> kvState = new HeapValueState<>(descriptor, new NestedMapsStateTable<Integer, VoidNamespace, Integer>(keyedStateBackend, registeredBackendStateMetaInfo), IntSerializer.INSTANCE, VoidNamespaceSerializer.INSTANCE);
            kvStates.add(kvState);
            kvStateIds[i] = registries[i].registerKvState(jobId, new JobVertexID(), new KeyGroupRange(i, i), "choco", kvState);
        }
        int[] expectedRequests = new int[numServers];
        for (int key = 0; key < numKeys; key++) {
            int targetKeyGroupIndex = MathUtils.murmurHash(key) % numServers;
            expectedRequests[targetKeyGroupIndex]++;
            HeapValueState<Integer, VoidNamespace, Integer> kvState = kvStates.get(targetKeyGroupIndex);
            keyedStateBackend.setCurrentKey(key);
            kvState.setCurrentNamespace(VoidNamespace.INSTANCE);
            kvState.update(1337 + key);
        }
        // Location lookup service
        KvStateLocation location = new KvStateLocation(jobId, jobVertexId, numServers, "choco");
        for (int keyGroupIndex = 0; keyGroupIndex < numServers; keyGroupIndex++) {
            location.registerKvState(new KeyGroupRange(keyGroupIndex, keyGroupIndex), kvStateIds[keyGroupIndex], servers[keyGroupIndex].getAddress());
        }
        KvStateLocationLookupService lookupService = mock(KvStateLocationLookupService.class);
        when(lookupService.getKvStateLookupInfo(eq(jobId), eq("choco"))).thenReturn(Futures.successful(location));
        // The client
        networkClient = new KvStateClient(1, networkClientStats);
        client = new QueryableStateClient(lookupService, networkClient, testActorSystem.dispatcher());
        // Send all queries
        List<Future<byte[]>> futures = new ArrayList<>(numKeys);
        for (int key = 0; key < numKeys; key++) {
            byte[] serializedKeyAndNamespace = KvStateRequestSerializer.serializeKeyAndNamespace(key, IntSerializer.INSTANCE, VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE);
            futures.add(client.getKvState(jobId, "choco", key, serializedKeyAndNamespace));
        }
        // Verify results
        Future<Iterable<byte[]>> future = Futures.sequence(futures, testActorSystem.dispatcher());
        Iterable<byte[]> results = Await.result(future, timeout);
        int index = 0;
        for (byte[] buffer : results) {
            int deserializedValue = KvStateRequestSerializer.deserializeValue(buffer, IntSerializer.INSTANCE);
            assertEquals(1337 + index, deserializedValue);
            index++;
        }
        // Verify requests
        for (int i = 0; i < numServers; i++) {
            int numRetries = 10;
            for (int retry = 0; retry < numRetries; retry++) {
                try {
                    assertEquals("Unexpected number of requests", expectedRequests[i], serverStats[i].getNumRequests());
                    assertEquals("Unexpected success requests", expectedRequests[i], serverStats[i].getNumSuccessful());
                    assertEquals("Unexpected failed requests", 0, serverStats[i].getNumFailed());
                    break;
                } catch (Throwable t) {
                    // Retry
                    if (retry == numRetries - 1) {
                        throw t;
                    } else {
                        Thread.sleep(100);
                    }
                }
            }
        }
    } finally {
        if (client != null) {
            client.shutDown();
        }
        if (networkClient != null) {
            networkClient.shutDown();
        }
        for (KvStateServer server : servers) {
            if (server != null) {
                server.shutDown();
            }
        }
    }
}

Example 74 with JobVertexID

use of org.apache.flink.runtime.jobgraph.JobVertexID in project flink by apache.

the class BackPressureStatsTrackerTest method mockExecutionVertex.

private ExecutionVertex mockExecutionVertex(ExecutionJobVertex jobVertex, int subTaskIndex) {
    Execution exec = mock(Execution.class);
    when(exec.getAttemptId()).thenReturn(new ExecutionAttemptID());
    JobVertexID id = jobVertex.getJobVertexId();
    ExecutionVertex vertex = mock(ExecutionVertex.class);
    when(vertex.getJobvertexId()).thenReturn(id);
    when(vertex.getCurrentExecutionAttempt()).thenReturn(exec);
    when(vertex.getParallelSubtaskIndex()).thenReturn(subTaskIndex);
    return vertex;
}

Example 75 with JobVertexID

use of org.apache.flink.runtime.jobgraph.JobVertexID in project flink by apache.

the class CheckpointStatsDetailsSubtasksHandler method handleRequest.

@Override
public String handleRequest(AccessExecutionGraph graph, Map<String, String> params) throws Exception {
    long checkpointId = CheckpointStatsDetailsHandler.parseCheckpointId(params);
    if (checkpointId == -1) {
        return "{}";
    }
    JobVertexID vertexId = AbstractJobVertexRequestHandler.parseJobVertexId(params);
    if (vertexId == null) {
        return "{}";
    }
    CheckpointStatsSnapshot snapshot = graph.getCheckpointStatsSnapshot();
    if (snapshot == null) {
        return "{}";
    }
    AbstractCheckpointStats checkpoint = snapshot.getHistory().getCheckpointById(checkpointId);
    if (checkpoint != null) {
        cache.tryAdd(checkpoint);
    } else {
        checkpoint = cache.tryGet(checkpointId);
        if (checkpoint == null) {
            return "{}";
        }
    }
    TaskStateStats taskStats = checkpoint.getTaskStateStats(vertexId);
    if (taskStats == null) {
        return "{}";
    }
    return createSubtaskCheckpointDetailsJson(checkpoint, taskStats);
}