Examples with LookupKvStateLocation org.apache.flink.runtime.query.KvStateMessage.LookupKvStateLocation used on opensource projects

Example 1 with LookupKvStateLocation

use of org.apache.flink.runtime.query.KvStateMessage.LookupKvStateLocation in project flink by apache.

the class JobManagerTest method testKvStateMessages.

/**
	 * Tests that the JobManager handles {@link org.apache.flink.runtime.query.KvStateMessage}
	 * instances as expected.
	 */
@Test
public void testKvStateMessages() throws Exception {
    Deadline deadline = new FiniteDuration(100, TimeUnit.SECONDS).fromNow();
    Configuration config = new Configuration();
    config.setString(ConfigConstants.AKKA_ASK_TIMEOUT, "100ms");
    UUID leaderSessionId = null;
    ActorGateway jobManager = new AkkaActorGateway(JobManager.startJobManagerActors(config, system, TestingUtils.defaultExecutor(), TestingUtils.defaultExecutor(), TestingJobManager.class, MemoryArchivist.class)._1(), leaderSessionId);
    LeaderRetrievalService leaderRetrievalService = new StandaloneLeaderRetrievalService(AkkaUtils.getAkkaURL(system, jobManager.actor()));
    Configuration tmConfig = new Configuration();
    tmConfig.setInteger(ConfigConstants.TASK_MANAGER_MEMORY_SIZE_KEY, 4);
    tmConfig.setInteger(ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS, 8);
    ActorRef taskManager = TaskManager.startTaskManagerComponentsAndActor(tmConfig, ResourceID.generate(), system, "localhost", scala.Option.<String>empty(), scala.Option.apply(leaderRetrievalService), true, TestingTaskManager.class);
    Future<Object> registrationFuture = jobManager.ask(new NotifyWhenAtLeastNumTaskManagerAreRegistered(1), deadline.timeLeft());
    Await.ready(registrationFuture, deadline.timeLeft());
    //
    // Location lookup
    //
    LookupKvStateLocation lookupNonExistingJob = new LookupKvStateLocation(new JobID(), "any-name");
    Future<KvStateLocation> lookupFuture = jobManager.ask(lookupNonExistingJob, deadline.timeLeft()).mapTo(ClassTag$.MODULE$.<KvStateLocation>apply(KvStateLocation.class));
    try {
        Await.result(lookupFuture, deadline.timeLeft());
        fail("Did not throw expected Exception");
    } catch (IllegalStateException ignored) {
    // Expected
    }
    JobGraph jobGraph = new JobGraph("croissant");
    JobVertex jobVertex1 = new JobVertex("cappuccino");
    jobVertex1.setParallelism(4);
    jobVertex1.setMaxParallelism(16);
    jobVertex1.setInvokableClass(BlockingNoOpInvokable.class);
    JobVertex jobVertex2 = new JobVertex("americano");
    jobVertex2.setParallelism(4);
    jobVertex2.setMaxParallelism(16);
    jobVertex2.setInvokableClass(BlockingNoOpInvokable.class);
    jobGraph.addVertex(jobVertex1);
    jobGraph.addVertex(jobVertex2);
    Future<JobSubmitSuccess> submitFuture = jobManager.ask(new SubmitJob(jobGraph, ListeningBehaviour.DETACHED), deadline.timeLeft()).mapTo(ClassTag$.MODULE$.<JobSubmitSuccess>apply(JobSubmitSuccess.class));
    Await.result(submitFuture, deadline.timeLeft());
    Object lookupUnknownRegistrationName = new LookupKvStateLocation(jobGraph.getJobID(), "unknown");
    lookupFuture = jobManager.ask(lookupUnknownRegistrationName, deadline.timeLeft()).mapTo(ClassTag$.MODULE$.<KvStateLocation>apply(KvStateLocation.class));
    try {
        Await.result(lookupFuture, deadline.timeLeft());
        fail("Did not throw expected Exception");
    } catch (UnknownKvStateLocation ignored) {
    // Expected
    }
    //
    // Registration
    //
    NotifyKvStateRegistered registerNonExistingJob = new NotifyKvStateRegistered(new JobID(), new JobVertexID(), new KeyGroupRange(0, 0), "any-name", new KvStateID(), new KvStateServerAddress(InetAddress.getLocalHost(), 1233));
    jobManager.tell(registerNonExistingJob);
    LookupKvStateLocation lookupAfterRegistration = new LookupKvStateLocation(registerNonExistingJob.getJobId(), registerNonExistingJob.getRegistrationName());
    lookupFuture = jobManager.ask(lookupAfterRegistration, deadline.timeLeft()).mapTo(ClassTag$.MODULE$.<KvStateLocation>apply(KvStateLocation.class));
    try {
        Await.result(lookupFuture, deadline.timeLeft());
        fail("Did not throw expected Exception");
    } catch (IllegalStateException ignored) {
    // Expected
    }
    NotifyKvStateRegistered registerForExistingJob = new NotifyKvStateRegistered(jobGraph.getJobID(), jobVertex1.getID(), new KeyGroupRange(0, 0), "register-me", new KvStateID(), new KvStateServerAddress(InetAddress.getLocalHost(), 1293));
    jobManager.tell(registerForExistingJob);
    lookupAfterRegistration = new LookupKvStateLocation(registerForExistingJob.getJobId(), registerForExistingJob.getRegistrationName());
    lookupFuture = jobManager.ask(lookupAfterRegistration, deadline.timeLeft()).mapTo(ClassTag$.MODULE$.<KvStateLocation>apply(KvStateLocation.class));
    KvStateLocation location = Await.result(lookupFuture, deadline.timeLeft());
    assertNotNull(location);
    assertEquals(jobGraph.getJobID(), location.getJobId());
    assertEquals(jobVertex1.getID(), location.getJobVertexId());
    assertEquals(jobVertex1.getMaxParallelism(), location.getNumKeyGroups());
    assertEquals(1, location.getNumRegisteredKeyGroups());
    KeyGroupRange keyGroupRange = registerForExistingJob.getKeyGroupRange();
    assertEquals(1, keyGroupRange.getNumberOfKeyGroups());
    assertEquals(registerForExistingJob.getKvStateId(), location.getKvStateID(keyGroupRange.getStartKeyGroup()));
    assertEquals(registerForExistingJob.getKvStateServerAddress(), location.getKvStateServerAddress(keyGroupRange.getStartKeyGroup()));
    //
    // Unregistration
    //
    NotifyKvStateUnregistered unregister = new NotifyKvStateUnregistered(registerForExistingJob.getJobId(), registerForExistingJob.getJobVertexId(), registerForExistingJob.getKeyGroupRange(), registerForExistingJob.getRegistrationName());
    jobManager.tell(unregister);
    lookupFuture = jobManager.ask(lookupAfterRegistration, deadline.timeLeft()).mapTo(ClassTag$.MODULE$.<KvStateLocation>apply(KvStateLocation.class));
    try {
        Await.result(lookupFuture, deadline.timeLeft());
        fail("Did not throw expected Exception");
    } catch (UnknownKvStateLocation ignored) {
    // Expected
    }
    //
    // Duplicate registration fails task
    //
    NotifyKvStateRegistered register = new NotifyKvStateRegistered(jobGraph.getJobID(), jobVertex1.getID(), new KeyGroupRange(0, 0), "duplicate-me", new KvStateID(), new KvStateServerAddress(InetAddress.getLocalHost(), 1293));
    NotifyKvStateRegistered duplicate = new NotifyKvStateRegistered(jobGraph.getJobID(), // <--- different operator, but...
    jobVertex2.getID(), new KeyGroupRange(0, 0), // ...same name
    "duplicate-me", new KvStateID(), new KvStateServerAddress(InetAddress.getLocalHost(), 1293));
    Future<TestingJobManagerMessages.JobStatusIs> failedFuture = jobManager.ask(new NotifyWhenJobStatus(jobGraph.getJobID(), JobStatus.FAILED), deadline.timeLeft()).mapTo(ClassTag$.MODULE$.<JobStatusIs>apply(JobStatusIs.class));
    jobManager.tell(register);
    jobManager.tell(duplicate);
    // Wait for failure
    JobStatusIs jobStatus = Await.result(failedFuture, deadline.timeLeft());
    assertEquals(JobStatus.FAILED, jobStatus.state());
}

Example 2 with LookupKvStateLocation

use of org.apache.flink.runtime.query.KvStateMessage.LookupKvStateLocation in project flink by apache.

the class AkkaKvStateLocationLookupServiceTest method testLeaderSessionIdChange.

/**
	 * Tests that messages are properly decorated with the leader session ID.
	 */
@Test
public void testLeaderSessionIdChange() throws Exception {
    TestingLeaderRetrievalService leaderRetrievalService = new TestingLeaderRetrievalService();
    Queue<LookupKvStateLocation> received = new LinkedBlockingQueue<>();
    AkkaKvStateLocationLookupService lookupService = new AkkaKvStateLocationLookupService(leaderRetrievalService, testActorSystem, TIMEOUT, new AkkaKvStateLocationLookupService.DisabledLookupRetryStrategyFactory());
    lookupService.start();
    // Create test actors with random leader session IDs
    KvStateLocation expected1 = new KvStateLocation(new JobID(), new JobVertexID(), 8282, "salt");
    UUID leaderSessionId1 = UUID.randomUUID();
    ActorRef testActor1 = LookupResponseActor.create(received, leaderSessionId1, expected1);
    String testActorAddress1 = AkkaUtils.getAkkaURL(testActorSystem, testActor1);
    KvStateLocation expected2 = new KvStateLocation(new JobID(), new JobVertexID(), 22321, "pepper");
    UUID leaderSessionId2 = UUID.randomUUID();
    ActorRef testActor2 = LookupResponseActor.create(received, leaderSessionId1, expected2);
    String testActorAddress2 = AkkaUtils.getAkkaURL(testActorSystem, testActor2);
    JobID jobId = new JobID();
    //
    // Notify about first leader
    //
    leaderRetrievalService.notifyListener(testActorAddress1, leaderSessionId1);
    KvStateLocation location = Await.result(lookupService.getKvStateLookupInfo(jobId, "rock"), TIMEOUT);
    assertEquals(expected1, location);
    assertEquals(1, received.size());
    verifyLookupMsg(received.poll(), jobId, "rock");
    //
    // Notify about second leader
    //
    leaderRetrievalService.notifyListener(testActorAddress2, leaderSessionId2);
    location = Await.result(lookupService.getKvStateLookupInfo(jobId, "roll"), TIMEOUT);
    assertEquals(expected2, location);
    assertEquals(1, received.size());
    verifyLookupMsg(received.poll(), jobId, "roll");
}

Example 3 with LookupKvStateLocation

use of org.apache.flink.runtime.query.KvStateMessage.LookupKvStateLocation in project flink by apache.

the class AkkaKvStateLocationLookupServiceTest method testUnexpectedResponseType.

@Test
public void testUnexpectedResponseType() throws Exception {
    TestingLeaderRetrievalService leaderRetrievalService = new TestingLeaderRetrievalService();
    Queue<LookupKvStateLocation> received = new LinkedBlockingQueue<>();
    AkkaKvStateLocationLookupService lookupService = new AkkaKvStateLocationLookupService(leaderRetrievalService, testActorSystem, TIMEOUT, new AkkaKvStateLocationLookupService.DisabledLookupRetryStrategyFactory());
    lookupService.start();
    // Create test actors with random leader session IDs
    String expected = "unexpected-response-type";
    ActorRef testActor = LookupResponseActor.create(received, null, expected);
    String testActorAddress = AkkaUtils.getAkkaURL(testActorSystem, testActor);
    leaderRetrievalService.notifyListener(testActorAddress, null);
    try {
        Await.result(lookupService.getKvStateLookupInfo(new JobID(), "spicy"), TIMEOUT);
        fail("Did not throw expected Exception");
    } catch (Throwable ignored) {
    // Expected
    }
}

Example 4 with LookupKvStateLocation

use of org.apache.flink.runtime.query.KvStateMessage.LookupKvStateLocation in project flink by apache.

the class AkkaKvStateLocationLookupServiceTest method testNoJobManagerRegistered.

/**
	 * Tests responses if no leader notification has been reported or leadership
	 * has been lost (leaderAddress = <code>null</code>).
	 */
@Test
public void testNoJobManagerRegistered() throws Exception {
    TestingLeaderRetrievalService leaderRetrievalService = new TestingLeaderRetrievalService();
    Queue<LookupKvStateLocation> received = new LinkedBlockingQueue<>();
    AkkaKvStateLocationLookupService lookupService = new AkkaKvStateLocationLookupService(leaderRetrievalService, testActorSystem, TIMEOUT, new AkkaKvStateLocationLookupService.DisabledLookupRetryStrategyFactory());
    lookupService.start();
    //
    try {
        JobID jobId = new JobID();
        String name = "coffee";
        Future<KvStateLocation> locationFuture = lookupService.getKvStateLookupInfo(jobId, name);
        Await.result(locationFuture, TIMEOUT);
        fail("Did not throw expected Exception");
    } catch (UnknownJobManager ignored) {
    // Expected
    }
    assertEquals("Received unexpected lookup", 0, received.size());
    //
    // Leader registration => communicate with new leader
    //
    UUID leaderSessionId = null;
    KvStateLocation expected = new KvStateLocation(new JobID(), new JobVertexID(), 8282, "tea");
    ActorRef testActor = LookupResponseActor.create(received, leaderSessionId, expected);
    String testActorAddress = AkkaUtils.getAkkaURL(testActorSystem, testActor);
    // Notify the service about a leader
    leaderRetrievalService.notifyListener(testActorAddress, leaderSessionId);
    JobID jobId = new JobID();
    String name = "tea";
    // Verify that the leader response is handled
    KvStateLocation location = Await.result(lookupService.getKvStateLookupInfo(jobId, name), TIMEOUT);
    assertEquals(expected, location);
    // Verify that the correct message was sent to the leader
    assertEquals(1, received.size());
    verifyLookupMsg(received.poll(), jobId, name);
    //
    // Leader loss => fail with UnknownJobManager
    //
    leaderRetrievalService.notifyListener(null, null);
    try {
        Future<KvStateLocation> locationFuture = lookupService.getKvStateLookupInfo(new JobID(), "coffee");
        Await.result(locationFuture, TIMEOUT);
        fail("Did not throw expected Exception");
    } catch (UnknownJobManager ignored) {
    // Expected
    }
    // No new messages received
    assertEquals(0, received.size());
}

Example 5 with LookupKvStateLocation

use of org.apache.flink.runtime.query.KvStateMessage.LookupKvStateLocation in project flink by apache.

the class AkkaKvStateLocationLookupServiceTest method testRetryOnUnknownJobManager.

/**
	 * Tests that lookups are retried when no leader notification is available.
	 */
@Test
public void testRetryOnUnknownJobManager() throws Exception {
    final Queue<LookupRetryStrategy> retryStrategies = new LinkedBlockingQueue<>();
    LookupRetryStrategyFactory retryStrategy = new LookupRetryStrategyFactory() {

        @Override
        public LookupRetryStrategy createRetryStrategy() {
            return retryStrategies.poll();
        }
    };
    final TestingLeaderRetrievalService leaderRetrievalService = new TestingLeaderRetrievalService();
    AkkaKvStateLocationLookupService lookupService = new AkkaKvStateLocationLookupService(leaderRetrievalService, testActorSystem, TIMEOUT, retryStrategy);
    lookupService.start();
    //
    // Test call to retry
    //
    final AtomicBoolean hasRetried = new AtomicBoolean();
    retryStrategies.add(new LookupRetryStrategy() {

        @Override
        public FiniteDuration getRetryDelay() {
            return FiniteDuration.Zero();
        }

        @Override
        public boolean tryRetry() {
            if (hasRetried.compareAndSet(false, true)) {
                return true;
            }
            return false;
        }
    });
    Future<KvStateLocation> locationFuture = lookupService.getKvStateLookupInfo(new JobID(), "yessir");
    Await.ready(locationFuture, TIMEOUT);
    assertTrue("Did not retry ", hasRetried.get());
    //
    // Test leader notification after retry
    //
    Queue<LookupKvStateLocation> received = new LinkedBlockingQueue<>();
    KvStateLocation expected = new KvStateLocation(new JobID(), new JobVertexID(), 12122, "garlic");
    ActorRef testActor = LookupResponseActor.create(received, null, expected);
    final String testActorAddress = AkkaUtils.getAkkaURL(testActorSystem, testActor);
    retryStrategies.add(new LookupRetryStrategy() {

        @Override
        public FiniteDuration getRetryDelay() {
            return FiniteDuration.apply(100, TimeUnit.MILLISECONDS);
        }

        @Override
        public boolean tryRetry() {
            leaderRetrievalService.notifyListener(testActorAddress, null);
            return true;
        }
    });
    KvStateLocation location = Await.result(lookupService.getKvStateLookupInfo(new JobID(), "yessir"), TIMEOUT);
    assertEquals(expected, location);
}