Examples with Deadline scala.concurrent.duration.Deadline used on opensource projects

Example 21 with Deadline

use of scala.concurrent.duration.Deadline in project flink by apache.

the class KvStateClientTest method testRequestUnavailableHost.

/**
	 * Tests that a request to an unavailable host is failed with ConnectException.
	 */
@Test
public void testRequestUnavailableHost() throws Exception {
    Deadline deadline = TEST_TIMEOUT.fromNow();
    AtomicKvStateRequestStats stats = new AtomicKvStateRequestStats();
    KvStateClient client = null;
    try {
        client = new KvStateClient(1, stats);
        int availablePort = NetUtils.getAvailablePort();
        KvStateServerAddress serverAddress = new KvStateServerAddress(InetAddress.getLocalHost(), availablePort);
        Future<byte[]> future = client.getKvState(serverAddress, new KvStateID(), new byte[0]);
        try {
            Await.result(future, deadline.timeLeft());
            fail("Did not throw expected ConnectException");
        } catch (ConnectException ignored) {
        // Expected
        }
    } finally {
        if (client != null) {
            client.shutDown();
        }
        assertEquals("Channel leak", 0, stats.getNumConnections());
    }
}

Example 22 with Deadline

use of scala.concurrent.duration.Deadline in project flink by apache.

the class JMXJobManagerMetricTest method testJobManagerJMXMetricAccess.

/**
	 * Tests that metrics registered on the JobManager are actually accessible via JMX.
	 *
	 * @throws Exception
	 */
@Test
public void testJobManagerJMXMetricAccess() throws Exception {
    Deadline deadline = new FiniteDuration(2, TimeUnit.MINUTES).fromNow();
    Configuration flinkConfiguration = new Configuration();
    flinkConfiguration.setString(ConfigConstants.METRICS_REPORTERS_LIST, "test");
    flinkConfiguration.setString(ConfigConstants.METRICS_REPORTER_PREFIX + "test." + ConfigConstants.METRICS_REPORTER_CLASS_SUFFIX, JMXReporter.class.getName());
    flinkConfiguration.setString(ConfigConstants.METRICS_REPORTER_PREFIX + "test.port", "9060-9075");
    flinkConfiguration.setString(ConfigConstants.METRICS_SCOPE_NAMING_JM_JOB, "jobmanager.<job_name>");
    TestingCluster flink = new TestingCluster(flinkConfiguration);
    try {
        flink.start();
        JobVertex sourceJobVertex = new JobVertex("Source");
        sourceJobVertex.setInvokableClass(BlockingInvokable.class);
        JobGraph jobGraph = new JobGraph("TestingJob", sourceJobVertex);
        jobGraph.setSnapshotSettings(new JobSnapshottingSettings(Collections.<JobVertexID>emptyList(), Collections.<JobVertexID>emptyList(), Collections.<JobVertexID>emptyList(), 500, 500, 50, 5, ExternalizedCheckpointSettings.none(), null, true));
        flink.waitForActorsToBeAlive();
        flink.submitJobDetached(jobGraph);
        Future<Object> jobRunning = flink.getLeaderGateway(deadline.timeLeft()).ask(new TestingJobManagerMessages.WaitForAllVerticesToBeRunning(jobGraph.getJobID()), deadline.timeLeft());
        Await.ready(jobRunning, deadline.timeLeft());
        MBeanServer mBeanServer = ManagementFactory.getPlatformMBeanServer();
        Set<ObjectName> nameSet = mBeanServer.queryNames(new ObjectName("org.apache.flink.jobmanager.job.lastCheckpointSize:job_name=TestingJob,*"), null);
        Assert.assertEquals(1, nameSet.size());
        assertEquals(-1L, mBeanServer.getAttribute(nameSet.iterator().next(), "Value"));
        Future<Object> jobFinished = flink.getLeaderGateway(deadline.timeLeft()).ask(new TestingJobManagerMessages.NotifyWhenJobRemoved(jobGraph.getJobID()), deadline.timeLeft());
        BlockingInvokable.unblock();
        // wait til the job has finished
        Await.ready(jobFinished, deadline.timeLeft());
    } finally {
        flink.stop();
    }
}

Example 23 with Deadline

use of scala.concurrent.duration.Deadline in project flink by apache.

the class WebRuntimeMonitorITCase method testRedirectToLeader.

/**
	 * Tests that the monitor associated with the following job manager redirects to the leader.
	 */
@Test
public void testRedirectToLeader() throws Exception {
    final Deadline deadline = TestTimeout.fromNow();
    ActorSystem[] jobManagerSystem = new ActorSystem[2];
    WebRuntimeMonitor[] webMonitor = new WebRuntimeMonitor[2];
    List<LeaderRetrievalService> leaderRetrievalServices = new ArrayList<>();
    try (TestingServer zooKeeper = new TestingServer()) {
        final Configuration config = ZooKeeperTestUtils.createZooKeeperHAConfig(zooKeeper.getConnectString(), temporaryFolder.getRoot().getPath());
        File logDir = temporaryFolder.newFolder();
        Path logFile = Files.createFile(new File(logDir, "jobmanager.log").toPath());
        Files.createFile(new File(logDir, "jobmanager.out").toPath());
        config.setInteger(ConfigConstants.JOB_MANAGER_WEB_PORT_KEY, 0);
        config.setString(ConfigConstants.JOB_MANAGER_WEB_LOG_PATH_KEY, logFile.toString());
        for (int i = 0; i < jobManagerSystem.length; i++) {
            jobManagerSystem[i] = AkkaUtils.createActorSystem(new Configuration(), new Some<>(new Tuple2<String, Object>("localhost", 0)));
        }
        for (int i = 0; i < webMonitor.length; i++) {
            LeaderRetrievalService lrs = ZooKeeperUtils.createLeaderRetrievalService(config);
            leaderRetrievalServices.add(lrs);
            webMonitor[i] = new WebRuntimeMonitor(config, lrs, jobManagerSystem[i]);
        }
        ActorRef[] jobManager = new ActorRef[2];
        String[] jobManagerAddress = new String[2];
        for (int i = 0; i < jobManager.length; i++) {
            Configuration jmConfig = config.clone();
            jmConfig.setInteger(ConfigConstants.JOB_MANAGER_WEB_PORT_KEY, webMonitor[i].getServerPort());
            jobManager[i] = JobManager.startJobManagerActors(jmConfig, jobManagerSystem[i], TestingUtils.defaultExecutor(), TestingUtils.defaultExecutor(), JobManager.class, MemoryArchivist.class)._1();
            jobManagerAddress[i] = AkkaUtils.getAkkaURL(jobManagerSystem[i], jobManager[i]);
            webMonitor[i].start(jobManagerAddress[i]);
        }
        LeaderRetrievalService lrs = ZooKeeperUtils.createLeaderRetrievalService(config);
        leaderRetrievalServices.add(lrs);
        TestingListener leaderListener = new TestingListener();
        lrs.start(leaderListener);
        leaderListener.waitForNewLeader(deadline.timeLeft().toMillis());
        String leaderAddress = leaderListener.getAddress();
        int leaderIndex = leaderAddress.equals(jobManagerAddress[0]) ? 0 : 1;
        int followerIndex = (leaderIndex + 1) % 2;
        ActorSystem leadingSystem = jobManagerSystem[leaderIndex];
        ActorSystem followerSystem = jobManagerSystem[followerIndex];
        WebMonitor leadingWebMonitor = webMonitor[leaderIndex];
        WebMonitor followerWebMonitor = webMonitor[followerIndex];
        // For test stability reason we have to wait until we are sure that both leader
        // listeners have been notified.
        JobManagerRetriever leadingRetriever = Whitebox.getInternalState(leadingWebMonitor, "retriever");
        JobManagerRetriever followerRetriever = Whitebox.getInternalState(followerWebMonitor, "retriever");
        // Wait for the initial notifications
        waitForLeaderNotification(leadingSystem, jobManager[leaderIndex], leadingRetriever, deadline);
        waitForLeaderNotification(leadingSystem, jobManager[leaderIndex], followerRetriever, deadline);
        try (HttpTestClient leaderClient = new HttpTestClient("localhost", leadingWebMonitor.getServerPort());
            HttpTestClient followingClient = new HttpTestClient("localhost", followerWebMonitor.getServerPort())) {
            String expected = new Scanner(new File(MAIN_RESOURCES_PATH + "/index.html")).useDelimiter("\\A").next();
            // Request the file from the leading web server
            leaderClient.sendGetRequest("index.html", deadline.timeLeft());
            HttpTestClient.SimpleHttpResponse response = leaderClient.getNextResponse(deadline.timeLeft());
            assertEquals(HttpResponseStatus.OK, response.getStatus());
            assertEquals(response.getType(), MimeTypes.getMimeTypeForExtension("html"));
            assertEquals(expected, response.getContent());
            // Request the file from the following web server
            followingClient.sendGetRequest("index.html", deadline.timeLeft());
            response = followingClient.getNextResponse(deadline.timeLeft());
            assertEquals(HttpResponseStatus.TEMPORARY_REDIRECT, response.getStatus());
            assertTrue(response.getLocation().contains(String.valueOf(leadingWebMonitor.getServerPort())));
            // Kill the leader
            leadingSystem.shutdown();
            // Wait for the notification of the follower
            waitForLeaderNotification(followerSystem, jobManager[followerIndex], followerRetriever, deadline);
            // Same request to the new leader
            followingClient.sendGetRequest("index.html", deadline.timeLeft());
            response = followingClient.getNextResponse(deadline.timeLeft());
            assertEquals(HttpResponseStatus.OK, response.getStatus());
            assertEquals(response.getType(), MimeTypes.getMimeTypeForExtension("html"));
            assertEquals(expected, response.getContent());
            // Simple overview request
            followingClient.sendGetRequest("/overview", deadline.timeLeft());
            response = followingClient.getNextResponse(deadline.timeLeft());
            assertEquals(HttpResponseStatus.OK, response.getStatus());
            assertEquals(response.getType(), MimeTypes.getMimeTypeForExtension("json"));
            assertTrue(response.getContent().contains("\"taskmanagers\":1") || response.getContent().contains("\"taskmanagers\":0"));
        }
    } finally {
        for (ActorSystem system : jobManagerSystem) {
            if (system != null) {
                system.shutdown();
            }
        }
        for (WebMonitor monitor : webMonitor) {
            monitor.stop();
        }
        for (LeaderRetrievalService lrs : leaderRetrievalServices) {
            lrs.stop();
        }
    }
}

Example 24 with Deadline

use of scala.concurrent.duration.Deadline in project flink by apache.

the class WebRuntimeMonitorITCase method testLeaderNotAvailable.

@Test
public void testLeaderNotAvailable() throws Exception {
    final Deadline deadline = TestTimeout.fromNow();
    ActorSystem actorSystem = null;
    WebRuntimeMonitor webRuntimeMonitor = null;
    try (TestingServer zooKeeper = new TestingServer()) {
        File logDir = temporaryFolder.newFolder();
        Path logFile = Files.createFile(new File(logDir, "jobmanager.log").toPath());
        Files.createFile(new File(logDir, "jobmanager.out").toPath());
        final Configuration config = new Configuration();
        config.setInteger(ConfigConstants.JOB_MANAGER_WEB_PORT_KEY, 0);
        config.setString(ConfigConstants.JOB_MANAGER_WEB_LOG_PATH_KEY, logFile.toString());
        config.setString(HighAvailabilityOptions.HA_MODE, "ZOOKEEPER");
        config.setString(HighAvailabilityOptions.HA_ZOOKEEPER_QUORUM, zooKeeper.getConnectString());
        actorSystem = AkkaUtils.createDefaultActorSystem();
        LeaderRetrievalService leaderRetrievalService = mock(LeaderRetrievalService.class);
        webRuntimeMonitor = new WebRuntimeMonitor(config, leaderRetrievalService, actorSystem);
        webRuntimeMonitor.start("akka://schmakka");
        try (HttpTestClient client = new HttpTestClient("localhost", webRuntimeMonitor.getServerPort())) {
            client.sendGetRequest("index.html", deadline.timeLeft());
            HttpTestClient.SimpleHttpResponse response = client.getNextResponse();
            assertEquals(HttpResponseStatus.SERVICE_UNAVAILABLE, response.getStatus());
            assertEquals(MimeTypes.getMimeTypeForExtension("txt"), response.getType());
            assertTrue(response.getContent().contains("refresh"));
        }
    } finally {
        if (actorSystem != null) {
            actorSystem.shutdown();
        }
        if (webRuntimeMonitor != null) {
            webRuntimeMonitor.stop();
        }
    }
}

Example 25 with Deadline

use of scala.concurrent.duration.Deadline in project flink by apache.

the class WebRuntimeMonitorITCase method testNoEscape.

// ------------------------------------------------------------------------
// Tests that access outside of the web root is not allowed
// ------------------------------------------------------------------------
/**
	 * Files are copied from the flink-dist jar to a temporary directory and
	 * then served from there. Only allow to access files in this temporary
	 * directory.
	 */
@Test
public void testNoEscape() throws Exception {
    final Deadline deadline = TestTimeout.fromNow();
    TestingCluster flink = null;
    WebRuntimeMonitor webMonitor = null;
    try {
        flink = new TestingCluster(new Configuration());
        flink.start(true);
        webMonitor = startWebRuntimeMonitor(flink);
        try (HttpTestClient client = new HttpTestClient("localhost", webMonitor.getServerPort())) {
            String expectedIndex = new Scanner(new File(MAIN_RESOURCES_PATH + "/index.html")).useDelimiter("\\A").next();
            // 1) Request index.html from web server
            client.sendGetRequest("index.html", deadline.timeLeft());
            HttpTestClient.SimpleHttpResponse response = client.getNextResponse(deadline.timeLeft());
            assertEquals(HttpResponseStatus.OK, response.getStatus());
            assertEquals(response.getType(), MimeTypes.getMimeTypeForExtension("html"));
            assertEquals(expectedIndex, response.getContent());
            // 2) Request file outside of web root
            // Create a test file in the web base dir (parent of web root)
            File illegalFile = new File(webMonitor.getBaseDir(new Configuration()), "test-file-" + UUID.randomUUID());
            illegalFile.deleteOnExit();
            assertTrue("Failed to create test file", illegalFile.createNewFile());
            // Request the created file from the web server
            client.sendGetRequest("../" + illegalFile.getName(), deadline.timeLeft());
            response = client.getNextResponse(deadline.timeLeft());
            assertEquals("Unexpected status code " + response.getStatus() + " for file outside of web root.", HttpResponseStatus.NOT_FOUND, response.getStatus());
            // 3) Request non-existing file
            client.sendGetRequest("not-existing-resource", deadline.timeLeft());
            response = client.getNextResponse(deadline.timeLeft());
            assertEquals("Unexpected status code " + response.getStatus() + " for file outside of web root.", HttpResponseStatus.NOT_FOUND, response.getStatus());
        }
    } finally {
        if (flink != null) {
            flink.shutdown();
        }
        if (webMonitor != null) {
            webMonitor.stop();
        }
    }
}