Examples with Some scala.Some used on opensource projects

Example 11 with Some

use of scala.Some in project flink by apache.

the class JobManagerHAJobGraphRecoveryITCase method testClientNonDetachedListeningBehaviour.

/**
	 * Tests that clients receive updates after recovery by a new leader.
	 */
@Test
public void testClientNonDetachedListeningBehaviour() throws Exception {
    Configuration config = ZooKeeperTestUtils.createZooKeeperHAConfig(ZooKeeper.getConnectString(), FileStateBackendBasePath.getPath());
    // Test actor system
    ActorSystem testSystem = null;
    // JobManager setup. Start the job managers as separate processes in order to not run the
    // actors postStop, which cleans up all running jobs.
    JobManagerProcess[] jobManagerProcess = new JobManagerProcess[2];
    LeaderRetrievalService leaderRetrievalService = null;
    ActorSystem taskManagerSystem = null;
    try {
        final Deadline deadline = TestTimeOut.fromNow();
        // Test actor system
        testSystem = AkkaUtils.createActorSystem(new Configuration(), new Some<>(new Tuple2<String, Object>("localhost", 0)));
        // The job managers
        jobManagerProcess[0] = new JobManagerProcess(0, config);
        jobManagerProcess[1] = new JobManagerProcess(1, config);
        jobManagerProcess[0].startProcess();
        jobManagerProcess[1].startProcess();
        // Leader listener
        TestingListener leaderListener = new TestingListener();
        leaderRetrievalService = ZooKeeperUtils.createLeaderRetrievalService(config);
        leaderRetrievalService.start(leaderListener);
        // The task manager
        taskManagerSystem = AkkaUtils.createActorSystem(AkkaUtils.getDefaultAkkaConfig());
        TaskManager.startTaskManagerComponentsAndActor(config, ResourceID.generate(), taskManagerSystem, "localhost", Option.<String>empty(), Option.<LeaderRetrievalService>empty(), false, TaskManager.class);
        // Client test actor
        TestActorRef<RecordingTestClient> clientRef = TestActorRef.create(testSystem, Props.create(RecordingTestClient.class));
        JobGraph jobGraph = createBlockingJobGraph();
        {
            // Initial submission
            leaderListener.waitForNewLeader(deadline.timeLeft().toMillis());
            String leaderAddress = leaderListener.getAddress();
            UUID leaderId = leaderListener.getLeaderSessionID();
            // The client
            AkkaActorGateway client = new AkkaActorGateway(clientRef, leaderId);
            // Get the leader ref
            ActorRef leaderRef = AkkaUtils.getActorRef(leaderAddress, testSystem, deadline.timeLeft());
            ActorGateway leader = new AkkaActorGateway(leaderRef, leaderId);
            int numSlots = 0;
            while (numSlots == 0) {
                Future<?> slotsFuture = leader.ask(JobManagerMessages.getRequestTotalNumberOfSlots(), deadline.timeLeft());
                numSlots = (Integer) Await.result(slotsFuture, deadline.timeLeft());
            }
            // Submit the job in non-detached mode
            leader.tell(new SubmitJob(jobGraph, ListeningBehaviour.EXECUTION_RESULT_AND_STATE_CHANGES), client);
            JobManagerActorTestUtils.waitForJobStatus(jobGraph.getJobID(), JobStatus.RUNNING, leader, deadline.timeLeft());
        }
        // Who's the boss?
        JobManagerProcess leadingJobManagerProcess;
        if (jobManagerProcess[0].getJobManagerAkkaURL(deadline.timeLeft()).equals(leaderListener.getAddress())) {
            leadingJobManagerProcess = jobManagerProcess[0];
        } else {
            leadingJobManagerProcess = jobManagerProcess[1];
        }
        // Kill the leading job manager process
        leadingJobManagerProcess.destroy();
        {
            // Recovery by the standby JobManager
            leaderListener.waitForNewLeader(deadline.timeLeft().toMillis());
            String leaderAddress = leaderListener.getAddress();
            UUID leaderId = leaderListener.getLeaderSessionID();
            ActorRef leaderRef = AkkaUtils.getActorRef(leaderAddress, testSystem, deadline.timeLeft());
            ActorGateway leader = new AkkaActorGateway(leaderRef, leaderId);
            JobManagerActorTestUtils.waitForJobStatus(jobGraph.getJobID(), JobStatus.RUNNING, leader, deadline.timeLeft());
            // Cancel the job
            leader.tell(new JobManagerMessages.CancelJob(jobGraph.getJobID()));
        }
        // Wait for the execution result
        clientRef.underlyingActor().awaitJobResult(deadline.timeLeft().toMillis());
        int jobSubmitSuccessMessages = 0;
        for (Object msg : clientRef.underlyingActor().getMessages()) {
            if (msg instanceof JobManagerMessages.JobSubmitSuccess) {
                jobSubmitSuccessMessages++;
            }
        }
        // At least two submissions should be ack-ed (initial and recovery). This is quite
        // conservative, but it is still possible that these messages are overtaken by the
        // final message.
        assertEquals(2, jobSubmitSuccessMessages);
    } catch (Throwable t) {
        // Print early (in some situations the process logs get too big
        // for Travis and the root problem is not shown)
        t.printStackTrace();
        // In case of an error, print the job manager process logs.
        if (jobManagerProcess[0] != null) {
            jobManagerProcess[0].printProcessLog();
        }
        if (jobManagerProcess[1] != null) {
            jobManagerProcess[1].printProcessLog();
        }
        throw t;
    } finally {
        if (jobManagerProcess[0] != null) {
            jobManagerProcess[0].destroy();
        }
        if (jobManagerProcess[1] != null) {
            jobManagerProcess[1].destroy();
        }
        if (leaderRetrievalService != null) {
            leaderRetrievalService.stop();
        }
        if (taskManagerSystem != null) {
            taskManagerSystem.shutdown();
        }
        if (testSystem != null) {
            testSystem.shutdown();
        }
    }
}

Example 12 with Some

use of scala.Some in project flink by apache.

the class JobManagerProcessReapingTest method testReapProcessOnFailure.

@Test
public void testReapProcessOnFailure() {
    Process jmProcess = null;
    ActorSystem localSystem = null;
    final StringWriter processOutput = new StringWriter();
    try {
        String javaCommand = getJavaCommandPath();
        // is available on this machine
        if (javaCommand == null) {
            System.out.println("---- Skipping JobManagerProcessReapingTest : Could not find java executable ----");
            return;
        }
        // create a logging file for the process
        File tempLogFile = File.createTempFile("testlogconfig", "properties");
        tempLogFile.deleteOnExit();
        CommonTestUtils.printLog4jDebugConfig(tempLogFile);
        // start a JobManger process
        // the log level must be at least INFO, otherwise the bound port cannot be retrieved
        String[] command = new String[] { javaCommand, "-Dlog.level=DEBUG", "-Dlog4j.configuration=file:" + tempLogFile.getAbsolutePath(), "-Xms256m", "-Xmx256m", "-classpath", getCurrentClasspath(), JobManagerTestEntryPoint.class.getName() };
        // spawn the process and collect its output
        ProcessBuilder bld = new ProcessBuilder(command);
        jmProcess = bld.start();
        new PipeForwarder(jmProcess.getErrorStream(), processOutput);
        // start another actor system so we can send something to the JobManager
        Tuple2<String, Object> localAddress = new Tuple2<String, Object>("localhost", 0);
        localSystem = AkkaUtils.createActorSystem(new Configuration(), new Some<Tuple2<String, Object>>(localAddress));
        // grab the reference to the JobManager. try multiple times, until the process
        // is started and the JobManager is up
        ActorRef jobManagerRef = null;
        Throwable lastError = null;
        // Log message on JobManager must be: Starting JobManager at ...://flink@...:port/..."
        // otherwise, the pattern does not match and, thus, cannot retrieve the bound port
        String pattern = "Starting JobManager at [^:]*://flink@[^:]*:(\\d*)/";
        Pattern r = Pattern.compile(pattern);
        int jobManagerPort = -1;
        for (int i = 0; i < 40; i++) {
            Matcher m = r.matcher(processOutput.toString());
            if (m.find()) {
                jobManagerPort = Integer.parseInt(m.group(1));
                break;
            }
            Thread.sleep(500);
        }
        if (jobManagerPort != -1) {
            try {
                jobManagerRef = JobManager.getJobManagerActorRef("akka.tcp", NetUtils.unresolvedHostAndPortToNormalizedString("localhost", jobManagerPort), localSystem, new FiniteDuration(25, TimeUnit.SECONDS));
            } catch (Throwable t) {
                // job manager probably not ready yet
                lastError = t;
            }
        } else {
            fail("Could not determine port of started JobManager.");
        }
        assertTrue("JobManager process died", isProcessAlive(jmProcess));
        if (jobManagerRef == null) {
            if (lastError != null) {
                lastError.printStackTrace();
            }
            fail("JobManager process did not launch the JobManager properly. Failed to look up JobManager actor at" + " localhost:" + jobManagerPort);
        }
        // kill the JobManager actor
        jobManagerRef.tell(PoisonPill.getInstance(), ActorRef.noSender());
        // wait for max 5 seconds for the process to terminate
        {
            long now = System.currentTimeMillis();
            long deadline = now + 5000;
            while (now < deadline && isProcessAlive(jmProcess)) {
                Thread.sleep(100);
                now = System.currentTimeMillis();
            }
        }
        assertFalse("JobManager process did not terminate upon actor death", isProcessAlive(jmProcess));
        int returnCode = jmProcess.exitValue();
        assertEquals("JobManager died, but not because of the process reaper", JobManager.RUNTIME_FAILURE_RETURN_CODE(), returnCode);
    } catch (Exception e) {
        e.printStackTrace();
        printProcessLog(processOutput.toString());
        fail(e.getMessage());
    } catch (Error e) {
        e.printStackTrace();
        printProcessLog(processOutput.toString());
        throw e;
    } finally {
        if (jmProcess != null) {
            jmProcess.destroy();
        }
        if (localSystem != null) {
            localSystem.shutdown();
        }
    }
}

Example 13 with Some

use of scala.Some in project carbondata by apache.

the class PreAggregateDataMapProvider method freeMeta.

@Override
public void freeMeta(CarbonTable mainTable, DataMapSchema dataMapSchema) {
    dropTableCommand = new CarbonDropTableCommand(true, new Some<>(dataMapSchema.getRelationIdentifier().getDatabaseName()), dataMapSchema.getRelationIdentifier().getTableName(), true);
    dropTableCommand.processMetadata(sparkSession);
}