Examples with Configuration org.apache.flink.configuration.Configuration used on opensource projects

Example 11 with Configuration

use of org.apache.flink.configuration.Configuration in project flink by apache.

the class JobSubmitTest method setupJobManager.

@BeforeClass
public static void setupJobManager() {
    jmConfig = new Configuration();
    int port = NetUtils.getAvailablePort();
    jmConfig.setString(ConfigConstants.JOB_MANAGER_IPC_ADDRESS_KEY, "localhost");
    jmConfig.setInteger(ConfigConstants.JOB_MANAGER_IPC_PORT_KEY, port);
    scala.Option<Tuple2<String, Object>> listeningAddress = scala.Option.apply(new Tuple2<String, Object>("localhost", port));
    jobManagerSystem = AkkaUtils.createActorSystem(jmConfig, listeningAddress);
    // only start JobManager (no ResourceManager)
    JobManager.startJobManagerActors(jmConfig, jobManagerSystem, TestingUtils.defaultExecutor(), TestingUtils.defaultExecutor(), JobManager.class, MemoryArchivist.class)._1();
    try {
        LeaderRetrievalService lrs = LeaderRetrievalUtils.createLeaderRetrievalService(jmConfig);
        jmGateway = LeaderRetrievalUtils.retrieveLeaderGateway(lrs, jobManagerSystem, timeout);
    } catch (Exception e) {
        fail("Could not retrieve the JobManager gateway. " + e.getMessage());
    }
}

Example 12 with Configuration

use of org.apache.flink.configuration.Configuration in project flink by apache.

the class JobManagerRunnerMockTest method setUp.

@Before
public void setUp() throws Exception {
    RpcService mockRpc = mock(RpcService.class);
    when(mockRpc.getAddress()).thenReturn("localhost");
    jobManager = mock(JobMaster.class);
    jobManagerGateway = mock(JobMasterGateway.class);
    when(jobManager.getSelf()).thenReturn(jobManagerGateway);
    when(jobManager.getRpcService()).thenReturn(mockRpc);
    PowerMockito.whenNew(JobMaster.class).withAnyArguments().thenReturn(jobManager);
    jobCompletion = new TestingOnCompletionActions();
    leaderElectionService = mock(LeaderElectionService.class);
    when(leaderElectionService.hasLeadership()).thenReturn(true);
    SubmittedJobGraphStore submittedJobGraphStore = mock(SubmittedJobGraphStore.class);
    blobStore = mock(BlobStore.class);
    HighAvailabilityServices haServices = mock(HighAvailabilityServices.class);
    when(haServices.getJobManagerLeaderElectionService(any(JobID.class))).thenReturn(leaderElectionService);
    when(haServices.getSubmittedJobGraphStore()).thenReturn(submittedJobGraphStore);
    when(haServices.createBlobStore()).thenReturn(blobStore);
    when(haServices.getRunningJobsRegistry()).thenReturn(runningJobsRegistry);
    HeartbeatServices heartbeatServices = mock(HeartbeatServices.class);
    runner = PowerMockito.spy(new JobManagerRunner(ResourceID.generate(), new JobGraph("test", new JobVertex("vertex")), mock(Configuration.class), mockRpc, haServices, heartbeatServices, JobManagerServices.fromConfiguration(new Configuration(), haServices), new MetricRegistry(MetricRegistryConfiguration.defaultMetricRegistryConfiguration()), jobCompletion, jobCompletion));
}

Example 13 with Configuration

use of org.apache.flink.configuration.Configuration in project flink by apache.

the class ZooKeeperLeaderElectionTest method testZooKeeperReelectionWithReplacement.

/**
	 * Tests the repeated reelection of {@link LeaderContender} once the current leader dies.
	 * Furthermore, it tests that new LeaderElectionServices can be started later on and that they
	 * successfully register at ZooKeeper and take part in the leader election.
	 */
@Test
public void testZooKeeperReelectionWithReplacement() throws Exception {
    Configuration configuration = new Configuration();
    configuration.setString(HighAvailabilityOptions.HA_ZOOKEEPER_QUORUM, testingServer.getConnectString());
    configuration.setString(HighAvailabilityOptions.HA_MODE, "zookeeper");
    int num = 3;
    int numTries = 30;
    ZooKeeperLeaderElectionService[] leaderElectionService = new ZooKeeperLeaderElectionService[num];
    TestingContender[] contenders = new TestingContender[num];
    ZooKeeperLeaderRetrievalService leaderRetrievalService = null;
    TestingListener listener = new TestingListener();
    try {
        leaderRetrievalService = ZooKeeperUtils.createLeaderRetrievalService(configuration);
        leaderRetrievalService.start(listener);
        for (int i = 0; i < num; i++) {
            leaderElectionService[i] = ZooKeeperUtils.createLeaderElectionService(configuration);
            contenders[i] = new TestingContender(TEST_URL + "_" + i + "_0", leaderElectionService[i]);
            leaderElectionService[i].start(contenders[i]);
        }
        String pattern = TEST_URL + "_" + "(\\d+)" + "_" + "(\\d+)";
        Pattern regex = Pattern.compile(pattern);
        for (int i = 0; i < numTries; i++) {
            listener.waitForNewLeader(timeout.toMillis());
            String address = listener.getAddress();
            Matcher m = regex.matcher(address);
            if (m.find()) {
                int index = Integer.parseInt(m.group(1));
                int lastTry = Integer.parseInt(m.group(2));
                assertEquals(listener.getLeaderSessionID(), contenders[index].getLeaderSessionID());
                // stop leader election service = revoke leadership
                leaderElectionService[index].stop();
                // create new leader election service which takes part in the leader election
                leaderElectionService[index] = ZooKeeperUtils.createLeaderElectionService(configuration);
                contenders[index] = new TestingContender(TEST_URL + "_" + index + "_" + (lastTry + 1), leaderElectionService[index]);
                leaderElectionService[index].start(contenders[index]);
            } else {
                throw new Exception("Did not find the leader's index.");
            }
        }
    } finally {
        if (leaderRetrievalService != null) {
            leaderRetrievalService.stop();
        }
        for (ZooKeeperLeaderElectionService electionService : leaderElectionService) {
            if (electionService != null) {
                electionService.stop();
            }
        }
    }
}

Example 14 with Configuration

use of org.apache.flink.configuration.Configuration in project flink by apache.

the class ZooKeeperLeaderElectionTest method testMultipleLeaders.

/**
	 * Tests that the current leader is notified when his leader connection information in ZooKeeper
	 * are overwritten. The leader must re-establish the correct leader connection information in
	 * ZooKeeper.
	 */
@Test
public void testMultipleLeaders() throws Exception {
    final String FAULTY_CONTENDER_URL = "faultyContender";
    final String leaderPath = "/leader";
    Configuration configuration = new Configuration();
    configuration.setString(HighAvailabilityOptions.HA_ZOOKEEPER_QUORUM, testingServer.getConnectString());
    configuration.setString(HighAvailabilityOptions.HA_MODE, "zookeeper");
    configuration.setString(ConfigConstants.HA_ZOOKEEPER_LEADER_PATH, leaderPath);
    ZooKeeperLeaderElectionService leaderElectionService = null;
    ZooKeeperLeaderRetrievalService leaderRetrievalService = null;
    ZooKeeperLeaderRetrievalService leaderRetrievalService2 = null;
    TestingListener listener = new TestingListener();
    TestingListener listener2 = new TestingListener();
    TestingContender contender;
    try {
        leaderElectionService = ZooKeeperUtils.createLeaderElectionService(configuration);
        leaderRetrievalService = ZooKeeperUtils.createLeaderRetrievalService(configuration);
        leaderRetrievalService2 = ZooKeeperUtils.createLeaderRetrievalService(configuration);
        contender = new TestingContender(TEST_URL, leaderElectionService);
        leaderElectionService.start(contender);
        leaderRetrievalService.start(listener);
        listener.waitForNewLeader(timeout.toMillis());
        assertEquals(listener.getLeaderSessionID(), contender.getLeaderSessionID());
        assertEquals(TEST_URL, listener.getAddress());
        CuratorFramework client = ZooKeeperUtils.startCuratorFramework(configuration);
        ByteArrayOutputStream baos = new ByteArrayOutputStream();
        ObjectOutputStream oos = new ObjectOutputStream(baos);
        oos.writeUTF(FAULTY_CONTENDER_URL);
        oos.writeObject(null);
        oos.close();
        // overwrite the current leader address, the leader should notice that and correct it
        boolean dataWritten = false;
        while (!dataWritten) {
            client.delete().forPath(leaderPath);
            try {
                client.create().forPath(leaderPath, baos.toByteArray());
                dataWritten = true;
            } catch (KeeperException.NodeExistsException e) {
            // this can happen if the leader election service was faster
            }
        }
        leaderRetrievalService2.start(listener2);
        listener2.waitForNewLeader(timeout.toMillis());
        if (FAULTY_CONTENDER_URL.equals(listener2.getAddress())) {
            listener2.waitForNewLeader(timeout.toMillis());
        }
        assertEquals(listener2.getLeaderSessionID(), contender.getLeaderSessionID());
        assertEquals(listener2.getAddress(), contender.getAddress());
    } finally {
        if (leaderElectionService != null) {
            leaderElectionService.stop();
        }
        if (leaderRetrievalService != null) {
            leaderRetrievalService.stop();
        }
        if (leaderRetrievalService2 != null) {
            leaderRetrievalService2.stop();
        }
    }
}

Example 15 with Configuration

use of org.apache.flink.configuration.Configuration in project flink by apache.

the class ZooKeeperLeaderElectionTest method testZooKeeperReelection.

/**
	 * Tests repeatedly the reelection of still available LeaderContender. After a contender has
	 * been elected as the leader, it is removed. This forces the ZooKeeperLeaderElectionService
	 * to elect a new leader.
	 */
@Test
public void testZooKeeperReelection() throws Exception {
    Configuration configuration = new Configuration();
    configuration.setString(HighAvailabilityOptions.HA_ZOOKEEPER_QUORUM, testingServer.getConnectString());
    configuration.setString(HighAvailabilityOptions.HA_MODE, "zookeeper");
    Deadline deadline = new FiniteDuration(5, TimeUnit.MINUTES).fromNow();
    int num = 20;
    ZooKeeperLeaderElectionService[] leaderElectionService = new ZooKeeperLeaderElectionService[num];
    TestingContender[] contenders = new TestingContender[num];
    ZooKeeperLeaderRetrievalService leaderRetrievalService = null;
    TestingListener listener = new TestingListener();
    try {
        leaderRetrievalService = ZooKeeperUtils.createLeaderRetrievalService(configuration);
        LOG.debug("Start leader retrieval service for the TestingListener.");
        leaderRetrievalService.start(listener);
        for (int i = 0; i < num; i++) {
            leaderElectionService[i] = ZooKeeperUtils.createLeaderElectionService(configuration);
            contenders[i] = new TestingContender(TEST_URL + "_" + i, leaderElectionService[i]);
            LOG.debug("Start leader election service for contender #{}.", i);
            leaderElectionService[i].start(contenders[i]);
        }
        String pattern = TEST_URL + "_" + "(\\d+)";
        Pattern regex = Pattern.compile(pattern);
        int numberSeenLeaders = 0;
        while (deadline.hasTimeLeft() && numberSeenLeaders < num) {
            LOG.debug("Wait for new leader #{}.", numberSeenLeaders);
            String address = listener.waitForNewLeader(deadline.timeLeft().toMillis());
            Matcher m = regex.matcher(address);
            if (m.find()) {
                int index = Integer.parseInt(m.group(1));
                TestingContender contender = contenders[index];
                // check that the retrieval service has retrieved the correct leader
                if (address.equals(contender.getAddress()) && listener.getLeaderSessionID().equals(contender.getLeaderSessionID())) {
                    // kill the election service of the leader
                    LOG.debug("Stop leader election service of contender #{}.", numberSeenLeaders);
                    leaderElectionService[index].stop();
                    leaderElectionService[index] = null;
                    numberSeenLeaders++;
                }
            } else {
                fail("Did not find the leader's index.");
            }
        }
        assertFalse(deadline.isOverdue());
        assertEquals(num, numberSeenLeaders);
    } finally {
        if (leaderRetrievalService != null) {
            leaderRetrievalService.stop();
        }
        for (ZooKeeperLeaderElectionService electionService : leaderElectionService) {
            if (electionService != null) {
                electionService.stop();
            }
        }
    }
}