Examples with ElectionHandler org.apache.twill.api.ElectionHandler used on opensource projects

Example 1 with ElectionHandler

use of org.apache.twill.api.ElectionHandler in project cdap by caskdata.

the class DistributedProgramRunnableModuleTest method createModule.

@Test
public void createModule() throws Exception {
    DistributedProgramRunnableModule distributedProgramRunnableModule = new DistributedProgramRunnableModule(CConfiguration.create(), new Configuration());
    Guice.createInjector(distributedProgramRunnableModule.createModule(new ProgramId("ns", "app", ProgramType.MAPREDUCE, "program"), RunIds.generate().getId(), "0", "user/host@KDC.NET"));
    Guice.createInjector(distributedProgramRunnableModule.createModule(new TwillContext() {

        @Override
        public RunId getRunId() {
            return null;
        }

        @Override
        public RunId getApplicationRunId() {
            return null;
        }

        @Override
        public int getInstanceCount() {
            return 0;
        }

        @Override
        public InetAddress getHost() {
            // used by DistributedProgramRunnableModule#createModule(TwillContext)
            return new InetSocketAddress("localhost", 0).getAddress();
        }

        @Override
        public String[] getArguments() {
            return new String[0];
        }

        @Override
        public String[] getApplicationArguments() {
            return new String[0];
        }

        @Override
        public TwillRunnableSpecification getSpecification() {
            return null;
        }

        @Override
        public int getInstanceId() {
            return 0;
        }

        @Override
        public int getVirtualCores() {
            return 0;
        }

        @Override
        public int getMaxMemoryMB() {
            return 0;
        }

        @Override
        public ServiceDiscovered discover(String name) {
            return null;
        }

        @Override
        public Cancellable electLeader(String name, ElectionHandler participantHandler) {
            return null;
        }

        @Override
        public Lock createLock(String name) {
            return null;
        }

        @Override
        public Cancellable announce(String serviceName, int port) {
            return null;
        }

        @Override
        public Cancellable announce(String serviceName, int port, byte[] payload) {
            return null;
        }
    }, new ProgramId("ns", "app", ProgramType.MAPREDUCE, "program"), RunIds.generate().getId(), "0", "user/host@KDC.NET"));
}

Example 2 with ElectionHandler

use of org.apache.twill.api.ElectionHandler in project cdap by caskdata.

the class LeaderElectionInfoServiceTest method testParticipants.

@Test
public void testParticipants() throws Exception {
    final int size = 5;
    String prefix = "/election";
    List<ZKClientService> zkClients = new ArrayList<>();
    ZKClientService infoZKClient = DefaultZKClientService.Builder.of(zkServer.getConnectionStr()).build();
    infoZKClient.startAndWait();
    zkClients.add(infoZKClient);
    // Start the LeaderElectionInfoService
    LeaderElectionInfoService infoService = new LeaderElectionInfoService(infoZKClient, prefix);
    infoService.startAndWait();
    // This will timeout as there is no leader election node created yet
    try {
        infoService.getParticipants(1, TimeUnit.SECONDS);
        Assert.fail("Expected timeout");
    } catch (TimeoutException e) {
    // Expected
    }
    // Starts multiple leader elections
    List<LeaderElection> leaderElections = new ArrayList<>();
    for (int i = 0; i < size; i++) {
        ZKClientService zkClient = DefaultZKClientService.Builder.of(zkServer.getConnectionStr()).build();
        zkClient.startAndWait();
        zkClients.add(zkClient);
        final int participantId = i;
        LeaderElection leaderElection = new LeaderElection(zkClient, prefix, new ElectionHandler() {

            @Override
            public void leader() {
                LOG.info("Leader: {}", participantId);
            }

            @Override
            public void follower() {
                LOG.info("Follow: {}", participantId);
            }
        });
        leaderElection.start();
        leaderElections.add(leaderElection);
    }
    // Get the dynamic participants map
    final SortedMap<Integer, LeaderElectionInfoService.Participant> participants = infoService.getParticipants(5, TimeUnit.SECONDS);
    // Expects to set all participants with hostname information
    Tasks.waitFor(true, new Callable<Boolean>() {

        @Override
        public Boolean call() throws Exception {
            if (participants.size() != size) {
                return false;
            }
            return Iterables.all(participants.values(), new Predicate<LeaderElectionInfoService.Participant>() {

                @Override
                public boolean apply(LeaderElectionInfoService.Participant input) {
                    return input.getHostname() != null;
                }
            });
        }
    }, 5, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS);
    // Fetch the static snapshot. It should be the same as the dynamic participants.
    SortedMap<Integer, LeaderElectionInfoService.Participant> snapshot = infoService.fetchCurrentParticipants();
    Assert.assertEquals(size, snapshot.size());
    Assert.assertEquals(participants, snapshot);
    int expectedSize = size;
    for (LeaderElection leaderElection : leaderElections) {
        leaderElection.stopAndWait();
        Tasks.waitFor(--expectedSize, new Callable<Integer>() {

            @Override
            public Integer call() throws Exception {
                return participants.size();
            }
        }, 5, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS);
    }
    // Fetch the static snapshot again. It should be empty and the same as the dynamic one.
    snapshot = infoService.fetchCurrentParticipants();
    Assert.assertTrue(snapshot.isEmpty());
    Assert.assertEquals(participants, snapshot);
    infoService.stopAndWait();
    for (ZKClientService zkClient : zkClients) {
        zkClient.stopAndWait();
    }
}

Example 3 with ElectionHandler

use of org.apache.twill.api.ElectionHandler in project cdap by caskdata.

the class DistributedStreamService method performLeaderElection.

/**
   * Elect one leader among the {@link DistributedStreamService}s running in different Twill runnables.
   */
private void performLeaderElection() {
    // Start the resource coordinator that will map Streams to Stream handlers
    leaderElection = new LeaderElection(// TODO: Should unify this leader election with DistributedStreamFileJanitorService
    zkClient, "/election/" + STREAMS_COORDINATOR, new ElectionHandler() {

        @Override
        public void leader() {
            LOG.info("Became Stream handler leader. Starting resource coordinator.");
            resourceCoordinator = new ResourceCoordinator(getCoordinatorZKClient(), discoveryServiceClient, new BalancedAssignmentStrategy());
            resourceCoordinator.startAndWait();
            updateRequirement();
        }

        @Override
        public void follower() {
            LOG.info("Became Stream handler follower.");
            if (resourceCoordinator != null) {
                resourceCoordinator.stopAndWait();
            }
        }
    });
    leaderElection.start();
}

Example 4 with ElectionHandler

use of org.apache.twill.api.ElectionHandler in project cdap by caskdata.

the class InMemoryElectionTest method testElection.

@Test(timeout = 5000)
public void testElection() throws ExecutionException, InterruptedException, BrokenBarrierException {
    final InMemoryElectionRegistry electionRegistry = new InMemoryElectionRegistry();
    ExecutorService executor = Executors.newCachedThreadPool();
    // Create 5 participants to join leader election process simultaneously
    int participantCount = 5;
    final CyclicBarrier barrier = new CyclicBarrier(participantCount + 1);
    final Semaphore leaderSem = new Semaphore(0);
    final Semaphore followerSem = new Semaphore(0);
    final CountDownLatch[] stopLatch = new CountDownLatch[participantCount];
    try {
        final AtomicInteger currentLeader = new AtomicInteger(-1);
        for (int i = 0; i < participantCount; i++) {
            stopLatch[i] = new CountDownLatch(1);
            final int idx = i;
            executor.submit(new Runnable() {

                @Override
                public void run() {
                    try {
                        barrier.await();
                        Cancellable cancel = electionRegistry.join("test", new ElectionHandler() {

                            @Override
                            public void leader() {
                                currentLeader.set(idx);
                                leaderSem.release();
                            }

                            @Override
                            public void follower() {
                                followerSem.release();
                            }
                        });
                        stopLatch[idx].await(10, TimeUnit.SECONDS);
                        cancel.cancel();
                    } catch (Exception e) {
                        LOG.error(e.getMessage(), e);
                    }
                }
            });
        }
        // Sync the joining
        barrier.await();
        // There should be 1 leader and 4 followers
        leaderSem.tryAcquire(10, TimeUnit.SECONDS);
        followerSem.tryAcquire(participantCount - 1, 10, TimeUnit.SECONDS);
        // Continuously stopping leader until there is one left.
        for (int i = 0; i < participantCount - 1; i++) {
            stopLatch[currentLeader.get()].countDown();
            // Each time when the leader is unregistered from the leader election, a new leader would rise and
            // the old leader would become a follower.
            leaderSem.tryAcquire(10, TimeUnit.SECONDS);
            followerSem.tryAcquire(10, TimeUnit.SECONDS);
        }
        // Withdraw the last leader, it'd become follower as well.
        stopLatch[currentLeader.get()].countDown();
        followerSem.tryAcquire(10, TimeUnit.SECONDS);
    } finally {
        executor.shutdown();
        executor.awaitTermination(5L, TimeUnit.SECONDS);
    }
}

Example 5 with ElectionHandler

use of org.apache.twill.api.ElectionHandler in project cdap by caskdata.

the class DistributedKeyManager method doInit.

@Override
protected void doInit() throws IOException {
    this.keyCache.addListener(this);
    try {
        keyCache.init();
    } catch (InterruptedException ie) {
        throw Throwables.propagate(ie);
    }
    this.leaderElection = new LeaderElection(zookeeper, "/leader", new ElectionHandler() {

        @Override
        public void leader() {
            leader.set(true);
            LOG.info("Transitioned to leader");
            if (currentKey == null) {
                rotateKey();
            }
        }

        @Override
        public void follower() {
            leader.set(false);
            LOG.info("Transitioned to follower");
        }
    });
    this.leaderElection.start();
    startExpirationThread();
}