Examples with Latch org.apache.samza.coordinator.Latch used on opensource projects

Example 1 with Latch

use of org.apache.samza.coordinator.Latch in project samza by apache.

the class TestZkProcessorLatch method testLatchExpires.

@Test
public void testLatchExpires() {
    final String latchId = "testLatchExpires";
    final int latchSize = 3;
    Latch latch = new ZkProcessorLatch(latchSize, latchId, "test", testZkUtils);
    try {
        latch.countDown();
        latch.await(5, TimeUnit.SECONDS);
    } catch (TimeoutException e) {
    // expected
    } catch (Exception e) {
        Assert.fail(String.format("Expected only TimeoutException! Received %s", e));
    }
}

Example 2 with Latch

use of org.apache.samza.coordinator.Latch in project samza by apache.

the class TestZkProcessorLatch method testLatchSizeOneWithTwoParticipants.

@Test
public void testLatchSizeOneWithTwoParticipants() {
    final int latchSize = 1;
    final String latchId = "testLatchSizeOneWithTwoParticipants";
    ExecutorService pool = Executors.newFixedThreadPool(3);
    Future f1 = pool.submit(() -> {
        String participant1 = "participant1";
        ZkUtils zkUtils = getZkUtilsWithNewClient(participant1);
        zkUtils.connect();
        Latch latch = new ZkProcessorLatch(latchSize, latchId, participant1, zkUtils);
        // latch.countDown(); only one thread counts down
        try {
            latch.await(30, TimeUnit.SECONDS);
        } catch (TimeoutException e) {
            Assert.fail(String.format("await timed out from  %s - %s", participant1, e.getLocalizedMessage()));
        } finally {
            zkUtils.close();
        }
    });
    Future f2 = pool.submit(getParticipantRunnable(latchSize, latchId, "participant2"));
    try {
        f1.get(30, TimeUnit.SECONDS);
        f2.get(30, TimeUnit.SECONDS);
    } catch (Exception e) {
        Assert.fail("failed to get future." + e.getLocalizedMessage());
    } finally {
        pool.shutdownNow();
    }
    try {
        List<String> latchParticipants = testZkUtils.getZkClient().getChildren(String.format("%s/%s_%s", KEY_BUILDER.getRootPath(), ZkProcessorLatch.LATCH_PATH, latchId));
        Assert.assertNotNull(latchParticipants);
        Assert.assertEquals(1, latchParticipants.size());
        Assert.assertEquals("0000000000", latchParticipants.get(0));
    } catch (Exception e) {
        Assert.fail("Failed to read the latch status from ZK directly" + e.getLocalizedMessage());
    }
}

Example 3 with Latch

use of org.apache.samza.coordinator.Latch in project samza by apache.

the class TestLocalApplicationRunner method testStreamCreationWithCoordination.

@Test
public void testStreamCreationWithCoordination() throws Exception {
    Map<String, String> config = new HashMap<>();
    LocalApplicationRunner runner = new LocalApplicationRunner(new MapConfig(config));
    StreamApplication app = mock(StreamApplication.class);
    doNothing().when(app).init(anyObject(), anyObject());
    ExecutionPlanner planner = mock(ExecutionPlanner.class);
    Field plannerField = runner.getClass().getSuperclass().getDeclaredField("planner");
    plannerField.setAccessible(true);
    plannerField.set(runner, planner);
    StreamManager streamManager = mock(StreamManager.class);
    Field streamManagerField = runner.getClass().getSuperclass().getDeclaredField("streamManager");
    streamManagerField.setAccessible(true);
    streamManagerField.set(runner, streamManager);
    ArgumentCaptor<List> captor = ArgumentCaptor.forClass(List.class);
    ExecutionPlan plan = new ExecutionPlan() {

        @Override
        public List<JobConfig> getJobConfigs() {
            return Collections.emptyList();
        }

        @Override
        public List<StreamSpec> getIntermediateStreams() {
            return Collections.singletonList(new StreamSpec("test-stream", "test-stream", "test-system"));
        }

        @Override
        public String getPlanAsJson() throws Exception {
            return "";
        }
    };
    when(planner.plan(anyObject())).thenReturn(plan);
    LocalApplicationRunner spy = spy(runner);
    CoordinationUtils coordinationUtils = mock(CoordinationUtils.class);
    LeaderElector leaderElector = new LeaderElector() {

        private LeaderElectorListener leaderElectorListener;

        @Override
        public void setLeaderElectorListener(LeaderElectorListener listener) {
            this.leaderElectorListener = listener;
        }

        @Override
        public void tryBecomeLeader() {
            leaderElectorListener.onBecomingLeader();
        }

        @Override
        public void resignLeadership() {
        }

        @Override
        public boolean amILeader() {
            return false;
        }
    };
    Latch latch = new Latch() {

        boolean done = false;

        @Override
        public void await(long timeout, TimeUnit tu) throws TimeoutException {
            // in this test, latch is released before wait
            assertTrue(done);
        }

        @Override
        public void countDown() {
            done = true;
        }
    };
    when(coordinationUtils.getLeaderElector()).thenReturn(leaderElector);
    when(coordinationUtils.getLatch(anyInt(), anyString())).thenReturn(latch);
    doReturn(coordinationUtils).when(spy).createCoordinationUtils();
    try {
        spy.run(app);
    } catch (Throwable t) {
        //no jobs exception
        assertNotNull(t);
    }
    verify(streamManager).createStreams(captor.capture());
    List<StreamSpec> streamSpecs = captor.getValue();
    assertEquals(streamSpecs.size(), 1);
    assertEquals(streamSpecs.get(0).getId(), "test-stream");
}

Example 4 with Latch

use of org.apache.samza.coordinator.Latch in project samza by apache.

the class LocalApplicationRunner method createStreams.

/**
   * Create intermediate streams using {@link org.apache.samza.execution.StreamManager}.
   * If {@link CoordinationUtils} is provided, this function will first invoke leader election, and the leader
   * will create the streams. All the runner processes will wait on the latch that is released after the leader finishes
   * stream creation.
   * @param intStreams list of intermediate {@link StreamSpec}s
   * @throws Exception exception for latch timeout
   */
/* package private */
void createStreams(List<StreamSpec> intStreams) throws Exception {
    if (!intStreams.isEmpty()) {
        if (coordinationUtils != null) {
            Latch initLatch = coordinationUtils.getLatch(1, INIT_LATCH_ID);
            LeaderElector leaderElector = coordinationUtils.getLeaderElector();
            leaderElector.setLeaderElectorListener(() -> {
                getStreamManager().createStreams(intStreams);
                initLatch.countDown();
            });
            leaderElector.tryBecomeLeader();
            initLatch.await(LATCH_TIMEOUT_MINUTES, TimeUnit.MINUTES);
        } else {
            // each application process will try creating the streams, which
            // requires stream creation to be idempotent
            getStreamManager().createStreams(intStreams);
        }
    }
}