Examples with DistributedLock org.apache.samza.coordinator.DistributedLock used on opensource projects

Example 1 with DistributedLock

use of org.apache.samza.coordinator.DistributedLock in project samza by apache.

the class TestLocalApplicationRunner method prepareTestForRunId.

private void prepareTestForRunId() throws Exception {
    coordinationUtils = mock(CoordinationUtils.class);
    DistributedLock lock = mock(DistributedLock.class);
    when(lock.lock(anyObject())).thenReturn(true);
    when(coordinationUtils.getLock(anyString())).thenReturn(lock);
    clusterMembership = mock(ClusterMembership.class);
    when(clusterMembership.getNumberOfProcessors()).thenReturn(1);
    when(coordinationUtils.getClusterMembership()).thenReturn(clusterMembership);
    metadataStore = mock(ZkMetadataStore.class);
    when(metadataStore.get(any())).thenReturn(null);
    PowerMockito.whenNew(ZkMetadataStore.class).withAnyArguments().thenReturn(metadataStore);
    ApplicationDescriptorImpl<? extends ApplicationDescriptor> appDesc = ApplicationDescriptorUtil.getAppDescriptor(mockApp, config);
    runner = spy(new LocalApplicationRunner(mockApp, config, coordinationUtils));
    localPlanner = spy(new LocalJobPlanner(appDesc, coordinationUtils, "FAKE_UID", "FAKE_RUNID"));
    doReturn(localPlanner).when(runner).getPlanner();
    StreamProcessor sp = mock(StreamProcessor.class);
    CoordinatorStreamStore coordinatorStreamStore = mock(CoordinatorStreamStore.class);
    doReturn(sp).when(runner).createStreamProcessor(anyObject(), anyObject(), anyObject(), anyObject(), any(CoordinatorStreamStore.class));
    doReturn(coordinatorStreamStore).when(runner).createCoordinatorStreamStore(any(Config.class));
}

Example 2 with DistributedLock

use of org.apache.samza.coordinator.DistributedLock in project samza by apache.

the class LocalJobPlanner method checkAndCreateStreams.

private void checkAndCreateStreams(String lockId, List<StreamSpec> intStreams, StreamManager streamManager) throws TimeoutException {
    MetadataStore metadataStore = getMetadataStore();
    DistributedLock distributedLock = coordinationUtils.getLock(lockId);
    if (distributedLock == null || metadataStore == null) {
        LOG.warn("Processor {} failed to create utils. Each processor will attempt to create streams.", processorId);
        // each application process will try creating the streams, which requires stream creation to be idempotent
        streamManager.createStreams(intStreams);
        return;
    }
    // Start timer for timeout
    long startTime = System.currentTimeMillis();
    long lockTimeout = TimeUnit.MILLISECONDS.convert(CoordinationConstants.LOCK_TIMEOUT_MS, TimeUnit.MILLISECONDS);
    // does not die of timeout exception and comes back and checks for state and proceeds
    while ((System.currentTimeMillis() - startTime) < lockTimeout) {
        if (metadataStore.get(String.format(STREAM_CREATED_STATE_KEY, lockId)) != null) {
            LOG.info("Processor {} found streams created state data. They must've been created by another processor.", processorId);
            break;
        }
        try {
            if (distributedLock.lock(Duration.ofMillis(10000))) {
                LOG.info("lock acquired for streams creation by Processor " + processorId);
                streamManager.createStreams(intStreams);
                String streamCreatedMessage = "Streams created by processor " + processorId;
                metadataStore.put(String.format(STREAM_CREATED_STATE_KEY, lockId), streamCreatedMessage.getBytes("UTF-8"));
                metadataStore.flush();
                distributedLock.unlock();
                break;
            } else {
                LOG.info("Processor {} failed to get the lock for stream initialization. Will try again until time out", processorId);
            }
        } catch (UnsupportedEncodingException e) {
            String msg = String.format("Processor {} failed to encode string for stream initialization", processorId);
            throw new SamzaException(msg, e);
        }
    }
    if ((System.currentTimeMillis() - startTime) >= lockTimeout) {
        throw new TimeoutException(String.format("Processor {} failed to get the lock for stream initialization within {} milliseconds.", processorId, CoordinationConstants.LOCK_TIMEOUT_MS));
    }
}

Example 3 with DistributedLock

use of org.apache.samza.coordinator.DistributedLock in project samza by apache.

the class TestLocalJobPlanner method createLocalJobPlanner.

private LocalJobPlanner createLocalJobPlanner(ApplicationDescriptorImpl<? extends ApplicationDescriptor> appDesc) throws Exception {
    CoordinationUtils coordinationUtils = mock(CoordinationUtils.class);
    DistributedLock distributedLock = mock(DistributedLock.class);
    when(distributedLock.lock(anyObject())).thenReturn(true);
    when(coordinationUtils.getLock(anyString())).thenReturn(distributedLock);
    ZkMetadataStore zkMetadataStore = mock(ZkMetadataStore.class);
    when(zkMetadataStore.get(any())).thenReturn(null);
    PowerMockito.whenNew(ZkMetadataStore.class).withAnyArguments().thenReturn(zkMetadataStore);
    return spy(new LocalJobPlanner(appDesc, coordinationUtils, "FAKE_UID", "FAKE_RUNID"));
}

Example 4 with DistributedLock

use of org.apache.samza.coordinator.DistributedLock in project samza by apache.

the class TestLocalApplicationRunner method prepareTest.

private void prepareTest() throws Exception {
    CoordinationUtils coordinationUtils = mock(CoordinationUtils.class);
    DistributedLock distributedLock = mock(DistributedLock.class);
    when(distributedLock.lock(anyObject())).thenReturn(true);
    when(coordinationUtils.getLock(anyString())).thenReturn(distributedLock);
    ZkMetadataStore zkMetadataStore = mock(ZkMetadataStore.class);
    when(zkMetadataStore.get(any())).thenReturn(null);
    PowerMockito.whenNew(ZkMetadataStore.class).withAnyArguments().thenReturn(zkMetadataStore);
    ApplicationDescriptorImpl<? extends ApplicationDescriptor> appDesc = ApplicationDescriptorUtil.getAppDescriptor(mockApp, config);
    localPlanner = spy(new LocalJobPlanner(appDesc, coordinationUtils, "FAKE_UID", "FAKE_RUNID"));
    runner = spy(new LocalApplicationRunner(mockApp, config, coordinationUtils));
    doReturn(localPlanner).when(runner).getPlanner();
}

Example 5 with DistributedLock

use of org.apache.samza.coordinator.DistributedLock in project samza by apache.

the class TestLocalJobPlanner method testStreamCreationWithCoordination.

@Test
public void testStreamCreationWithCoordination() throws Exception {
    StreamApplicationDescriptorImpl appDesc = mock(StreamApplicationDescriptorImpl.class);
    doReturn(mock(Config.class)).when(appDesc).getConfig();
    localPlanner = createLocalJobPlanner(appDesc);
    StreamManager streamManager = mock(StreamManager.class);
    doReturn(streamManager).when(localPlanner).buildAndStartStreamManager(any(Config.class));
    ExecutionPlan plan = mock(ExecutionPlan.class);
    when(plan.getIntermediateStreams()).thenReturn(Collections.singletonList(new StreamSpec("test-stream", "test-stream", "test-system")));
    when(plan.getPlanAsJson()).thenReturn("");
    when(plan.getJobConfigs()).thenReturn(Collections.singletonList(mock(JobConfig.class)));
    doReturn(plan).when(localPlanner).getExecutionPlan(any());
    CoordinationUtils coordinationUtils = mock(CoordinationUtils.class);
    CoordinationUtilsFactory coordinationUtilsFactory = mock(CoordinationUtilsFactory.class);
    JobCoordinatorConfig mockJcConfig = mock(JobCoordinatorConfig.class);
    when(mockJcConfig.getCoordinationUtilsFactory()).thenReturn(coordinationUtilsFactory);
    PowerMockito.whenNew(JobCoordinatorConfig.class).withAnyArguments().thenReturn(mockJcConfig);
    DistributedLock lock = mock(DistributedLock.class);
    when(lock.lock(anyObject())).thenReturn(true);
    when(coordinationUtils.getLock(anyString())).thenReturn(lock);
    when(coordinationUtilsFactory.getCoordinationUtils(anyString(), anyString(), anyObject())).thenReturn(coordinationUtils);
    localPlanner.prepareJobs();
    ArgumentCaptor<List> captor = ArgumentCaptor.forClass(List.class);
    verify(streamManager).createStreams(captor.capture());
    List<StreamSpec> streamSpecs = captor.getValue();
    assertEquals(streamSpecs.size(), 1);
    assertEquals(streamSpecs.get(0).getId(), "test-stream");
    verify(streamManager).stop();
}