Examples with EnsurePath org.apache.curator.utils.EnsurePath used on opensource projects

Example 1 with EnsurePath

use of org.apache.curator.utils.EnsurePath in project hadoop by apache.

the class ZKDelegationTokenSecretManager method startThreads.

@Override
public void startThreads() throws IOException {
    if (!isExternalClient) {
        try {
            zkClient.start();
        } catch (Exception e) {
            throw new IOException("Could not start Curator Framework", e);
        }
    } else {
        // If namespace parents are implicitly created, they won't have ACLs.
        // So, let's explicitly create them.
        CuratorFramework nullNsFw = zkClient.usingNamespace(null);
        EnsurePath ensureNs = nullNsFw.newNamespaceAwareEnsurePath("/" + zkClient.getNamespace());
        try {
            ensureNs.ensure(nullNsFw.getZookeeperClient());
        } catch (Exception e) {
            throw new IOException("Could not create namespace", e);
        }
    }
    listenerThreadPool = Executors.newSingleThreadExecutor();
    try {
        delTokSeqCounter = new SharedCount(zkClient, ZK_DTSM_SEQNUM_ROOT, 0);
        if (delTokSeqCounter != null) {
            delTokSeqCounter.start();
        }
    } catch (Exception e) {
        throw new IOException("Could not start Sequence Counter", e);
    }
    try {
        keyIdSeqCounter = new SharedCount(zkClient, ZK_DTSM_KEYID_ROOT, 0);
        if (keyIdSeqCounter != null) {
            keyIdSeqCounter.start();
        }
    } catch (Exception e) {
        throw new IOException("Could not start KeyId Counter", e);
    }
    try {
        createPersistentNode(ZK_DTSM_MASTER_KEY_ROOT);
        createPersistentNode(ZK_DTSM_TOKENS_ROOT);
    } catch (Exception e) {
        throw new RuntimeException("Could not create ZK paths");
    }
    try {
        keyCache = new PathChildrenCache(zkClient, ZK_DTSM_MASTER_KEY_ROOT, true);
        if (keyCache != null) {
            keyCache.start(StartMode.BUILD_INITIAL_CACHE);
            keyCache.getListenable().addListener(new PathChildrenCacheListener() {

                @Override
                public void childEvent(CuratorFramework client, PathChildrenCacheEvent event) throws Exception {
                    switch(event.getType()) {
                        case CHILD_ADDED:
                            processKeyAddOrUpdate(event.getData().getData());
                            break;
                        case CHILD_UPDATED:
                            processKeyAddOrUpdate(event.getData().getData());
                            break;
                        case CHILD_REMOVED:
                            processKeyRemoved(event.getData().getPath());
                            break;
                        default:
                            break;
                    }
                }
            }, listenerThreadPool);
            loadFromZKCache(false);
        }
    } catch (Exception e) {
        throw new IOException("Could not start PathChildrenCache for keys", e);
    }
    try {
        tokenCache = new PathChildrenCache(zkClient, ZK_DTSM_TOKENS_ROOT, true);
        if (tokenCache != null) {
            tokenCache.start(StartMode.BUILD_INITIAL_CACHE);
            tokenCache.getListenable().addListener(new PathChildrenCacheListener() {

                @Override
                public void childEvent(CuratorFramework client, PathChildrenCacheEvent event) throws Exception {
                    switch(event.getType()) {
                        case CHILD_ADDED:
                            processTokenAddOrUpdate(event.getData());
                            break;
                        case CHILD_UPDATED:
                            processTokenAddOrUpdate(event.getData());
                            break;
                        case CHILD_REMOVED:
                            processTokenRemoved(event.getData());
                            break;
                        default:
                            break;
                    }
                }
            }, listenerThreadPool);
            loadFromZKCache(true);
        }
    } catch (Exception e) {
        throw new IOException("Could not start PathChildrenCache for tokens", e);
    }
    super.startThreads();
}

Example 2 with EnsurePath

use of org.apache.curator.utils.EnsurePath in project flink by apache.

the class ZooKeeperCompletedCheckpointStoreTest method testCheckpointRecovery.

/**
	 * Tests that the completed checkpoint store can retrieve all checkpoints stored in ZooKeeper
	 * and ignores those which cannot be retrieved via their state handles.
	 */
@Test
public void testCheckpointRecovery() throws Exception {
    final List<Tuple2<RetrievableStateHandle<CompletedCheckpoint>, String>> checkpointsInZooKeeper = new ArrayList<>(4);
    final CompletedCheckpoint completedCheckpoint1 = mock(CompletedCheckpoint.class);
    when(completedCheckpoint1.getCheckpointID()).thenReturn(1L);
    final CompletedCheckpoint completedCheckpoint2 = mock(CompletedCheckpoint.class);
    when(completedCheckpoint2.getCheckpointID()).thenReturn(2L);
    final Collection<Long> expectedCheckpointIds = new HashSet<>(2);
    expectedCheckpointIds.add(1L);
    expectedCheckpointIds.add(2L);
    final RetrievableStateHandle<CompletedCheckpoint> failingRetrievableStateHandle = mock(RetrievableStateHandle.class);
    when(failingRetrievableStateHandle.retrieveState()).thenThrow(new Exception("Test exception"));
    final RetrievableStateHandle<CompletedCheckpoint> retrievableStateHandle1 = mock(RetrievableStateHandle.class);
    when(retrievableStateHandle1.retrieveState()).thenReturn(completedCheckpoint1);
    final RetrievableStateHandle<CompletedCheckpoint> retrievableStateHandle2 = mock(RetrievableStateHandle.class);
    when(retrievableStateHandle2.retrieveState()).thenReturn(completedCheckpoint2);
    checkpointsInZooKeeper.add(Tuple2.of(retrievableStateHandle1, "/foobar1"));
    checkpointsInZooKeeper.add(Tuple2.of(failingRetrievableStateHandle, "/failing1"));
    checkpointsInZooKeeper.add(Tuple2.of(retrievableStateHandle2, "/foobar2"));
    checkpointsInZooKeeper.add(Tuple2.of(failingRetrievableStateHandle, "/failing2"));
    final CuratorFramework client = mock(CuratorFramework.class, Mockito.RETURNS_DEEP_STUBS);
    final RetrievableStateStorageHelper<CompletedCheckpoint> storageHelperMock = mock(RetrievableStateStorageHelper.class);
    ZooKeeperStateHandleStore<CompletedCheckpoint> zooKeeperStateHandleStoreMock = spy(new ZooKeeperStateHandleStore<>(client, storageHelperMock, Executors.directExecutor()));
    whenNew(ZooKeeperStateHandleStore.class).withAnyArguments().thenReturn(zooKeeperStateHandleStoreMock);
    doReturn(checkpointsInZooKeeper).when(zooKeeperStateHandleStoreMock).getAllSortedByName();
    final int numCheckpointsToRetain = 1;
    // Mocking for the delete operation on the CuratorFramework client
    // It assures that the callback is executed synchronously
    final EnsurePath ensurePathMock = mock(EnsurePath.class);
    final CuratorEvent curatorEventMock = mock(CuratorEvent.class);
    when(curatorEventMock.getType()).thenReturn(CuratorEventType.DELETE);
    when(curatorEventMock.getResultCode()).thenReturn(0);
    when(client.newNamespaceAwareEnsurePath(anyString())).thenReturn(ensurePathMock);
    when(client.delete().deletingChildrenIfNeeded().inBackground(any(BackgroundCallback.class), any(Executor.class))).thenAnswer(new Answer<Pathable<Void>>() {

        @Override
        public Pathable<Void> answer(InvocationOnMock invocation) throws Throwable {
            final BackgroundCallback callback = (BackgroundCallback) invocation.getArguments()[0];
            Pathable<Void> result = mock(Pathable.class);
            when(result.forPath(anyString())).thenAnswer(new Answer<Void>() {

                @Override
                public Void answer(InvocationOnMock invocation) throws Throwable {
                    callback.processResult(client, curatorEventMock);
                    return null;
                }
            });
            return result;
        }
    });
    final String checkpointsPath = "foobar";
    final RetrievableStateStorageHelper<CompletedCheckpoint> stateSotrage = mock(RetrievableStateStorageHelper.class);
    ZooKeeperCompletedCheckpointStore zooKeeperCompletedCheckpointStore = new ZooKeeperCompletedCheckpointStore(numCheckpointsToRetain, client, checkpointsPath, stateSotrage, Executors.directExecutor());
    zooKeeperCompletedCheckpointStore.recover();
    CompletedCheckpoint latestCompletedCheckpoint = zooKeeperCompletedCheckpointStore.getLatestCheckpoint();
    // check that we return the latest retrievable checkpoint
    // this should remove the latest checkpoint because it is broken
    assertEquals(completedCheckpoint2.getCheckpointID(), latestCompletedCheckpoint.getCheckpointID());
    // this should remove the second broken checkpoint because we're iterating over all checkpoints
    List<CompletedCheckpoint> completedCheckpoints = zooKeeperCompletedCheckpointStore.getAllCheckpoints();
    Collection<Long> actualCheckpointIds = new HashSet<>(completedCheckpoints.size());
    for (CompletedCheckpoint completedCheckpoint : completedCheckpoints) {
        actualCheckpointIds.add(completedCheckpoint.getCheckpointID());
    }
    assertEquals(expectedCheckpointIds, actualCheckpointIds);
    // check that we did not discard any of the state handles which were retrieved
    verify(retrievableStateHandle1, never()).discardState();
    verify(retrievableStateHandle2, never()).discardState();
    // check that we have discarded the state handles which could not be retrieved
    verify(failingRetrievableStateHandle, times(2)).discardState();
}