Examples with PathChildrenCacheEvent org.apache.curator.framework.recipes.cache.PathChildrenCacheEvent used on opensource projects

Example 51 with PathChildrenCacheEvent

use of org.apache.curator.framework.recipes.cache.PathChildrenCacheEvent in project druid by apache.

the class LoadQueuePeonTest method testFailAssignForLoadDropTimeout.

@Test
public void testFailAssignForLoadDropTimeout() throws Exception {
    final DataSegment segment = dataSegmentWithInterval("2014-10-22T00:00:00Z/P1D");
    final CountDownLatch loadRequestSignal = new CountDownLatch(1);
    final CountDownLatch segmentLoadedSignal = new CountDownLatch(1);
    final CountDownLatch delayedSegmentLoadedSignal = new CountDownLatch(2);
    final CountDownLatch loadRequestRemoveSignal = new CountDownLatch(1);
    loadQueuePeon = new CuratorLoadQueuePeon(curator, LOAD_QUEUE_PATH, jsonMapper, Execs.scheduledSingleThreaded("test_load_queue_peon_scheduled-%d"), Execs.singleThreaded("test_load_queue_peon-%d"), // set time-out to 1 ms so that LoadQueuePeon will fail the assignment quickly
    new TestDruidCoordinatorConfig(null, null, null, null, new Duration(1), null, null, null, null, null, null, null, null, null, null, null, 10, new Duration("PT1s"), false));
    loadQueuePeon.start();
    loadQueueCache.getListenable().addListener(new PathChildrenCacheListener() {

        @Override
        public void childEvent(CuratorFramework client, PathChildrenCacheEvent event) {
            switch(event.getType()) {
                case CHILD_ADDED:
                    loadRequestSignal.countDown();
                    break;
                case CHILD_REMOVED:
                    loadRequestRemoveSignal.countDown();
                    break;
                default:
            }
        }
    });
    loadQueueCache.start();
    loadQueuePeon.loadSegment(segment, new LoadPeonCallback() {

        @Override
        public void execute() {
            segmentLoadedSignal.countDown();
            delayedSegmentLoadedSignal.countDown();
        }
    });
    String loadRequestPath = ZKPaths.makePath(LOAD_QUEUE_PATH, segment.getId().toString());
    Assert.assertTrue(timing.forWaiting().awaitLatch(loadRequestSignal));
    Assert.assertNotNull(curator.checkExists().forPath(loadRequestPath));
    Assert.assertEquals(segment, ((SegmentChangeRequestLoad) jsonMapper.readValue(curator.getData().decompressed().forPath(loadRequestPath), DataSegmentChangeRequest.class)).getSegment());
    // simulate incompletion of load request since request has timed out
    Assert.assertTrue(timing.forWaiting().awaitLatch(segmentLoadedSignal));
    Assert.assertEquals(1, loadQueuePeon.getSegmentsToLoad().size());
    Assert.assertEquals(1200L, loadQueuePeon.getLoadQueueSize());
    Assert.assertEquals(1, loadQueuePeon.getTimedOutSegments().size());
    // simulate completion of load request by historical after time out on coordinator
    curator.delete().guaranteed().forPath(loadRequestPath);
    Assert.assertTrue(timing.forWaiting().awaitLatch(delayedSegmentLoadedSignal));
    Assert.assertTrue(timing.forWaiting().awaitLatch(loadRequestRemoveSignal));
    Assert.assertEquals(0, loadQueuePeon.getSegmentsToLoad().size());
    Assert.assertEquals(0L, loadQueuePeon.getLoadQueueSize());
    Assert.assertEquals(0, loadQueuePeon.getTimedOutSegments().size());
}

Example 52 with PathChildrenCacheEvent

use of org.apache.curator.framework.recipes.cache.PathChildrenCacheEvent in project druid by apache.

the class Announcer method announce.

/**
 * Announces the provided bytes at the given path.  Announcement means that it will create an ephemeral node
 * and monitor it to make sure that it always exists until it is unannounced or this object is closed.
 *
 * @param path                  The path to announce at
 * @param bytes                 The payload to announce
 * @param removeParentIfCreated remove parent of "path" if we had created that parent
 */
public void announce(String path, byte[] bytes, boolean removeParentIfCreated) {
    synchronized (toAnnounce) {
        if (!started) {
            toAnnounce.add(new Announceable(path, bytes, removeParentIfCreated));
            return;
        }
    }
    final ZKPaths.PathAndNode pathAndNode = ZKPaths.getPathAndNode(path);
    final String parentPath = pathAndNode.getPath();
    boolean buildParentPath = false;
    ConcurrentMap<String, byte[]> subPaths = announcements.get(parentPath);
    if (subPaths == null) {
        try {
            if (curator.checkExists().forPath(parentPath) == null) {
                buildParentPath = true;
            }
        } catch (Exception e) {
            log.debug(e, "Problem checking if the parent existed, ignoring.");
        }
        // I don't have a watcher on this path yet, create a Map and start watching.
        announcements.putIfAbsent(parentPath, new ConcurrentHashMap<>());
        // Guaranteed to be non-null, but might be a map put in there by another thread.
        final ConcurrentMap<String, byte[]> finalSubPaths = announcements.get(parentPath);
        // Synchronize to make sure that I only create a listener once.
        synchronized (finalSubPaths) {
            if (!listeners.containsKey(parentPath)) {
                final PathChildrenCache cache = factory.make(curator, parentPath);
                cache.getListenable().addListener(new PathChildrenCacheListener() {

                    private final AtomicReference<Set<String>> pathsLost = new AtomicReference<Set<String>>(null);

                    @Override
                    public void childEvent(CuratorFramework client, PathChildrenCacheEvent event) throws Exception {
                        // NOTE: ZooKeeper does not guarantee that we will get every event, and thus PathChildrenCache doesn't
                        // as well. If one of the below events are missed, Announcer might not work properly.
                        log.debug("Path[%s] got event[%s]", parentPath, event);
                        switch(event.getType()) {
                            case CHILD_REMOVED:
                                final ChildData child = event.getData();
                                final ZKPaths.PathAndNode childPath = ZKPaths.getPathAndNode(child.getPath());
                                final byte[] value = finalSubPaths.get(childPath.getNode());
                                if (value != null) {
                                    log.info("Node[%s] dropped, reinstating.", child.getPath());
                                    createAnnouncement(child.getPath(), value);
                                }
                                break;
                            case CONNECTION_LOST:
                                // Lost connection, which means session is broken, take inventory of what has been seen.
                                // This is to protect from a race condition in which the ephemeral node could have been
                                // created but not actually seen by the PathChildrenCache, which means that it won't know
                                // that it disappeared and thus will not generate a CHILD_REMOVED event for us.  Under normal
                                // circumstances, this can only happen upon connection loss; but technically if you have
                                // an adversary in the system, they could also delete the ephemeral node before the cache sees
                                // it.  This does not protect from that case, so don't have adversaries.
                                Set<String> pathsToReinstate = new HashSet<>();
                                for (String node : finalSubPaths.keySet()) {
                                    String path = ZKPaths.makePath(parentPath, node);
                                    log.info("Node[%s] is added to reinstate.", path);
                                    pathsToReinstate.add(path);
                                }
                                if (!pathsToReinstate.isEmpty() && !pathsLost.compareAndSet(null, pathsToReinstate)) {
                                    log.info("Already had a pathsLost set!?[%s]", parentPath);
                                }
                                break;
                            case CONNECTION_RECONNECTED:
                                final Set<String> thePathsLost = pathsLost.getAndSet(null);
                                if (thePathsLost != null) {
                                    for (String path : thePathsLost) {
                                        log.info("Reinstating [%s]", path);
                                        final ZKPaths.PathAndNode split = ZKPaths.getPathAndNode(path);
                                        createAnnouncement(path, announcements.get(split.getPath()).get(split.getNode()));
                                    }
                                }
                                break;
                            case CHILD_ADDED:
                                if (addedChildren != null) {
                                    addedChildren.add(event.getData().getPath());
                                }
                            // fall through
                            case INITIALIZED:
                            case CHILD_UPDATED:
                            case CONNECTION_SUSPENDED:
                        }
                    }
                });
                synchronized (toAnnounce) {
                    if (started) {
                        if (buildParentPath) {
                            createPath(parentPath, removeParentIfCreated);
                        }
                        startCache(cache);
                        listeners.put(parentPath, cache);
                    }
                }
            }
        }
        subPaths = finalSubPaths;
    }
    boolean created = false;
    synchronized (toAnnounce) {
        if (started) {
            byte[] oldBytes = subPaths.putIfAbsent(pathAndNode.getNode(), bytes);
            if (oldBytes == null) {
                created = true;
            } else if (!Arrays.equals(oldBytes, bytes)) {
                throw new IAE("Cannot reannounce different values under the same path");
            }
        }
    }
    if (created) {
        try {
            createAnnouncement(path, bytes);
        } catch (Exception e) {
            throw new RuntimeException(e);
        }
    }
}

Example 53 with PathChildrenCacheEvent

use of org.apache.curator.framework.recipes.cache.PathChildrenCacheEvent in project druid by druid-io.

the class DruidCoordinatorTest method createCountDownLatchAndSetPathChildrenCacheListenerWithLatch.

private CountDownLatch createCountDownLatchAndSetPathChildrenCacheListenerWithLatch(int latchCount, PathChildrenCache pathChildrenCache, Map<String, DataSegment> segments, DruidServer server) {
    final CountDownLatch countDownLatch = new CountDownLatch(latchCount);
    pathChildrenCache.getListenable().addListener((CuratorFramework client, PathChildrenCacheEvent event) -> {
        if (CuratorUtils.isChildAdded(event)) {
            DataSegment segment = findSegmentRelatedToCuratorEvent(segments, event);
            if (segment != null && server.getSegment(segment.getId()) == null) {
                if (countDownLatch.getCount() > 0) {
                    server.addDataSegment(segment);
                    curator.delete().guaranteed().forPath(event.getData().getPath());
                    countDownLatch.countDown();
                } else {
                    Assert.fail("The segment path " + event.getData().getPath() + " is not expected");
                }
            }
        }
    });
    return countDownLatch;
}

Example 54 with PathChildrenCacheEvent

use of org.apache.curator.framework.recipes.cache.PathChildrenCacheEvent in project druid by druid-io.

the class LoadQueuePeonTest method testFailAssignForLoadDropTimeout.

@Test
public void testFailAssignForLoadDropTimeout() throws Exception {
    final DataSegment segment = dataSegmentWithInterval("2014-10-22T00:00:00Z/P1D");
    final CountDownLatch loadRequestSignal = new CountDownLatch(1);
    final CountDownLatch segmentLoadedSignal = new CountDownLatch(1);
    final CountDownLatch delayedSegmentLoadedSignal = new CountDownLatch(2);
    final CountDownLatch loadRequestRemoveSignal = new CountDownLatch(1);
    loadQueuePeon = new CuratorLoadQueuePeon(curator, LOAD_QUEUE_PATH, jsonMapper, Execs.scheduledSingleThreaded("test_load_queue_peon_scheduled-%d"), Execs.singleThreaded("test_load_queue_peon-%d"), // set time-out to 1 ms so that LoadQueuePeon will fail the assignment quickly
    new TestDruidCoordinatorConfig(null, null, null, null, new Duration(1), null, null, null, null, null, null, null, null, null, null, null, 10, new Duration("PT1s"), false));
    loadQueuePeon.start();
    loadQueueCache.getListenable().addListener(new PathChildrenCacheListener() {

        @Override
        public void childEvent(CuratorFramework client, PathChildrenCacheEvent event) {
            switch(event.getType()) {
                case CHILD_ADDED:
                    loadRequestSignal.countDown();
                    break;
                case CHILD_REMOVED:
                    loadRequestRemoveSignal.countDown();
                    break;
                default:
            }
        }
    });
    loadQueueCache.start();
    loadQueuePeon.loadSegment(segment, new LoadPeonCallback() {

        @Override
        public void execute() {
            segmentLoadedSignal.countDown();
            delayedSegmentLoadedSignal.countDown();
        }
    });
    String loadRequestPath = ZKPaths.makePath(LOAD_QUEUE_PATH, segment.getId().toString());
    Assert.assertTrue(timing.forWaiting().awaitLatch(loadRequestSignal));
    Assert.assertNotNull(curator.checkExists().forPath(loadRequestPath));
    Assert.assertEquals(segment, ((SegmentChangeRequestLoad) jsonMapper.readValue(curator.getData().decompressed().forPath(loadRequestPath), DataSegmentChangeRequest.class)).getSegment());
    // simulate incompletion of load request since request has timed out
    Assert.assertTrue(timing.forWaiting().awaitLatch(segmentLoadedSignal));
    Assert.assertEquals(1, loadQueuePeon.getSegmentsToLoad().size());
    Assert.assertEquals(1200L, loadQueuePeon.getLoadQueueSize());
    Assert.assertEquals(1, loadQueuePeon.getTimedOutSegments().size());
    // simulate completion of load request by historical after time out on coordinator
    curator.delete().guaranteed().forPath(loadRequestPath);
    Assert.assertTrue(timing.forWaiting().awaitLatch(delayedSegmentLoadedSignal));
    Assert.assertTrue(timing.forWaiting().awaitLatch(loadRequestRemoveSignal));
    Assert.assertEquals(0, loadQueuePeon.getSegmentsToLoad().size());
    Assert.assertEquals(0L, loadQueuePeon.getLoadQueueSize());
    Assert.assertEquals(0, loadQueuePeon.getTimedOutSegments().size());
}

Example 55 with PathChildrenCacheEvent

use of org.apache.curator.framework.recipes.cache.PathChildrenCacheEvent in project curator by apache.

the class TestPersistentTtlNode method testEventsOnParent.

@Test
public void testEventsOnParent() throws Exception {
    try (CuratorFramework client = CuratorFrameworkFactory.newClient(server.getConnectString(), new RetryOneTime(1))) {
        client.start();
        try (PersistentTtlNode node = new PersistentTtlNode(client, "/test", ttlMs, new byte[0])) {
            try (PathChildrenCache cache = new PathChildrenCache(client, "/", true)) {
                final Semaphore changes = new Semaphore(0);
                PathChildrenCacheListener listener = new PathChildrenCacheListener() {

                    @Override
                    public void childEvent(CuratorFramework client, PathChildrenCacheEvent event) throws Exception {
                        if ((event.getType() == PathChildrenCacheEvent.Type.CHILD_UPDATED) && "/test".equals(event.getData().getPath())) {
                            changes.release();
                        }
                    }
                };
                cache.getListenable().addListener(listener);
                node.start();
                assertTrue(node.waitForInitialCreate(timing.session(), TimeUnit.MILLISECONDS));
                cache.start(BUILD_INITIAL_CACHE);
                assertEquals(changes.availablePermits(), 0);
                timing.sleepABit();
                assertEquals(changes.availablePermits(), 0);
                client.setData().forPath("/test", "changed".getBytes());
                assertTrue(timing.acquireSemaphore(changes));
                timing.sleepABit();
                assertEquals(changes.availablePermits(), 0);
            }
        }
        timing.sleepABit();
        assertNull(client.checkExists().forPath("/test"));
    }
}