Examples with PathChildrenCacheListener org.apache.curator.framework.recipes.cache.PathChildrenCacheListener used on opensource projects

Example 1 with PathChildrenCacheListener

use of org.apache.curator.framework.recipes.cache.PathChildrenCacheListener in project hadoop by apache.

the class ZKDelegationTokenSecretManager method startThreads.

@Override
public void startThreads() throws IOException {
    if (!isExternalClient) {
        try {
            zkClient.start();
        } catch (Exception e) {
            throw new IOException("Could not start Curator Framework", e);
        }
    } else {
        // If namespace parents are implicitly created, they won't have ACLs.
        // So, let's explicitly create them.
        CuratorFramework nullNsFw = zkClient.usingNamespace(null);
        EnsurePath ensureNs = nullNsFw.newNamespaceAwareEnsurePath("/" + zkClient.getNamespace());
        try {
            ensureNs.ensure(nullNsFw.getZookeeperClient());
        } catch (Exception e) {
            throw new IOException("Could not create namespace", e);
        }
    }
    listenerThreadPool = Executors.newSingleThreadExecutor();
    try {
        delTokSeqCounter = new SharedCount(zkClient, ZK_DTSM_SEQNUM_ROOT, 0);
        if (delTokSeqCounter != null) {
            delTokSeqCounter.start();
        }
    } catch (Exception e) {
        throw new IOException("Could not start Sequence Counter", e);
    }
    try {
        keyIdSeqCounter = new SharedCount(zkClient, ZK_DTSM_KEYID_ROOT, 0);
        if (keyIdSeqCounter != null) {
            keyIdSeqCounter.start();
        }
    } catch (Exception e) {
        throw new IOException("Could not start KeyId Counter", e);
    }
    try {
        createPersistentNode(ZK_DTSM_MASTER_KEY_ROOT);
        createPersistentNode(ZK_DTSM_TOKENS_ROOT);
    } catch (Exception e) {
        throw new RuntimeException("Could not create ZK paths");
    }
    try {
        keyCache = new PathChildrenCache(zkClient, ZK_DTSM_MASTER_KEY_ROOT, true);
        if (keyCache != null) {
            keyCache.start(StartMode.BUILD_INITIAL_CACHE);
            keyCache.getListenable().addListener(new PathChildrenCacheListener() {

                @Override
                public void childEvent(CuratorFramework client, PathChildrenCacheEvent event) throws Exception {
                    switch(event.getType()) {
                        case CHILD_ADDED:
                            processKeyAddOrUpdate(event.getData().getData());
                            break;
                        case CHILD_UPDATED:
                            processKeyAddOrUpdate(event.getData().getData());
                            break;
                        case CHILD_REMOVED:
                            processKeyRemoved(event.getData().getPath());
                            break;
                        default:
                            break;
                    }
                }
            }, listenerThreadPool);
            loadFromZKCache(false);
        }
    } catch (Exception e) {
        throw new IOException("Could not start PathChildrenCache for keys", e);
    }
    try {
        tokenCache = new PathChildrenCache(zkClient, ZK_DTSM_TOKENS_ROOT, true);
        if (tokenCache != null) {
            tokenCache.start(StartMode.BUILD_INITIAL_CACHE);
            tokenCache.getListenable().addListener(new PathChildrenCacheListener() {

                @Override
                public void childEvent(CuratorFramework client, PathChildrenCacheEvent event) throws Exception {
                    switch(event.getType()) {
                        case CHILD_ADDED:
                            processTokenAddOrUpdate(event.getData());
                            break;
                        case CHILD_UPDATED:
                            processTokenAddOrUpdate(event.getData());
                            break;
                        case CHILD_REMOVED:
                            processTokenRemoved(event.getData());
                            break;
                        default:
                            break;
                    }
                }
            }, listenerThreadPool);
            loadFromZKCache(true);
        }
    } catch (Exception e) {
        throw new IOException("Could not start PathChildrenCache for tokens", e);
    }
    super.startThreads();
}

Example 2 with PathChildrenCacheListener

use of org.apache.curator.framework.recipes.cache.PathChildrenCacheListener in project druid by druid-io.

the class DruidCoordinatorTest method testCoordinatorRun.

@Test(timeout = 60_000L)
public void testCoordinatorRun() throws Exception {
    String dataSource = "dataSource1";
    String tier = "hot";
    // Setup MetadataRuleManager
    Rule foreverLoadRule = new ForeverLoadRule(ImmutableMap.of(tier, 2));
    EasyMock.expect(metadataRuleManager.getRulesWithDefault(EasyMock.anyString())).andReturn(ImmutableList.of(foreverLoadRule)).atLeastOnce();
    metadataRuleManager.stop();
    EasyMock.expectLastCall().once();
    EasyMock.replay(metadataRuleManager);
    // Setup MetadataSegmentManager
    DruidDataSource[] druidDataSources = { new DruidDataSource(dataSource, Collections.<String, String>emptyMap()) };
    final DataSegment dataSegment = new DataSegment(dataSource, new Interval("2010-01-01/P1D"), "v1", null, null, null, null, 0x9, 0);
    druidDataSources[0].addSegment("0", dataSegment);
    EasyMock.expect(databaseSegmentManager.isStarted()).andReturn(true).anyTimes();
    EasyMock.expect(databaseSegmentManager.getInventory()).andReturn(ImmutableList.of(druidDataSources[0])).atLeastOnce();
    EasyMock.replay(databaseSegmentManager);
    ImmutableDruidDataSource immutableDruidDataSource = EasyMock.createNiceMock(ImmutableDruidDataSource.class);
    EasyMock.expect(immutableDruidDataSource.getSegments()).andReturn(ImmutableSet.of(dataSegment)).atLeastOnce();
    EasyMock.replay(immutableDruidDataSource);
    // Setup ServerInventoryView
    druidServer = new DruidServer("server1", "localhost", 5L, "historical", tier, 0);
    loadManagementPeons.put("server1", loadQueuePeon);
    EasyMock.expect(serverInventoryView.getInventory()).andReturn(ImmutableList.of(druidServer)).atLeastOnce();
    EasyMock.expect(serverInventoryView.isStarted()).andReturn(true).anyTimes();
    EasyMock.replay(serverInventoryView);
    coordinator.start();
    // Wait for this coordinator to become leader
    leaderAnnouncerLatch.await();
    // This coordinator should be leader by now
    Assert.assertTrue(coordinator.isLeader());
    Assert.assertEquals(druidNode.getHostAndPort(), coordinator.getCurrentLeader());
    final CountDownLatch assignSegmentLatch = new CountDownLatch(1);
    pathChildrenCache.getListenable().addListener(new PathChildrenCacheListener() {

        @Override
        public void childEvent(CuratorFramework curatorFramework, PathChildrenCacheEvent pathChildrenCacheEvent) throws Exception {
            if (pathChildrenCacheEvent.getType().equals(PathChildrenCacheEvent.Type.CHILD_ADDED)) {
                //Coordinator should try to assign segment to druidServer historical
                //Simulate historical loading segment
                druidServer.addDataSegment(dataSegment.getIdentifier(), dataSegment);
                assignSegmentLatch.countDown();
            }
        }
    });
    pathChildrenCache.start();
    assignSegmentLatch.await();
    Assert.assertEquals(ImmutableMap.of(dataSource, 100.0), coordinator.getLoadStatus());
    curator.delete().guaranteed().forPath(ZKPaths.makePath(LOADPATH, dataSegment.getIdentifier()));
    // Wait for coordinator thread to run so that replication status is updated
    while (coordinator.getSegmentAvailability().snapshot().get(dataSource) != 0) {
        Thread.sleep(50);
    }
    Map segmentAvailability = coordinator.getSegmentAvailability().snapshot();
    Assert.assertEquals(1, segmentAvailability.size());
    Assert.assertEquals(0L, segmentAvailability.get(dataSource));
    while (coordinator.getLoadPendingDatasources().get(dataSource).get() > 0) {
        Thread.sleep(50);
    }
    // wait historical data to be updated
    long startMillis = System.currentTimeMillis();
    long coordinatorRunPeriodMillis = druidCoordinatorConfig.getCoordinatorPeriod().getMillis();
    while (System.currentTimeMillis() - startMillis < coordinatorRunPeriodMillis) {
        Thread.sleep(100);
    }
    Map<String, CountingMap<String>> replicationStatus = coordinator.getReplicationStatus();
    Assert.assertNotNull(replicationStatus);
    Assert.assertEquals(1, replicationStatus.entrySet().size());
    CountingMap<String> dataSourceMap = replicationStatus.get(tier);
    Assert.assertNotNull(dataSourceMap);
    Assert.assertEquals(1, dataSourceMap.size());
    Assert.assertNotNull(dataSourceMap.get(dataSource));
    // The load rules asks for 2 replicas, therefore 1 replica should still be pending
    while (dataSourceMap.get(dataSource).get() != 1L) {
        Thread.sleep(50);
    }
    coordinator.stop();
    leaderUnannouncerLatch.await();
    Assert.assertFalse(coordinator.isLeader());
    Assert.assertNull(coordinator.getCurrentLeader());
    EasyMock.verify(serverInventoryView);
    EasyMock.verify(metadataRuleManager);
}

Example 3 with PathChildrenCacheListener

use of org.apache.curator.framework.recipes.cache.PathChildrenCacheListener in project druid by druid-io.

the class LoadQueuePeonTest method testFailAssign.

@Test
public void testFailAssign() throws Exception {
    final DataSegment segment = dataSegmentWithInterval("2014-10-22T00:00:00Z/P1D");
    final CountDownLatch loadRequestSignal = new CountDownLatch(1);
    final CountDownLatch segmentLoadedSignal = new CountDownLatch(1);
    loadQueuePeon = new LoadQueuePeon(curator, LOAD_QUEUE_PATH, jsonMapper, Execs.scheduledSingleThreaded("test_load_queue_peon_scheduled-%d"), Execs.singleThreaded("test_load_queue_peon-%d"), // set time-out to 1 ms so that LoadQueuePeon will fail the assignment quickly
    new TestDruidCoordinatorConfig(null, null, null, new Duration(1), null, null, 10, null, false, false, new Duration("PT1s")));
    loadQueuePeon.start();
    loadQueueCache.getListenable().addListener(new PathChildrenCacheListener() {

        @Override
        public void childEvent(CuratorFramework client, PathChildrenCacheEvent event) throws Exception {
            if (event.getType() == PathChildrenCacheEvent.Type.CHILD_ADDED) {
                loadRequestSignal.countDown();
            }
        }
    });
    loadQueueCache.start();
    loadQueuePeon.loadSegment(segment, new LoadPeonCallback() {

        @Override
        public void execute() {
            segmentLoadedSignal.countDown();
        }
    });
    String loadRequestPath = ZKPaths.makePath(LOAD_QUEUE_PATH, segment.getIdentifier());
    Assert.assertTrue(timing.forWaiting().awaitLatch(loadRequestSignal));
    Assert.assertNotNull(curator.checkExists().forPath(loadRequestPath));
    Assert.assertEquals(segment, ((SegmentChangeRequestLoad) jsonMapper.readValue(curator.getData().decompressed().forPath(loadRequestPath), DataSegmentChangeRequest.class)).getSegment());
    // don't simulate completion of load request here
    Assert.assertTrue(timing.forWaiting().awaitLatch(segmentLoadedSignal));
    Assert.assertEquals(0, loadQueuePeon.getSegmentsToLoad().size());
    Assert.assertEquals(0L, loadQueuePeon.getLoadQueueSize());
}

Example 4 with PathChildrenCacheListener

use of org.apache.curator.framework.recipes.cache.PathChildrenCacheListener in project druid by druid-io.

the class RemoteTaskRunner method start.

@Override
@LifecycleStart
public void start() {
    try {
        if (started) {
            return;
        }
        final MutableInt waitingFor = new MutableInt(1);
        final Object waitingForMonitor = new Object();
        // Add listener for creation/deletion of workers
        workerPathCache.getListenable().addListener(new PathChildrenCacheListener() {

            @Override
            public void childEvent(CuratorFramework client, final PathChildrenCacheEvent event) throws Exception {
                final Worker worker;
                switch(event.getType()) {
                    case CHILD_ADDED:
                        worker = jsonMapper.readValue(event.getData().getData(), Worker.class);
                        synchronized (waitingForMonitor) {
                            waitingFor.increment();
                        }
                        Futures.addCallback(addWorker(worker), new FutureCallback<ZkWorker>() {

                            @Override
                            public void onSuccess(ZkWorker zkWorker) {
                                synchronized (waitingForMonitor) {
                                    waitingFor.decrement();
                                    waitingForMonitor.notifyAll();
                                }
                            }

                            @Override
                            public void onFailure(Throwable throwable) {
                                synchronized (waitingForMonitor) {
                                    waitingFor.decrement();
                                    waitingForMonitor.notifyAll();
                                }
                            }
                        });
                        break;
                    case CHILD_UPDATED:
                        worker = jsonMapper.readValue(event.getData().getData(), Worker.class);
                        updateWorker(worker);
                        break;
                    case CHILD_REMOVED:
                        worker = jsonMapper.readValue(event.getData().getData(), Worker.class);
                        removeWorker(worker);
                        break;
                    case INITIALIZED:
                        // Schedule cleanup for task status of the workers that might have disconnected while overlord was not running
                        List<String> workers;
                        try {
                            workers = cf.getChildren().forPath(indexerZkConfig.getStatusPath());
                        } catch (KeeperException.NoNodeException e) {
                            // statusPath doesn't exist yet; can occur if no middleManagers have started.
                            workers = ImmutableList.of();
                        }
                        for (String workerId : workers) {
                            final String workerAnnouncePath = JOINER.join(indexerZkConfig.getAnnouncementsPath(), workerId);
                            final String workerStatusPath = JOINER.join(indexerZkConfig.getStatusPath(), workerId);
                            if (!zkWorkers.containsKey(workerId) && cf.checkExists().forPath(workerAnnouncePath) == null) {
                                try {
                                    scheduleTasksCleanupForWorker(workerId, cf.getChildren().forPath(workerStatusPath));
                                } catch (Exception e) {
                                    log.warn(e, "Could not schedule cleanup for worker[%s] during startup (maybe someone removed the status znode[%s]?). Skipping.", workerId, workerStatusPath);
                                }
                            }
                        }
                        synchronized (waitingForMonitor) {
                            waitingFor.decrement();
                            waitingForMonitor.notifyAll();
                        }
                    default:
                        break;
                }
            }
        });
        workerPathCache.start(PathChildrenCache.StartMode.POST_INITIALIZED_EVENT);
        synchronized (waitingForMonitor) {
            while (waitingFor.intValue() > 0) {
                waitingForMonitor.wait();
            }
        }
        scheduleBlackListedNodesCleanUp();
        resourceManagement.startManagement(this);
        started = true;
    } catch (Exception e) {
        throw Throwables.propagate(e);
    }
}

Example 5 with PathChildrenCacheListener

use of org.apache.curator.framework.recipes.cache.PathChildrenCacheListener in project archaius by Netflix.

the class ZooKeeperConfigurationSource method start.

/** 
     * Adds a listener to the pathChildrenCache, initializes the cache, then starts the cache-management background thread
     * 
     * @throws Exception
     */
public void start() throws Exception {
    // create the watcher for future configuration updatess
    pathChildrenCache.getListenable().addListener(new PathChildrenCacheListener() {

        public void childEvent(CuratorFramework aClient, PathChildrenCacheEvent event) throws Exception {
            Type eventType = event.getType();
            ChildData data = event.getData();
            String path = null;
            if (data != null) {
                path = data.getPath();
                // scrub configRootPath out of the key name
                String key = removeRootPath(path);
                byte[] value = data.getData();
                String stringValue = new String(value, charset);
                logger.debug("received update to pathName [{}], eventType [{}]", path, eventType);
                logger.debug("key [{}], and value [{}]", key, stringValue);
                // fire event to all listeners
                Map<String, Object> added = null;
                Map<String, Object> changed = null;
                Map<String, Object> deleted = null;
                if (eventType == Type.CHILD_ADDED) {
                    added = new HashMap<String, Object>(1);
                    added.put(key, stringValue);
                } else if (eventType == Type.CHILD_UPDATED) {
                    changed = new HashMap<String, Object>(1);
                    changed.put(key, stringValue);
                } else if (eventType == Type.CHILD_REMOVED) {
                    deleted = new HashMap<String, Object>(1);
                    deleted.put(key, stringValue);
                }
                WatchedUpdateResult result = WatchedUpdateResult.createIncremental(added, changed, deleted);
                fireEvent(result);
            }
        }
    });
    // passing true to trigger an initial rebuild upon starting.  (blocking call)
    pathChildrenCache.start(true);
}