Examples with PathChildrenCache org.apache.flink.shaded.curator5.org.apache.curator.framework.recipes.cache.PathChildrenCache used on opensource projects

Example 71 with PathChildrenCache

use of org.apache.flink.shaded.curator5.org.apache.curator.framework.recipes.cache.PathChildrenCache in project druid by druid-io.

the class LoadQueuePeonTest method setUp.

@Before
public void setUp() throws Exception {
    setupServerAndCurator();
    curator.start();
    curator.blockUntilConnected();
    curator.create().creatingParentsIfNeeded().forPath(LOAD_QUEUE_PATH);
    loadQueueCache = new PathChildrenCache(curator, LOAD_QUEUE_PATH, true, true, Execs.singleThreaded("load_queue_cache-%d"));
}

Example 72 with PathChildrenCache

use of org.apache.flink.shaded.curator5.org.apache.curator.framework.recipes.cache.PathChildrenCache in project druid by druid-io.

the class CuratorDruidCoordinatorTest method setUp.

@Before
public void setUp() throws Exception {
    segmentsMetadataManager = EasyMock.createNiceMock(SegmentsMetadataManager.class);
    dataSourcesSnapshot = EasyMock.createNiceMock(DataSourcesSnapshot.class);
    coordinatorRuntimeParams = EasyMock.createNiceMock(DruidCoordinatorRuntimeParams.class);
    metadataRuleManager = EasyMock.createNiceMock(MetadataRuleManager.class);
    configManager = EasyMock.createNiceMock(JacksonConfigManager.class);
    EasyMock.expect(configManager.watch(EasyMock.eq(CoordinatorDynamicConfig.CONFIG_KEY), EasyMock.anyObject(Class.class), EasyMock.anyObject())).andReturn(new AtomicReference<>(CoordinatorDynamicConfig.builder().build())).anyTimes();
    EasyMock.expect(configManager.watch(EasyMock.eq(CoordinatorCompactionConfig.CONFIG_KEY), EasyMock.anyObject(Class.class), EasyMock.anyObject())).andReturn(new AtomicReference(CoordinatorCompactionConfig.empty())).anyTimes();
    EasyMock.replay(configManager);
    setupServerAndCurator();
    curator.start();
    curator.blockUntilConnected();
    curator.create().creatingParentsIfNeeded().forPath(SEGPATH);
    curator.create().creatingParentsIfNeeded().forPath(SOURCE_LOAD_PATH);
    curator.create().creatingParentsIfNeeded().forPath(DESTINATION_LOAD_PATH);
    objectMapper = new DefaultObjectMapper();
    druidCoordinatorConfig = new TestDruidCoordinatorConfig(new Duration(COORDINATOR_START_DELAY), new Duration(COORDINATOR_PERIOD), null, null, null, new Duration(COORDINATOR_PERIOD), null, null, null, null, null, null, null, null, null, null, 10, new Duration("PT0s"));
    sourceLoadQueueChildrenCache = new PathChildrenCache(curator, SOURCE_LOAD_PATH, true, true, Execs.singleThreaded("coordinator_test_path_children_cache_src-%d"));
    destinationLoadQueueChildrenCache = new PathChildrenCache(curator, DESTINATION_LOAD_PATH, true, true, Execs.singleThreaded("coordinator_test_path_children_cache_dest-%d"));
    sourceLoadQueuePeon = new CuratorLoadQueuePeon(curator, SOURCE_LOAD_PATH, objectMapper, peonExec, callbackExec, druidCoordinatorConfig);
    destinationLoadQueuePeon = new CuratorLoadQueuePeon(curator, DESTINATION_LOAD_PATH, objectMapper, peonExec, callbackExec, druidCoordinatorConfig);
    druidNode = new DruidNode("hey", "what", false, 1234, null, true, false);
    loadManagementPeons = new ConcurrentHashMap<>();
    scheduledExecutorFactory = (corePoolSize, nameFormat) -> Executors.newSingleThreadScheduledExecutor();
    leaderAnnouncerLatch = new CountDownLatch(1);
    leaderUnannouncerLatch = new CountDownLatch(1);
    coordinator = new DruidCoordinator(druidCoordinatorConfig, new ZkPathsConfig() {

        @Override
        public String getBase() {
            return "druid";
        }
    }, configManager, segmentsMetadataManager, baseView, metadataRuleManager, () -> curator, new NoopServiceEmitter(), scheduledExecutorFactory, null, null, new NoopServiceAnnouncer() {

        @Override
        public void announce(DruidNode node) {
            // count down when this coordinator becomes the leader
            leaderAnnouncerLatch.countDown();
        }

        @Override
        public void unannounce(DruidNode node) {
            leaderUnannouncerLatch.countDown();
        }
    }, druidNode, loadManagementPeons, null, null, new CoordinatorCustomDutyGroups(ImmutableSet.of()), new CostBalancerStrategyFactory(), EasyMock.createNiceMock(LookupCoordinatorManager.class), new TestDruidLeaderSelector(), null, ZkEnablementConfig.ENABLED);
}

Example 73 with PathChildrenCache

use of org.apache.flink.shaded.curator5.org.apache.curator.framework.recipes.cache.PathChildrenCache in project flink by apache.

the class ZooKeeperJobGraphsStoreITCase method createZooKeeperJobGraphStore.

@Nonnull
private JobGraphStore createZooKeeperJobGraphStore(String fullPath) throws Exception {
    final CuratorFramework client = ZooKeeper.getClient();
    // Ensure that the job graphs path exists
    client.newNamespaceAwareEnsurePath(fullPath).ensure(client.getZookeeperClient());
    // All operations will have the path as root
    CuratorFramework facade = client.usingNamespace(client.getNamespace() + fullPath);
    final ZooKeeperStateHandleStore<JobGraph> zooKeeperStateHandleStore = new ZooKeeperStateHandleStore<>(facade, localStateStorage);
    return new DefaultJobGraphStore<>(zooKeeperStateHandleStore, new ZooKeeperJobGraphStoreWatcher(new PathChildrenCache(facade, "/", false)), ZooKeeperJobGraphStoreUtil.INSTANCE);
}

Example 74 with PathChildrenCache

use of org.apache.flink.shaded.curator5.org.apache.curator.framework.recipes.cache.PathChildrenCache in project twister2 by DSC-SPIDAL.

the class ZKWorkerController method initialize.

/**
 * Initialize this ZKWorkerController
 * Connect to the ZK server
 * create a persistent znode for this worker
 *   set this WorkerInfo and its status in the body of that znode
 * create an ephemeral znode for this worker
 * create a cache for the persistent znode of the job for watching job scaling events
 *
 * initialState has to be either: WorkerState.STARTED or WorkerState.RESTARTED
 * <p>
 * The body of the persistent worker znode will be updated as the status of worker changes
 * from STARTED, COMPLETED,
 */
public void initialize(int restartCount1, long jsTime) throws Exception {
    this.restartCount = restartCount1;
    this.initialState = restartCount > 0 ? WorkerState.RESTARTED : WorkerState.STARTED;
    if (!(initialState == WorkerState.STARTED || initialState == WorkerState.RESTARTED)) {
        throw new Exception("initialState has to be either WorkerState.STARTED or " + "WorkerState.RESTARTED. Supplied value: " + initialState);
    }
    try {
        String zkServerAddresses = ZKContext.serverAddresses(config);
        int sessionTimeoutMs = FaultToleranceContext.sessionTimeout(config);
        client = ZKUtils.connectToServer(zkServerAddresses, sessionTimeoutMs);
        // we first create persistent znode for the worker
        if (initialState == WorkerState.STARTED) {
            if (JobZNodeManager.checkJstZNodeWaitIfNeeded(client, rootPath, jobID, jsTime)) {
                ZKPersStateManager.createWorkerPersState(client, rootPath, jobID, workerInfo);
            }
        } else {
            ZKPersStateManager.updateWorkerStatus(client, rootPath, jobID, workerInfo, restartCount, WorkerState.RESTARTED);
        }
        // if this worker is started with scaling up, or
        // if it is coming from failure
        // we need to handle past events
        numberOfPastEvents = ZKEventsManager.getNumberOfPastEvents(client, rootPath, jobID);
        pastEvents = new TreeMap<>(Collections.reverseOrder());
        int wID = workerInfo.getWorkerID();
        if (isRestarted()) {
            // delete ephemeral worker znode from previous run, if there is any
            ZKEphemStateManager.removeEphemZNode(client, rootPath, jobID, wID);
            // worker may have crashed during barrier operation
            if (ZKBarrierManager.existWorkerZNodeAtDefault(client, rootPath, jobID, wID)) {
                ZKBarrierManager.deleteWorkerZNodeFromDefault(client, rootPath, jobID, wID);
            }
            if (ZKBarrierManager.existWorkerZNodeAtInit(client, rootPath, jobID, wID)) {
                ZKBarrierManager.deleteWorkerZNodeFromInit(client, rootPath, jobID, wID);
            }
        }
        String eventsDir = ZKUtils.eventsDir(rootPath, jobID);
        eventsChildrenCache = new PathChildrenCache(client, eventsDir, true);
        addEventsChildrenCacheListener(eventsChildrenCache);
        eventsChildrenCache.start();
        // update numberOfWorkers from jobZnode
        // with scaling up/down, it may have been changed
        JobWithState jobWithState = JobZNodeManager.readJobZNode(client, rootPath, jobID);
        if (numberOfWorkers != jobWithState.getJob().getNumberOfWorkers()) {
            numberOfWorkers = jobWithState.getJob().getNumberOfWorkers();
            LOG.info("numberOfWorkers updated from persJobZnode as: " + numberOfWorkers);
        }
        createWorkerZnode();
        LOG.info("This worker: " + workerInfo.getWorkerID() + " initialized successfully.");
    } catch (Exception e) {
        LOG.log(Level.SEVERE, "Exception when initializing ZKWorkerController", e);
        throw e;
    }
}

Example 75 with PathChildrenCache

use of org.apache.flink.shaded.curator5.org.apache.curator.framework.recipes.cache.PathChildrenCache in project twister2 by DSC-SPIDAL.

the class ZKBarrierHandler method initialize.

/**
 * initialize ZKBarrierHandler,
 * create znode children caches for job master to watch barrier events
 */
public void initialize(JobMasterAPI.JobMasterState initialState) throws Twister2Exception {
    if (!(initialState == JobMasterAPI.JobMasterState.JM_STARTED || initialState == JobMasterAPI.JobMasterState.JM_RESTARTED)) {
        throw new Twister2Exception("initialState has to be either JobMasterState.JM_STARTED or " + "JobMasterState.JM_RESTARTED. Supplied value: " + initialState);
    }
    try {
        String zkServerAddresses = ZKContext.serverAddresses(config);
        int sessionTimeoutMs = FaultToleranceContext.sessionTimeout(config);
        client = ZKUtils.connectToServer(zkServerAddresses, sessionTimeoutMs);
        // with scaling up/down, it may have been changed
        if (initialState == JobMasterAPI.JobMasterState.JM_RESTARTED) {
            // do not get previous events on barriers
            // get current snapshots of both barriers at the restart
            String defaultBarrierDir = ZKUtils.defaultBarrierDir(rootPath, jobID);
            defaultBarrierCache = new PathChildrenCache(client, defaultBarrierDir, true);
            addBarrierChildrenCacheListener(defaultBarrierCache, JobMasterAPI.BarrierType.DEFAULT);
            defaultBarrierCache.start(PathChildrenCache.StartMode.BUILD_INITIAL_CACHE);
            TreeSet<Integer> existingWorkers = new TreeSet<>();
            long timeout = getInitialWorkersAtBarrier(defaultBarrierCache, existingWorkers);
            if (!existingWorkers.isEmpty()) {
                barrierMonitor.initDefaultAfterRestart(existingWorkers, timeout, numberOfWorkers);
                LOG.info("Existing workers at default barrier: " + existingWorkers.size());
                existingWorkers.clear();
            }
            // do not get previous events on barriers
            // get current snapshots of both barriers at restart
            String initBarrierDir = ZKUtils.initBarrierDir(rootPath, jobID);
            initBarrierCache = new PathChildrenCache(client, initBarrierDir, true);
            addBarrierChildrenCacheListener(initBarrierCache, JobMasterAPI.BarrierType.INIT);
            initBarrierCache.start(PathChildrenCache.StartMode.BUILD_INITIAL_CACHE);
            timeout = getInitialWorkersAtBarrier(initBarrierCache, existingWorkers);
            if (!existingWorkers.isEmpty()) {
                barrierMonitor.initInitAfterRestart(existingWorkers, timeout, numberOfWorkers);
                LOG.info("Existing workers at init barrier: " + existingWorkers);
            }
        } else {
            // We listen for status updates for the default barrier path
            String defaultBarrierDir = ZKUtils.defaultBarrierDir(rootPath, jobID);
            defaultBarrierCache = new PathChildrenCache(client, defaultBarrierDir, true);
            addBarrierChildrenCacheListener(defaultBarrierCache, JobMasterAPI.BarrierType.DEFAULT);
            defaultBarrierCache.start();
            // We listen for status updates for the init barrier path
            String initBarrierDir = ZKUtils.initBarrierDir(rootPath, jobID);
            initBarrierCache = new PathChildrenCache(client, initBarrierDir, true);
            addBarrierChildrenCacheListener(initBarrierCache, JobMasterAPI.BarrierType.INIT);
            initBarrierCache.start();
        }
    } catch (Twister2Exception e) {
        throw e;
    } catch (Exception e) {
        throw new Twister2Exception("Exception when initializing ZKMasterController.", e);
    }
}