Examples with CuratorFramework org.apache.flink.shaded.curator5.org.apache.curator.framework.CuratorFramework used on opensource projects

Example 41 with CuratorFramework

use of org.apache.flink.shaded.curator5.org.apache.curator.framework.CuratorFramework in project flink by apache.

the class Kafka08ITCase method runOffsetManipulationInZooKeeperTest.

@Test
public void runOffsetManipulationInZooKeeperTest() {
    try {
        final String topicName = "ZookeeperOffsetHandlerTest-Topic";
        final String groupId = "ZookeeperOffsetHandlerTest-Group";
        final Long offset = (long) (Math.random() * Long.MAX_VALUE);
        CuratorFramework curatorFramework = ((KafkaTestEnvironmentImpl) kafkaServer).createCuratorClient();
        kafkaServer.createTestTopic(topicName, 3, 2);
        ZookeeperOffsetHandler.setOffsetInZooKeeper(curatorFramework, groupId, topicName, 0, offset);
        Long fetchedOffset = ZookeeperOffsetHandler.getOffsetFromZooKeeper(curatorFramework, groupId, topicName, 0);
        curatorFramework.close();
        assertEquals(offset, fetchedOffset);
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 42 with CuratorFramework

use of org.apache.flink.shaded.curator5.org.apache.curator.framework.CuratorFramework in project flink by apache.

the class Kafka08ITCase method testOffsetAutocommitTest.

@Test(timeout = 60000)
public void testOffsetAutocommitTest() throws Exception {
    final int parallelism = 3;
    // write a sequence from 0 to 99 to each of the 3 partitions.
    final String topicName = writeSequence("testOffsetAutocommit", 100, parallelism, 1);
    StreamExecutionEnvironment env = StreamExecutionEnvironment.createRemoteEnvironment("localhost", flinkPort);
    // NOTE: We are not enabling the checkpointing!
    env.getConfig().disableSysoutLogging();
    env.getConfig().setRestartStrategy(RestartStrategies.noRestart());
    env.setParallelism(parallelism);
    // the readSequence operation sleeps for 20 ms between each record.
    // setting a delay of 25*20 = 500 for the commit interval makes
    // sure that we commit roughly 3-4 times while reading, however
    // at least once.
    Properties readProps = new Properties();
    readProps.putAll(standardProps);
    // make sure that auto commit is enabled in the properties
    readProps.setProperty("auto.commit.enable", "true");
    readProps.setProperty("auto.commit.interval.ms", "500");
    // read so that the offset can be committed to ZK
    readSequence(env, StartupMode.GROUP_OFFSETS, null, readProps, parallelism, topicName, 100, 0);
    // get the offset
    CuratorFramework curatorFramework = ((KafkaTestEnvironmentImpl) kafkaServer).createCuratorClient();
    Long o1 = ZookeeperOffsetHandler.getOffsetFromZooKeeper(curatorFramework, standardProps.getProperty("group.id"), topicName, 0);
    Long o2 = ZookeeperOffsetHandler.getOffsetFromZooKeeper(curatorFramework, standardProps.getProperty("group.id"), topicName, 1);
    Long o3 = ZookeeperOffsetHandler.getOffsetFromZooKeeper(curatorFramework, standardProps.getProperty("group.id"), topicName, 2);
    curatorFramework.close();
    LOG.info("Got final offsets from zookeeper o1={}, o2={}, o3={}", o1, o2, o3);
    // ensure that the offset has been committed
    boolean atLeastOneOffsetSet = (o1 != null && o1 > 0 && o1 <= 100) || (o2 != null && o2 > 0 && o2 <= 100) || (o3 != null && o3 > 0 && o3 <= 100);
    assertTrue("Expecting at least one offset to be set o1=" + o1 + " o2=" + o2 + " o3=" + o3, atLeastOneOffsetSet);
    deleteTestTopic(topicName);
}

Example 43 with CuratorFramework

use of org.apache.flink.shaded.curator5.org.apache.curator.framework.CuratorFramework in project flink by apache.

the class ZooKeeperUtilityFactory method createZooKeeperStateHandleStore.

/**
	 * Creates a {@link ZooKeeperStateHandleStore} instance with the provided arguments.
	 *
	 * @param zkStateHandleStorePath specifying the path in ZooKeeper to store the state handles to
	 * @param stateStorageHelper storing the actual state data
	 * @param executor to run asynchronous callbacks of the state handle store
	 * @param <T> Type of the state to be stored
	 * @return a ZooKeeperStateHandleStore instance
	 * @throws Exception if ZooKeeper could not create the provided state handle store path in
	 *     ZooKeeper
	 */
public <T extends Serializable> ZooKeeperStateHandleStore<T> createZooKeeperStateHandleStore(String zkStateHandleStorePath, RetrievableStateStorageHelper<T> stateStorageHelper, Executor executor) throws Exception {
    facade.newNamespaceAwareEnsurePath(zkStateHandleStorePath).ensure(facade.getZookeeperClient());
    CuratorFramework stateHandleStoreFacade = facade.usingNamespace(ZooKeeperUtils.generateZookeeperPath(facade.getNamespace(), zkStateHandleStorePath));
    return new ZooKeeperStateHandleStore<>(stateHandleStoreFacade, stateStorageHelper, executor);
}

Example 44 with CuratorFramework

use of org.apache.flink.shaded.curator5.org.apache.curator.framework.CuratorFramework in project flink by apache.

the class ZooKeeperStateHandleStoreITCase method testAddDiscardStateHandleAfterFailure.

/**
	 * Tests that the created state handle is discarded if ZooKeeper create fails.
	 */
@Test
public void testAddDiscardStateHandleAfterFailure() throws Exception {
    // Setup
    LongStateStorage stateHandleProvider = new LongStateStorage();
    CuratorFramework client = spy(ZooKeeper.getClient());
    when(client.create()).thenThrow(new RuntimeException("Expected test Exception."));
    ZooKeeperStateHandleStore<Long> store = new ZooKeeperStateHandleStore<>(client, stateHandleProvider, Executors.directExecutor());
    // Config
    final String pathInZooKeeper = "/testAddDiscardStateHandleAfterFailure";
    final Long state = 81282227L;
    try {
        // Test
        store.add(pathInZooKeeper, state);
        fail("Did not throw expected exception");
    } catch (Exception ignored) {
    }
    // Verify
    // State handle created and discarded
    assertEquals(1, stateHandleProvider.getStateHandles().size());
    assertEquals(state, stateHandleProvider.getStateHandles().get(0).retrieveState());
    assertEquals(1, stateHandleProvider.getStateHandles().get(0).getNumberOfDiscardCalls());
}

Example 45 with CuratorFramework

use of org.apache.flink.shaded.curator5.org.apache.curator.framework.CuratorFramework in project BRFS by zhangnianli.

the class FileCenter method main.

public static void main(String[] args) {
    id = new Random().nextInt(10);
    System.out.println("id = " + id);
    System.out.println("hahahahha");
    RetryPolicy retryPolicy = new ExponentialBackoffRetry(1000, 3);
    CuratorFramework client = CuratorFrameworkFactory.builder().namespace(ROOT).connectString(zk_address).retryPolicy(retryPolicy).build();
    client.start();
    try {
        Stat stat = client.checkExists().forPath(DUPS);
        System.out.println("stat =" + stat);
        if (stat == null) {
            System.out.println("create--" + client.create().forPath(DUPS));
        }
        ExecutorService pool = Executors.newFixedThreadPool(5);
        PathChildrenCache pathCache = new PathChildrenCache(client, DUPS, true, false, pool);
        pathCache.getListenable().addListener(new PathNodeListener());
        pathCache.start();
    // TreeCache cache = new TreeCache(client, DUPS);
    // cache.getListenable().addListener(new TreeNodeListener(), pool);
    // cache.start();
    } catch (Exception e) {
        e.printStackTrace();
    }
    synchronized (client) {
        try {
            client.wait();
        } catch (InterruptedException e) {
            e.printStackTrace();
        }
    }
    client.close();
}