Examples with CuratorFramework org.apache.curator.framework.CuratorFramework used on opensource projects

Example 46 with CuratorFramework

use of org.apache.curator.framework.CuratorFramework in project flink by apache.

the class Kafka08ITCase method testInvalidOffset.

@Test(timeout = 60000)
public void testInvalidOffset() throws Exception {
    final int parallelism = 1;
    // write 20 messages into topic:
    final String topic = writeSequence("invalidOffsetTopic", 20, parallelism, 1);
    // set invalid offset:
    CuratorFramework curatorClient = ((KafkaTestEnvironmentImpl) kafkaServer).createCuratorClient();
    ZookeeperOffsetHandler.setOffsetInZooKeeper(curatorClient, standardProps.getProperty("group.id"), topic, 0, 1234);
    curatorClient.close();
    // read from topic
    final int valuesCount = 20;
    final int startFrom = 0;
    final StreamExecutionEnvironment env = StreamExecutionEnvironment.createRemoteEnvironment("localhost", flinkPort);
    env.getConfig().disableSysoutLogging();
    readSequence(env, StartupMode.GROUP_OFFSETS, null, standardProps, parallelism, topic, valuesCount, startFrom);
    deleteTestTopic(topic);
}

Example 47 with CuratorFramework

use of org.apache.curator.framework.CuratorFramework in project flink by apache.

the class Kafka08ITCase method runOffsetManipulationInZooKeeperTest.

@Test
public void runOffsetManipulationInZooKeeperTest() {
    try {
        final String topicName = "ZookeeperOffsetHandlerTest-Topic";
        final String groupId = "ZookeeperOffsetHandlerTest-Group";
        final Long offset = (long) (Math.random() * Long.MAX_VALUE);
        CuratorFramework curatorFramework = ((KafkaTestEnvironmentImpl) kafkaServer).createCuratorClient();
        kafkaServer.createTestTopic(topicName, 3, 2);
        ZookeeperOffsetHandler.setOffsetInZooKeeper(curatorFramework, groupId, topicName, 0, offset);
        Long fetchedOffset = ZookeeperOffsetHandler.getOffsetFromZooKeeper(curatorFramework, groupId, topicName, 0);
        curatorFramework.close();
        assertEquals(offset, fetchedOffset);
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 48 with CuratorFramework

use of org.apache.curator.framework.CuratorFramework in project flink by apache.

the class Kafka08ITCase method testOffsetAutocommitTest.

@Test(timeout = 60000)
public void testOffsetAutocommitTest() throws Exception {
    final int parallelism = 3;
    // write a sequence from 0 to 99 to each of the 3 partitions.
    final String topicName = writeSequence("testOffsetAutocommit", 100, parallelism, 1);
    StreamExecutionEnvironment env = StreamExecutionEnvironment.createRemoteEnvironment("localhost", flinkPort);
    // NOTE: We are not enabling the checkpointing!
    env.getConfig().disableSysoutLogging();
    env.getConfig().setRestartStrategy(RestartStrategies.noRestart());
    env.setParallelism(parallelism);
    // the readSequence operation sleeps for 20 ms between each record.
    // setting a delay of 25*20 = 500 for the commit interval makes
    // sure that we commit roughly 3-4 times while reading, however
    // at least once.
    Properties readProps = new Properties();
    readProps.putAll(standardProps);
    // make sure that auto commit is enabled in the properties
    readProps.setProperty("auto.commit.enable", "true");
    readProps.setProperty("auto.commit.interval.ms", "500");
    // read so that the offset can be committed to ZK
    readSequence(env, StartupMode.GROUP_OFFSETS, null, readProps, parallelism, topicName, 100, 0);
    // get the offset
    CuratorFramework curatorFramework = ((KafkaTestEnvironmentImpl) kafkaServer).createCuratorClient();
    Long o1 = ZookeeperOffsetHandler.getOffsetFromZooKeeper(curatorFramework, standardProps.getProperty("group.id"), topicName, 0);
    Long o2 = ZookeeperOffsetHandler.getOffsetFromZooKeeper(curatorFramework, standardProps.getProperty("group.id"), topicName, 1);
    Long o3 = ZookeeperOffsetHandler.getOffsetFromZooKeeper(curatorFramework, standardProps.getProperty("group.id"), topicName, 2);
    curatorFramework.close();
    LOG.info("Got final offsets from zookeeper o1={}, o2={}, o3={}", o1, o2, o3);
    // ensure that the offset has been committed
    boolean atLeastOneOffsetSet = (o1 != null && o1 > 0 && o1 <= 100) || (o2 != null && o2 > 0 && o2 <= 100) || (o3 != null && o3 > 0 && o3 <= 100);
    assertTrue("Expecting at least one offset to be set o1=" + o1 + " o2=" + o2 + " o3=" + o3, atLeastOneOffsetSet);
    deleteTestTopic(topicName);
}

Example 49 with CuratorFramework

use of org.apache.curator.framework.CuratorFramework in project flink by apache.

the class ZooKeeperUtilityFactory method createZooKeeperStateHandleStore.

/**
	 * Creates a {@link ZooKeeperStateHandleStore} instance with the provided arguments.
	 *
	 * @param zkStateHandleStorePath specifying the path in ZooKeeper to store the state handles to
	 * @param stateStorageHelper storing the actual state data
	 * @param executor to run asynchronous callbacks of the state handle store
	 * @param <T> Type of the state to be stored
	 * @return a ZooKeeperStateHandleStore instance
	 * @throws Exception if ZooKeeper could not create the provided state handle store path in
	 *     ZooKeeper
	 */
public <T extends Serializable> ZooKeeperStateHandleStore<T> createZooKeeperStateHandleStore(String zkStateHandleStorePath, RetrievableStateStorageHelper<T> stateStorageHelper, Executor executor) throws Exception {
    facade.newNamespaceAwareEnsurePath(zkStateHandleStorePath).ensure(facade.getZookeeperClient());
    CuratorFramework stateHandleStoreFacade = facade.usingNamespace(ZooKeeperUtils.generateZookeeperPath(facade.getNamespace(), zkStateHandleStorePath));
    return new ZooKeeperStateHandleStore<>(stateHandleStoreFacade, stateStorageHelper, executor);
}

Example 50 with CuratorFramework

use of org.apache.curator.framework.CuratorFramework in project flink by apache.

the class ZooKeeperStateHandleStoreITCase method testAddDiscardStateHandleAfterFailure.

/**
	 * Tests that the created state handle is discarded if ZooKeeper create fails.
	 */
@Test
public void testAddDiscardStateHandleAfterFailure() throws Exception {
    // Setup
    LongStateStorage stateHandleProvider = new LongStateStorage();
    CuratorFramework client = spy(ZooKeeper.getClient());
    when(client.create()).thenThrow(new RuntimeException("Expected test Exception."));
    ZooKeeperStateHandleStore<Long> store = new ZooKeeperStateHandleStore<>(client, stateHandleProvider, Executors.directExecutor());
    // Config
    final String pathInZooKeeper = "/testAddDiscardStateHandleAfterFailure";
    final Long state = 81282227L;
    try {
        // Test
        store.add(pathInZooKeeper, state);
        fail("Did not throw expected exception");
    } catch (Exception ignored) {
    }
    // Verify
    // State handle created and discarded
    assertEquals(1, stateHandleProvider.getStateHandles().size());
    assertEquals(state, stateHandleProvider.getStateHandles().get(0).retrieveState());
    assertEquals(1, stateHandleProvider.getStateHandles().get(0).getNumberOfDiscardCalls());
}