Examples with ZNRecordSerializer org.apache.helix.zookeeper.datamodel.serializer.ZNRecordSerializer used on opensource projects

Example 1 with ZNRecordSerializer

use of org.apache.helix.zookeeper.datamodel.serializer.ZNRecordSerializer in project helix by apache.

the class RealmAwareZkClientFactoryTestBase method testRealmAwareZkClientCreatePersistent.

/**
 * Test the persistent create() call against a valid path and an invalid path.
 * Valid path is one that belongs to the realm designated by the sharding key.
 * Invalid path is one that does not belong to the realm designated by the sharding key.
 */
@Test(dependsOnMethods = "testRealmAwareZkClientCreation")
public void testRealmAwareZkClientCreatePersistent() {
    _realmAwareZkClient.setZkSerializer(new ZNRecordSerializer());
    // Test writing and reading against the validPath
    _realmAwareZkClient.createPersistent(TEST_VALID_PATH, true);
    _realmAwareZkClient.writeData(TEST_VALID_PATH, DUMMY_RECORD);
    Assert.assertEquals(_realmAwareZkClient.readData(TEST_VALID_PATH), DUMMY_RECORD);
    // Test writing and reading against the invalid path
    try {
        _realmAwareZkClient.createPersistent(TEST_INVALID_PATH, true);
        Assert.fail("Create() should not succeed on an invalid path!");
    } catch (IllegalArgumentException e) {
    // Okay - expected
    }
}

Example 2 with ZNRecordSerializer

use of org.apache.helix.zookeeper.datamodel.serializer.ZNRecordSerializer in project helix by apache.

the class TestMultiZkConectionConfig method beforeClass.

@BeforeClass
public void beforeClass() throws Exception {
    // Create 3 in-memory zookeepers and routing mapping
    for (int i = 0; i < NUM_ZK; i++) {
        String zkAddress = ZK_PREFIX + (ZK_START_PORT + i);
        ZK_SERVER_MAP.put(zkAddress, TestHelper.startZkServer(zkAddress));
        ZK_CLIENT_MAP.put(zkAddress, DedicatedZkClientFactory.getInstance().buildZkClient(new HelixZkClient.ZkConnectionConfig(zkAddress), new HelixZkClient.ZkClientConfig().setZkSerializer(new ZNRecordSerializer())));
        // One cluster per ZkServer created
        _rawRoutingData.put(zkAddress, Collections.singletonList("/" + CLUSTER_LIST.get(i)));
    }
    // Create a Mock MSDS
    final String msdsHostName = "localhost";
    final int msdsPort = 11117;
    final String msdsNamespace = "multiZkTest";
    _msdsEndpoint = "http://" + msdsHostName + ":" + msdsPort + "/admin/v2/namespaces/" + msdsNamespace;
    _msds = new MockMetadataStoreDirectoryServer(msdsHostName, msdsPort, msdsNamespace, _rawRoutingData);
    _msds.startServer();
    // Save previously-set system configs
    String prevMultiZkEnabled = System.getProperty(SystemPropertyKeys.MULTI_ZK_ENABLED);
    String prevMsdsServerEndpoint = System.getProperty(MetadataStoreRoutingConstants.MSDS_SERVER_ENDPOINT_KEY);
    if (prevMultiZkEnabled != null) {
        _configStore.put(SystemPropertyKeys.MULTI_ZK_ENABLED, prevMultiZkEnabled);
    }
    if (prevMsdsServerEndpoint != null) {
        _configStore.put(MetadataStoreRoutingConstants.MSDS_SERVER_ENDPOINT_KEY, prevMsdsServerEndpoint);
    }
    // Turn on multiZk mode in System config
    System.setProperty(SystemPropertyKeys.MULTI_ZK_ENABLED, "true");
    // MSDS endpoint: http://localhost:11117/admin/v2/namespaces/multiZkTest
    // We are not setting routing ZK in system property.
    // System.setProperty(MetadataStoreRoutingConstants.MSDS_SERVER_ENDPOINT_KEY, _msdsEndpoint);
    // Routing data may be set by other tests using the same endpoint; reset() for good measure
    RoutingDataManager.getInstance().reset();
    try {
        _zkClient = new FederatedZkClient(new RealmAwareZkClient.RealmAwareZkConnectionConfig.Builder().setRoutingDataSourceEndpoint(_msdsEndpoint + "," + ZK_PREFIX + ZK_START_PORT).setRoutingDataSourceType(RoutingDataReaderType.HTTP_ZK_FALLBACK.name()).build(), new RealmAwareZkClient.RealmAwareZkClientConfig());
        _zkClient.setZkSerializer(new ZNRecordSerializer());
    } catch (Exception ex) {
        for (StackTraceElement elm : ex.getStackTrace()) {
            System.out.println(elm);
        }
    }
    System.out.println("end start");
}

Example 3 with ZNRecordSerializer

use of org.apache.helix.zookeeper.datamodel.serializer.ZNRecordSerializer in project helix by apache.

the class ZkRoutingDataReader method getRawRoutingData.

/**
 * Returns a map form of metadata store routing data.
 * The map fields stand for metadata store realm address (key), and a corresponding list of ZK
 * path sharding keys (key).
 * @param endpoint
 * @return
 */
@Override
public Map<String, List<String>> getRawRoutingData(String endpoint) {
    ZkClient zkClient = new ZkClient.Builder().setZkServer(endpoint).setZkSerializer(new ZNRecordSerializer()).build();
    Map<String, List<String>> routingData = new HashMap<>();
    List<String> allRealmAddresses;
    try {
        allRealmAddresses = zkClient.getChildren(MetadataStoreRoutingConstants.ROUTING_DATA_PATH);
    } catch (ZkNoNodeException e) {
        throw new MultiZkException("Routing data directory ZNode " + MetadataStoreRoutingConstants.ROUTING_DATA_PATH + " does not exist. Routing ZooKeeper address: " + endpoint);
    }
    if (allRealmAddresses != null) {
        for (String realmAddress : allRealmAddresses) {
            ZNRecord record = zkClient.readData(MetadataStoreRoutingConstants.ROUTING_DATA_PATH + "/" + realmAddress, true);
            if (record != null) {
                List<String> shardingKeys = record.getListField(MetadataStoreRoutingConstants.ZNRECORD_LIST_FIELD_KEY);
                routingData.put(realmAddress, shardingKeys != null ? shardingKeys : Collections.emptyList());
            }
        }
    }
    zkClient.close();
    return routingData;
}

Example 4 with ZNRecordSerializer

use of org.apache.helix.zookeeper.datamodel.serializer.ZNRecordSerializer in project helix by apache.

the class TestFederatedZkClient method testUpdateRoutingDataOnCacheMissZK.

/**
 * This tests the routing data update feature only enabled when
 * RoutingSystemPropertyKeys.UPDATE_ROUTING_DATA_ON_CACHE_MISS is set to true.
 * Routing data source is ZK.
 */
@Test(dependsOnMethods = "testUpdateRoutingDataOnCacheMissMSDS")
public void testUpdateRoutingDataOnCacheMissZK() throws IOException, InvalidRoutingDataException {
    // Set up routing data in ZK with empty sharding key list
    String zkRealm = "localhost:2127";
    String newShardingKey = "/sharding-key-9";
    String newShardingKey2 = "/sharding-key-10";
    ZkClient zkClient = new ZkClient.Builder().setZkServer(zkRealm).setZkSerializer(new ZNRecordSerializer()).build();
    zkClient.create(MetadataStoreRoutingConstants.ROUTING_DATA_PATH, null, CreateMode.PERSISTENT);
    ZNRecord zkRealmRecord = new ZNRecord(zkRealm);
    List<String> keyList = // Need a non-empty keyList
    new ArrayList<>(TestConstants.TEST_KEY_LIST_1);
    zkRealmRecord.setListField(MetadataStoreRoutingConstants.ZNRECORD_LIST_FIELD_KEY, keyList);
    zkClient.create(MetadataStoreRoutingConstants.ROUTING_DATA_PATH + "/" + zkRealm, zkRealmRecord, CreateMode.PERSISTENT);
    // Enable routing data update upon cache miss
    System.setProperty(RoutingSystemPropertyKeys.UPDATE_ROUTING_DATA_ON_CACHE_MISS, "true");
    // Set the routing data update interval to 0 so there's no delay in testing
    System.setProperty(RoutingSystemPropertyKeys.ROUTING_DATA_UPDATE_INTERVAL_MS, "0");
    RoutingDataManager.getInstance().reset();
    RoutingDataManager.getInstance().getMetadataStoreRoutingData(RoutingDataReaderType.ZK, zkRealm);
    /*
     * Test is 2-tiered because cache update is 2-tiered
     * Case 1:
     * - RoutingDataManager does not have the key
     * - ZK has the key
     * This simulates a case where FederatedZkClient must do a I/O based update (must read from ZK).
     */
    // Add the key to ZK
    zkRealmRecord.getListField(MetadataStoreRoutingConstants.ZNRECORD_LIST_FIELD_KEY).add(newShardingKey);
    zkClient.writeData(MetadataStoreRoutingConstants.ROUTING_DATA_PATH + "/" + zkRealm, zkRealmRecord);
    // Verify that RoutingDataManager does not have the key
    MetadataStoreRoutingData routingData = RoutingDataManager.getInstance().getMetadataStoreRoutingData(RoutingDataReaderType.ZK, zkRealm);
    try {
        routingData.getMetadataStoreRealm(newShardingKey);
        Assert.fail("Must throw NoSuchElementException!");
    } catch (NoSuchElementException e) {
    // Expected
    }
    // Create a new FederatedZkClient
    FederatedZkClient federatedZkClient = new FederatedZkClient(new RealmAwareZkClient.RealmAwareZkConnectionConfig.Builder().setRoutingDataSourceType(RoutingDataReaderType.ZK.name()).setRoutingDataSourceEndpoint(zkRealm).build(), new RealmAwareZkClient.RealmAwareZkClientConfig());
    // exists() must succeed and RoutingDataManager should now have the key (cache update must
    // have happened)
    // False expected for the following call because the znode does not exist and we are checking
    // whether the call succeeds or not
    Assert.assertFalse(federatedZkClient.exists(newShardingKey));
    Assert.assertEquals(zkRealm, RoutingDataManager.getInstance().getMetadataStoreRoutingData(RoutingDataReaderType.ZK, zkRealm).getMetadataStoreRealm(newShardingKey));
    /*
     * Case 2:
     * - RoutingDataManager has the key
     * - ZK does not have the key
     * - continue using the same ZkClient because we want an existing federated client that does
     * not have the key
     */
    // Add newShardingKey2 to ZK's routing data (in order to give RoutingDataManager the key)
    zkRealmRecord.getListField(MetadataStoreRoutingConstants.ZNRECORD_LIST_FIELD_KEY).add(newShardingKey2);
    zkClient.writeData(MetadataStoreRoutingConstants.ROUTING_DATA_PATH + "/" + zkRealm, zkRealmRecord);
    // Update RoutingDataManager so it has the key
    RoutingDataManager.getInstance().reset();
    Assert.assertEquals(zkRealm, RoutingDataManager.getInstance().getMetadataStoreRoutingData(RoutingDataReaderType.ZK, zkRealm).getMetadataStoreRealm(newShardingKey2));
    // Remove newShardingKey2 from ZK
    zkRealmRecord.getListField(MetadataStoreRoutingConstants.ZNRECORD_LIST_FIELD_KEY).remove(newShardingKey2);
    zkClient.writeData(MetadataStoreRoutingConstants.ROUTING_DATA_PATH + "/" + zkRealm, zkRealmRecord);
    // exists() must succeed and RoutingDataManager should still have the key
    // This means that we do not do a hard update (I/O based update) because in-memory cache already
    // has the key
    // False expected for the following call because the znode does not exist and we are checking
    // whether the call succeeds or not
    Assert.assertFalse(federatedZkClient.exists(newShardingKey2));
    Assert.assertEquals(zkRealm, RoutingDataManager.getInstance().getMetadataStoreRoutingData(RoutingDataReaderType.ZK, zkRealm).getMetadataStoreRealm(newShardingKey2));
    // Also check that ZK does not have the new sharding key through resetting RoutingDataManager
    // and re-reading from ZK
    RoutingDataManager.getInstance().reset();
    try {
        RoutingDataManager.getInstance().getMetadataStoreRoutingData(RoutingDataReaderType.ZK, zkRealm).getMetadataStoreRealm(newShardingKey2);
        Assert.fail("NoSuchElementException expected!");
    } catch (NoSuchElementException e) {
    // Expected because ZK does not contain the key
    }
    // Clean up federatedZkClient
    federatedZkClient.close();
    // Clean up ZK writes and ZkClient
    zkClient.deleteRecursively(MetadataStoreRoutingConstants.ROUTING_DATA_PATH);
    zkClient.close();
    // Disable System property
    System.clearProperty(RoutingSystemPropertyKeys.UPDATE_ROUTING_DATA_ON_CACHE_MISS);
    System.clearProperty(RoutingSystemPropertyKeys.ROUTING_DATA_UPDATE_INTERVAL_MS);
}

Example 5 with ZNRecordSerializer

use of org.apache.helix.zookeeper.datamodel.serializer.ZNRecordSerializer in project helix by apache.

the class TestRawZkClient method testCreateContainer.

@Test
void testCreateContainer() {
    // Enable extended types and create a ZkClient
    System.setProperty("zookeeper.extendedTypesEnabled", "true");
    ZkClient zkClient = new ZkClient(ZkTestBase.ZK_ADDR);
    zkClient.setZkSerializer(new ZNRecordSerializer());
    // Make sure the test path is clear
    String parentPath = "/tmp";
    String path = "/tmp/createContainer";
    zkClient.deleteRecursively(parentPath);
    AssertJUnit.assertFalse(zkClient.exists(parentPath));
    AssertJUnit.assertFalse(zkClient.exists(path));
    ZNRecord record = new ZNRecord("record");
    String key = "key";
    String value = "value";
    record.setSimpleField(key, value);
    // Create a ZNode with the above ZNRecord and read back its data
    zkClient.createContainer(parentPath, record);
    AssertJUnit.assertTrue(zkClient.exists(parentPath));
    ZNRecord retrievedRecord = zkClient.readData(parentPath);
    AssertJUnit.assertEquals(value, retrievedRecord.getSimpleField(key));
    // Clear the path and test with createParents = true
    AssertJUnit.assertTrue(zkClient.delete(parentPath));
    zkClient.createContainer(path, true);
    AssertJUnit.assertTrue(zkClient.exists(path));
    // Clean up
    zkClient.deleteRecursively(parentPath);
    zkClient.close();
    System.clearProperty("zookeeper.extendedTypesEnabled");
}