Examples with ZNRecordSerializer org.apache.helix.zookeeper.datamodel.serializer.ZNRecordSerializer used on opensource projects

Example 26 with ZNRecordSerializer

use of org.apache.helix.zookeeper.datamodel.serializer.ZNRecordSerializer in project helix by apache.

the class TaskStateModelFactory method createZkClient.

/*
   * Create a RealmAwareZkClient to get thread pool sizes
   */
protected static RealmAwareZkClient createZkClient(HelixManager manager) {
    if (!(manager instanceof ZKHelixManager)) {
        // HelixManager, the initialization should be allowed.
        throw new UnsupportedOperationException("Only ZKHelixManager is supported for configurable thread pool.");
    }
    RealmAwareZkClient.RealmAwareZkClientConfig clientConfig = new RealmAwareZkClient.RealmAwareZkClientConfig().setZkSerializer(new ZNRecordSerializer());
    // Set operation retry timeout to prevent hanging infinitely
    clientConfig.setOperationRetryTimeout(Duration.ofMinutes(ZKCLIENT_OPERATION_RETRY_TIMEOUT).toMillis());
    String zkAddress = manager.getMetadataStoreConnectionString();
    if (Boolean.getBoolean(SystemPropertyKeys.MULTI_ZK_ENABLED) || zkAddress == null) {
        RealmAwareZkClient.RealmAwareZkConnectionConfig zkConnectionConfig = ((ZKHelixManager) manager).getRealmAwareZkConnectionConfig();
        // problem is fixed.
        if (zkConnectionConfig == null) {
            String clusterName = manager.getClusterName();
            String shardingKey = HelixUtil.clusterNameToShardingKey(clusterName);
            zkConnectionConfig = new RealmAwareZkClient.RealmAwareZkConnectionConfig.Builder().setRealmMode(RealmAwareZkClient.RealmMode.SINGLE_REALM).setZkRealmShardingKey(shardingKey).build();
        }
        try {
            return new FederatedZkClient(zkConnectionConfig, clientConfig);
        } catch (InvalidRoutingDataException | IllegalArgumentException e) {
            throw new HelixException("Failed to create FederatedZkClient!", e);
        }
    }
    // Note: operation retry timeout doesn't take effect due to github.com/apache/helix/issues/1682
    return SharedZkClientFactory.getInstance().buildZkClient(new HelixZkClient.ZkConnectionConfig(zkAddress), clientConfig.createHelixZkClientConfig());
}

Example 27 with ZNRecordSerializer

use of org.apache.helix.zookeeper.datamodel.serializer.ZNRecordSerializer in project helix by apache.

the class TestMSDAccessorLeaderElection method beforeClass.

@BeforeClass
public void beforeClass() throws Exception {
    super.beforeClass();
    _leaderBaseUri = getBaseUri().toString();
    _leaderBaseUri = _leaderBaseUri.substring(0, _leaderBaseUri.length() - 1);
    int newPort = getBaseUri().getPort() + 1;
    // Start a second server for testing Distributed Leader Election for writes
    _mockBaseUri = HttpConstants.HTTP_PROTOCOL_PREFIX + getBaseUri().getHost() + ":" + newPort;
    try {
        List<HelixRestNamespace> namespaces = new ArrayList<>();
        // Add test namespace
        namespaces.add(new HelixRestNamespace(TEST_NAMESPACE, HelixRestNamespace.HelixMetadataStoreType.ZOOKEEPER, _zkAddrTestNS, false));
        _mockHelixRestServer = new MockHelixRestServer(namespaces, newPort, getBaseUri().getPath(), Collections.singletonList(_auditLogger));
        _mockHelixRestServer.start();
    } catch (InterruptedException e) {
        LOG.error("MockHelixRestServer starting encounter an exception.", e);
    }
    // Calling the original endpoint to create an instance of MetadataStoreDirectory in case
    // it didn't exist yet.
    get(TEST_NAMESPACE_URI_PREFIX + "/metadata-store-realms", null, Response.Status.OK.getStatusCode(), true);
    // Set the new uri to be used in leader election
    System.setProperty(MetadataStoreRoutingConstants.MSDS_SERVER_HOSTNAME_KEY, getBaseUri().getHost());
    System.setProperty(MetadataStoreRoutingConstants.MSDS_SERVER_PORT_KEY, Integer.toString(newPort));
    // Start http client for testing
    _httpClient = HttpClients.createDefault();
    // Start zkclient to verify leader election behavior
    _zkClient = DedicatedZkClientFactory.getInstance().buildZkClient(new HelixZkClient.ZkConnectionConfig(_zkAddrTestNS), new HelixZkClient.ZkClientConfig().setZkSerializer(new ZNRecordSerializer()));
}

Example 28 with ZNRecordSerializer

use of org.apache.helix.zookeeper.datamodel.serializer.ZNRecordSerializer in project helix by apache.

the class TestZkMetadataStoreDirectory method beforeClass.

@BeforeClass
public void beforeClass() throws Exception {
    _zkList = new ArrayList<>(ZK_SERVER_MAP.keySet());
    clearRoutingData();
    // Populate routingZkAddrMap
    _routingZkAddrMap = new LinkedHashMap<>();
    int namespaceIndex = 0;
    String namespacePrefix = "namespace_";
    for (String zk : _zkList) {
        _routingZkAddrMap.put(namespacePrefix + namespaceIndex, zk);
    }
    // Write dummy mappings in ZK
    // Create a node that represents a realm address and add 3 sharding keys to it
    ZNRecord znRecord = new ZNRecord("RoutingInfo");
    _zkList.forEach(zk -> {
        ZK_SERVER_MAP.get(zk).getZkClient().setZkSerializer(new ZNRecordSerializer());
        // Write first realm and sharding keys pair
        znRecord.setListField(MetadataStoreRoutingConstants.ZNRECORD_LIST_FIELD_KEY, TEST_SHARDING_KEYS_1);
        ZK_SERVER_MAP.get(zk).getZkClient().createPersistent(MetadataStoreRoutingConstants.ROUTING_DATA_PATH + "/" + TEST_REALM_1, true);
        ZK_SERVER_MAP.get(zk).getZkClient().writeData(MetadataStoreRoutingConstants.ROUTING_DATA_PATH + "/" + TEST_REALM_1, znRecord);
        // Create another realm and sharding keys pair
        znRecord.setListField(MetadataStoreRoutingConstants.ZNRECORD_LIST_FIELD_KEY, TEST_SHARDING_KEYS_2);
        ZK_SERVER_MAP.get(zk).getZkClient().createPersistent(MetadataStoreRoutingConstants.ROUTING_DATA_PATH + "/" + TEST_REALM_2, true);
        ZK_SERVER_MAP.get(zk).getZkClient().writeData(MetadataStoreRoutingConstants.ROUTING_DATA_PATH + "/" + TEST_REALM_2, znRecord);
    });
    System.setProperty(MetadataStoreRoutingConstants.MSDS_SERVER_HOSTNAME_KEY, getBaseUri().getHost());
    System.setProperty(MetadataStoreRoutingConstants.MSDS_SERVER_PORT_KEY, Integer.toString(getBaseUri().getPort()));
    // Create metadataStoreDirectory
    for (Map.Entry<String, String> entry : _routingZkAddrMap.entrySet()) {
        _metadataStoreDirectory = ZkMetadataStoreDirectory.getInstance(entry.getKey(), entry.getValue());
    }
}

Example 29 with ZNRecordSerializer

use of org.apache.helix.zookeeper.datamodel.serializer.ZNRecordSerializer in project helix by apache.

the class MetadataStoreDirectoryAccessorTestBase method beforeClass.

@BeforeClass
public void beforeClass() throws Exception {
    _zkList = new ArrayList<>(ZK_SERVER_MAP.keySet());
    clearRoutingData();
    // Write dummy mappings in ZK
    // Create a node that represents a realm address and add 3 sharding keys to it
    ZNRecord znRecord = new ZNRecord("RoutingInfo");
    _zkList.forEach(zk -> {
        ZK_SERVER_MAP.get(zk).getZkClient().setZkSerializer(new ZNRecordSerializer());
        // Write first realm and sharding keys pair
        znRecord.setListField(MetadataStoreRoutingConstants.ZNRECORD_LIST_FIELD_KEY, TEST_SHARDING_KEYS_1);
        ZK_SERVER_MAP.get(zk).getZkClient().createPersistent(MetadataStoreRoutingConstants.ROUTING_DATA_PATH + "/" + TEST_REALM_1, true);
        ZK_SERVER_MAP.get(zk).getZkClient().writeData(MetadataStoreRoutingConstants.ROUTING_DATA_PATH + "/" + TEST_REALM_1, znRecord);
        // Create another realm and sharding keys pair
        znRecord.setListField(MetadataStoreRoutingConstants.ZNRECORD_LIST_FIELD_KEY, TEST_SHARDING_KEYS_2);
        ZK_SERVER_MAP.get(zk).getZkClient().createPersistent(MetadataStoreRoutingConstants.ROUTING_DATA_PATH + "/" + TEST_REALM_2, true);
        ZK_SERVER_MAP.get(zk).getZkClient().writeData(MetadataStoreRoutingConstants.ROUTING_DATA_PATH + "/" + TEST_REALM_2, znRecord);
    });
    _routingDataReader = new ZkRoutingDataReader(TEST_NAMESPACE, _zkAddrTestNS, null);
    System.setProperty(MetadataStoreRoutingConstants.MSDS_SERVER_HOSTNAME_KEY, getBaseUri().getHost());
    System.setProperty(MetadataStoreRoutingConstants.MSDS_SERVER_PORT_KEY, Integer.toString(getBaseUri().getPort()));
}

Example 30 with ZNRecordSerializer

use of org.apache.helix.zookeeper.datamodel.serializer.ZNRecordSerializer in project helix by apache.

the class HelixPropertyFactory method getCloudConfig.

/**
 * Retrieve the CloudConfig of the cluster if available.
 * Note: the reason we create a dedicated zk client here is because we need an isolated access to
 * ZK in order to create a HelixManager instance.
 * If shared zk client instance is used, this logic may break if users write tests that shut down
 * ZK server and start again at a 0 zxid because the shared client would have a higher zxid.
 * @param zkAddress
 * @param clusterName
 * @return
 */
private CloudConfig getCloudConfig(String zkAddress, String clusterName) {
    CloudConfig cloudConfig;
    RealmAwareZkClient dedicatedZkClient = null;
    try {
        if (Boolean.getBoolean(SystemPropertyKeys.MULTI_ZK_ENABLED) || zkAddress == null) {
            // DedicatedZkClient
            try {
                RealmAwareZkClient.RealmAwareZkConnectionConfig connectionConfig = new RealmAwareZkClient.RealmAwareZkConnectionConfig.Builder().setRealmMode(RealmAwareZkClient.RealmMode.SINGLE_REALM).setZkRealmShardingKey("/" + clusterName).build();
                dedicatedZkClient = DedicatedZkClientFactory.getInstance().buildZkClient(connectionConfig);
            } catch (IOException | InvalidRoutingDataException e) {
                throw new HelixException("Not able to connect on multi-ZK mode!", e);
            }
        } else {
            // Use a dedicated ZK client single-ZK mode
            HelixZkClient.ZkConnectionConfig connectionConfig = new HelixZkClient.ZkConnectionConfig(zkAddress);
            dedicatedZkClient = DedicatedZkClientFactory.getInstance().buildZkClient(connectionConfig);
        }
        dedicatedZkClient.setZkSerializer(new ZNRecordSerializer());
        ConfigAccessor configAccessor = new ConfigAccessor(dedicatedZkClient);
        // up yet, constructing a new ZKHelixManager should not throw an exception
        try {
            cloudConfig = configAccessor.getCloudConfig(clusterName) == null ? buildEmptyCloudConfig() : configAccessor.getCloudConfig(clusterName);
        } catch (HelixException e) {
            cloudConfig = buildEmptyCloudConfig();
        }
    } finally {
        // Use a try-finally to make sure zkclient connection is closed properly
        if (dedicatedZkClient != null && !dedicatedZkClient.isClosed()) {
            dedicatedZkClient.close();
        }
    }
    return cloudConfig;
}