Examples with ZkClient org.apache.helix.manager.zk.ZkClient used on opensource projects

Example 71 with ZkClient

use of org.apache.helix.manager.zk.ZkClient in project helix by apache.

the class TestStatusUpdate method testParticipantStatusUpdates.

// For now write participant StatusUpdates to log4j.
// TODO: Need to investigate another data channel to report to controller and re-enable
// this test
// @Test
public void testParticipantStatusUpdates() throws Exception {
    ZkClient zkClient = new ZkClient(ZkIntegrationTestBase.ZK_ADDR);
    zkClient.setZkSerializer(new ZNRecordSerializer());
    ZKHelixDataAccessor accessor = new ZKHelixDataAccessor(CLUSTER_NAME, new ZkBaseDataAccessor(zkClient));
    Builder keyBuilder = accessor.keyBuilder();
    List<ExternalView> extViews = accessor.getChildValues(keyBuilder.externalViews());
    Assert.assertNotNull(extViews);
    for (ExternalView extView : extViews) {
        String resourceName = extView.getResourceName();
        Set<String> partitionSet = extView.getPartitionSet();
        for (String partition : partitionSet) {
            Map<String, String> stateMap = extView.getStateMap(partition);
            for (String instance : stateMap.keySet()) {
                String state = stateMap.get(instance);
                StatusUpdateUtil.StatusUpdateContents statusUpdates = StatusUpdateUtil.StatusUpdateContents.getStatusUpdateContents(accessor, instance, resourceName, partition);
                Map<String, StatusUpdateUtil.TaskStatus> taskMessages = statusUpdates.getTaskMessages();
                List<StatusUpdateUtil.Transition> transitions = statusUpdates.getTransitions();
                if (state.equals("MASTER")) {
                    Assert.assertEquals(transitions.size() >= 2, true, "Invalid number of transitions");
                    StatusUpdateUtil.Transition lastTransition = transitions.get(transitions.size() - 1);
                    StatusUpdateUtil.Transition prevTransition = transitions.get(transitions.size() - 2);
                    Assert.assertEquals(taskMessages.get(lastTransition.getMsgID()), StatusUpdateUtil.TaskStatus.COMPLETED, "Incomplete transition");
                    Assert.assertEquals(taskMessages.get(prevTransition.getMsgID()), StatusUpdateUtil.TaskStatus.COMPLETED, "Incomplete transition");
                    Assert.assertEquals(lastTransition.getFromState(), "SLAVE", "Invalid State");
                    Assert.assertEquals(lastTransition.getToState(), "MASTER", "Invalid State");
                } else if (state.equals("SLAVE")) {
                    Assert.assertEquals(transitions.size() >= 1, true, "Invalid number of transitions");
                    StatusUpdateUtil.Transition lastTransition = transitions.get(transitions.size() - 1);
                    Assert.assertEquals(lastTransition.getFromState().equals("MASTER") || lastTransition.getFromState().equals("OFFLINE"), true, "Invalid transition");
                    Assert.assertEquals(lastTransition.getToState(), "SLAVE", "Invalid State");
                }
            }
        }
    }
}

Example 72 with ZkClient

use of org.apache.helix.manager.zk.ZkClient in project helix by apache.

the class TestDriver method setupCluster.

public static void setupCluster(String uniqClusterName, String zkAddr, int numResources, int numPartitionsPerResource, int numInstances, int replica, boolean doRebalance) throws Exception {
    ZkClient zkClient = new ZkClient(zkAddr);
    zkClient.setZkSerializer(new ZNRecordSerializer());
    // String clusterName = CLUSTER_PREFIX + "_" + uniqClusterName;
    String clusterName = uniqClusterName;
    if (zkClient.exists("/" + clusterName)) {
        LOG.warn("test cluster already exists:" + clusterName + ", test name:" + uniqClusterName + " is not unique or test has been run without cleaning up zk; deleting it");
        zkClient.deleteRecursively("/" + clusterName);
    }
    if (_testInfoMap.containsKey(uniqClusterName)) {
        LOG.warn("test info already exists:" + uniqClusterName + " is not unique or test has been run without cleaning up test info map; removing it");
        _testInfoMap.remove(uniqClusterName);
    }
    TestInfo testInfo = new TestInfo(clusterName, zkClient, numResources, numPartitionsPerResource, numInstances, replica);
    _testInfoMap.put(uniqClusterName, testInfo);
    ClusterSetup setupTool = new ClusterSetup(zkAddr);
    setupTool.addCluster(clusterName, true);
    for (int i = 0; i < numInstances; i++) {
        int port = START_PORT + i;
        setupTool.addInstanceToCluster(clusterName, PARTICIPANT_PREFIX + "_" + port);
    }
    for (int i = 0; i < numResources; i++) {
        String dbName = TEST_DB_PREFIX + i;
        setupTool.addResourceToCluster(clusterName, dbName, numPartitionsPerResource, STATE_MODEL);
        if (doRebalance) {
            setupTool.rebalanceStorageCluster(clusterName, dbName, replica);
        // String idealStatePath = "/" + clusterName + "/" +
        // PropertyType.IDEALSTATES.toString() + "/"
        // + dbName;
        // ZNRecord idealState = zkClient.<ZNRecord> readData(idealStatePath);
        // testInfo._idealStateMap.put(dbName, idealState);
        }
    }
}

Example 73 with ZkClient

use of org.apache.helix.manager.zk.ZkClient in project helix by apache.

the class TestEnableCompression method testEnableCompressionResource.

@Test()
public void testEnableCompressionResource() throws Exception {
    // Logger.getRootLogger().setLevel(Level.INFO);
    String className = TestHelper.getTestClassName();
    String methodName = TestHelper.getTestMethodName();
    String clusterName = className + "_" + methodName;
    System.out.println("START " + clusterName + " at " + new Date(System.currentTimeMillis()));
    MockParticipantManager[] participants = new MockParticipantManager[5];
    // ClusterSetup setupTool = new ClusterSetup(ZK_ADDR);
    int numNodes = 10;
    // participant port
    TestHelper.setupCluster(// participant port
    clusterName, // participant port
    ZK_ADDR, // participant port
    12918, // participant name prefix
    "localhost", // resource name prefix
    "TestDB", // no resources, will be added later
    0, // partitions per resource
    0, // number of nodes
    numNodes, // replicas
    0, "OnlineOffline", // dont rebalance
    false);
    List<String> instancesInCluster = _gSetupTool.getClusterManagementTool().getInstancesInCluster(clusterName);
    String resourceName = "TestResource";
    CustomModeISBuilder customModeISBuilder = new CustomModeISBuilder(resourceName);
    int numPartitions = 10000;
    int numReplica = 3;
    customModeISBuilder.setNumPartitions(numPartitions);
    customModeISBuilder.setNumReplica(numReplica);
    customModeISBuilder.setStateModel("OnlineOffline");
    for (int p = 0; p < numPartitions; p++) {
        String partitionName = resourceName + "_" + p;
        customModeISBuilder.add(partitionName);
        for (int r = 0; r < numReplica; r++) {
            String instanceName = instancesInCluster.get((p % numNodes + r) % numNodes);
            customModeISBuilder.assignInstanceAndState(partitionName, instanceName, "ONLINE");
        }
    }
    IdealState idealstate = customModeISBuilder.build();
    idealstate.getRecord().setBooleanField("enableCompression", true);
    _gSetupTool.getClusterManagementTool().addResource(clusterName, resourceName, idealstate);
    ZkClient zkClient = new ZkClient(ZK_ADDR, 60 * 1000, 60 * 1000, new BytesPushThroughSerializer());
    zkClient.waitUntilConnected(10, TimeUnit.SECONDS);
    ClusterControllerManager controller = new ClusterControllerManager(ZK_ADDR, clusterName, "controller_0");
    controller.syncStart();
    // start participants
    for (int i = 0; i < 5; i++) {
        String instanceName = "localhost_" + (12918 + i);
        participants[i] = new MockParticipantManager(ZK_ADDR, clusterName, instanceName);
        participants[i].syncStart();
    }
    boolean result = ClusterStateVerifier.verifyByZkCallback(new BestPossAndExtViewZkVerifier(ZK_ADDR, clusterName), 120000);
    Assert.assertTrue(result);
    List<String> compressedPaths = new ArrayList<String>();
    findCompressedZNodes(zkClient, "/", compressedPaths);
    System.out.println("compressed paths:" + compressedPaths);
    // ONLY IDEALSTATE and EXTERNAL VIEW must be compressed
    Assert.assertEquals(compressedPaths.size(), 2);
    String idealstatePath = PropertyPathBuilder.idealState(clusterName, resourceName);
    String externalViewPath = PropertyPathBuilder.externalView(clusterName, resourceName);
    Assert.assertTrue(compressedPaths.contains(idealstatePath));
    Assert.assertTrue(compressedPaths.contains(externalViewPath));
    // clean up
    controller.syncStop();
    for (int i = 0; i < 5; i++) {
        participants[i].syncStop();
    }
    System.out.println("END " + clusterName + " at " + new Date(System.currentTimeMillis()));
}

Example 74 with ZkClient

use of org.apache.helix.manager.zk.ZkClient in project helix by apache.

the class ResourceAccessor method getResourceHealth.

/**
 * Returns health profile of all resources in the cluster
 *
 * @param clusterId
 * @return JSON result
 */
@GET
@Path("health")
public Response getResourceHealth(@PathParam("clusterId") String clusterId) {
    ZkClient zkClient = getZkClient();
    List<String> resourcesInIdealState = zkClient.getChildren(PropertyPathBuilder.idealState(clusterId));
    List<String> resourcesInExternalView = zkClient.getChildren(PropertyPathBuilder.externalView(clusterId));
    Map<String, String> resourceHealthResult = new HashMap<>();
    for (String resourceName : resourcesInIdealState) {
        if (resourcesInExternalView.contains(resourceName)) {
            Map<String, String> partitionHealth = computePartitionHealth(clusterId, resourceName);
            if (partitionHealth.isEmpty() || partitionHealth.values().contains(HealthStatus.UNHEALTHY.name())) {
                // No partitions for a resource or there exists one or more UNHEALTHY partitions in this resource, UNHEALTHY
                resourceHealthResult.put(resourceName, HealthStatus.UNHEALTHY.name());
            } else if (partitionHealth.values().contains(HealthStatus.PARTIAL_HEALTHY.name())) {
                // No UNHEALTHY partition, but one or more partially healthy partitions, resource is partially healthy
                resourceHealthResult.put(resourceName, HealthStatus.PARTIAL_HEALTHY.name());
            } else {
                // No UNHEALTHY or partially healthy partitions and non-empty, resource is healthy
                resourceHealthResult.put(resourceName, HealthStatus.HEALTHY.name());
            }
        } else {
            // If a resource is not in ExternalView, then it is UNHEALTHY
            resourceHealthResult.put(resourceName, HealthStatus.UNHEALTHY.name());
        }
    }
    return JSONRepresentation(resourceHealthResult);
}

Example 75 with ZkClient

use of org.apache.helix.manager.zk.ZkClient in project helix by apache.

the class SetupCluster method main.

public static void main(String[] args) {
    if (args.length < 2) {
        System.err.println("USAGE: java SetupCluster zookeeperAddress(e.g. localhost:2181) numberOfNodes");
        System.exit(1);
    }
    final String zkAddr = args[0];
    final int numNodes = Integer.parseInt(args[1]);
    final String clusterName = DEFAULT_CLUSTER_NAME;
    ZkClient zkclient = null;
    try {
        zkclient = new ZkClient(zkAddr, ZkClient.DEFAULT_SESSION_TIMEOUT, ZkClient.DEFAULT_CONNECTION_TIMEOUT, new ZNRecordSerializer());
        ZKHelixAdmin admin = new ZKHelixAdmin(zkclient);
        // add cluster
        admin.addCluster(clusterName, true);
        // add state model definition
        StateModelConfigGenerator generator = new StateModelConfigGenerator();
        admin.addStateModelDef(clusterName, DEFAULT_STATE_MODEL, new StateModelDefinition(generator.generateConfigForOnlineOffline()));
        // addNodes
        for (int i = 0; i < numNodes; i++) {
            String port = "" + (12001 + i);
            String serverId = "localhost_" + port;
            InstanceConfig config = new InstanceConfig(serverId);
            config.setHostName("localhost");
            config.setPort(port);
            config.setInstanceEnabled(true);
            admin.addInstance(clusterName, config);
        }
        // add resource "repository" which has 1 partition
        String resourceName = DEFAULT_RESOURCE_NAME;
        admin.addResource(clusterName, resourceName, DEFAULT_PARTITION_NUMBER, DEFAULT_STATE_MODEL, RebalanceMode.SEMI_AUTO.toString());
        admin.rebalance(clusterName, resourceName, 1);
    } finally {
        if (zkclient != null) {
            zkclient.close();
        }
    }
}