Examples with ZKHelixAdmin org.apache.helix.manager.zk.ZKHelixAdmin used on opensource projects

Example 21 with ZKHelixAdmin

use of org.apache.helix.manager.zk.ZKHelixAdmin in project helix by apache.

the class TestFullAutoNodeTagging method testSafeAssignment.

/**
 * Basic test for tagging behavior. 10 participants, of which 4 are tagged. Launch all 10,
 * checking external view every time a tagged node is started. Then shut down all 10, checking
 * external view every time a tagged node is killed.
 */
@Test
public void testSafeAssignment() throws Exception {
    final int NUM_PARTICIPANTS = 10;
    final int NUM_PARTITIONS = 4;
    final int NUM_REPLICAS = 2;
    final String RESOURCE_NAME = "TestDB0";
    final String TAG = "ASSIGNABLE";
    final String[] TAGGED_NODES = { "localhost_12920", "localhost_12922", "localhost_12924", "localhost_12925" };
    Set<String> taggedNodes = Sets.newHashSet(TAGGED_NODES);
    String className = TestHelper.getTestClassName();
    String methodName = TestHelper.getTestMethodName();
    String clusterName = className + "_" + methodName;
    System.out.println("START " + clusterName + " at " + new Date(System.currentTimeMillis()));
    // Set up cluster
    // participant port
    TestHelper.setupCluster(// participant port
    clusterName, // participant port
    ZK_ADDR, // participant port
    12918, // participant name prefix
    "localhost", // resource name prefix
    "TestDB", // resources
    1, // partitions per resource
    NUM_PARTITIONS, // number of nodes
    NUM_PARTICIPANTS, // replicas
    NUM_REPLICAS, // use FULL_AUTO mode to test node tagging
    "MasterSlave", // use FULL_AUTO mode to test node tagging
    RebalanceMode.FULL_AUTO, // do rebalance
    true);
    // tag the resource and participants
    HelixAdmin helixAdmin = new ZKHelixAdmin(ZK_ADDR);
    for (String taggedNode : TAGGED_NODES) {
        helixAdmin.addInstanceTag(clusterName, taggedNode, TAG);
    }
    IdealState idealState = helixAdmin.getResourceIdealState(clusterName, RESOURCE_NAME);
    idealState.setInstanceGroupTag(TAG);
    helixAdmin.setResourceIdealState(clusterName, RESOURCE_NAME, idealState);
    // start controller
    ClusterControllerManager controller = new ClusterControllerManager(ZK_ADDR, clusterName, "controller");
    controller.syncStart();
    // start participants
    MockParticipantManager[] participants = new MockParticipantManager[NUM_PARTICIPANTS];
    for (int i = 0; i < NUM_PARTICIPANTS; i++) {
        final String instanceName = "localhost_" + (12918 + i);
        participants[i] = new MockParticipantManager(ZK_ADDR, clusterName, instanceName);
        participants[i].syncStart();
        // ensure that everything is valid if this is a tagged node that is starting
        if (taggedNodes.contains(instanceName)) {
            // make sure that the best possible matches the external view
            Thread.sleep(500);
            boolean result = ClusterStateVerifier.verifyByZkCallback(new BestPossAndExtViewZkVerifier(ZK_ADDR, clusterName));
            Assert.assertTrue(result);
            // make sure that the tagged state of the nodes is still balanced
            result = ClusterStateVerifier.verifyByZkCallback(new TaggedZkVerifier(clusterName, RESOURCE_NAME, TAGGED_NODES, false));
            Assert.assertTrue(result, "initial assignment with all tagged nodes live is invalid");
        }
    }
    // cleanup
    for (int i = 0; i < NUM_PARTICIPANTS; i++) {
        String participantName = participants[i].getInstanceName();
        participants[i].syncStop();
        if (taggedNodes.contains(participantName)) {
            // check that the external view is still correct even after removing tagged nodes
            taggedNodes.remove(participantName);
            Thread.sleep(500);
            boolean result = ClusterStateVerifier.verifyByZkCallback(new TaggedZkVerifier(clusterName, RESOURCE_NAME, TAGGED_NODES, taggedNodes.isEmpty()));
            Assert.assertTrue(result, "incorrect state after removing " + participantName + ", " + taggedNodes + " remain");
        }
    }
    controller.syncStop();
    System.out.println("END " + clusterName + " at " + new Date(System.currentTimeMillis()));
}

Example 22 with ZKHelixAdmin

use of org.apache.helix.manager.zk.ZKHelixAdmin in project ambry by linkedin.

the class HelixBootstrapUpgradeToolTest method verifyIdealStateForPartition.

/**
 * A helper method to verify IdealState for given replica's partition. It checks number of replicas (represented by
 * instances) under certain partition and verifies new added replica exists or removed old replica is no longer present.
 * @param replica the replica to check if it exists (could be either added replica or removed replica)
 * @param shouldExist if {@code true}, it means the given replica is newly added and should exist in IdealState.
 *                    {@code false} otherwise.
 * @param expectedReplicaCountForPartition expected number of replicas under certain partition
 * @param expectedResourceCount expected total number resource count in this cluster.
 */
private void verifyIdealStateForPartition(ReplicaId replica, boolean shouldExist, int expectedReplicaCountForPartition, long expectedResourceCount) {
    String dcName = replica.getDataNodeId().getDatacenterName();
    ZkInfo zkInfo = dcsToZkInfo.get(dcName);
    String clusterName = CLUSTER_NAME_PREFIX + CLUSTER_NAME_IN_STATIC_CLUSTER_MAP;
    ZKHelixAdmin admin = new ZKHelixAdmin("localhost:" + zkInfo.getPort());
    if (!activeDcSet.contains(dcName)) {
        Assert.assertFalse("Cluster should not be present, as dc " + dcName + " is not enabled", admin.getClusters().contains(clusterName));
    } else {
        List<String> resources = admin.getResourcesInCluster(clusterName);
        assertEquals("Resource count mismatch", expectedResourceCount, resources.size());
        String partitionName = replica.getPartitionId().toPathString();
        boolean resourceFound = false;
        for (String resource : resources) {
            IdealState idealState = admin.getResourceIdealState(clusterName, resource);
            if (idealState.getPartitionSet().contains(partitionName)) {
                resourceFound = true;
                Set<String> instances = idealState.getInstanceSet(partitionName);
                assertEquals("Replica number is not expected", expectedReplicaCountForPartition, instances.size());
                String instanceNameOfNewReplica = getInstanceName(replica.getDataNodeId());
                if (shouldExist) {
                    assertTrue("Instance set doesn't include new instance that holds added replica", instances.contains(instanceNameOfNewReplica));
                } else {
                    assertFalse("Instance that holds deleted replica should not exist in the set", instances.contains(instanceNameOfNewReplica));
                }
                break;
            }
        }
        assertTrue("No corresponding resource found for partition " + partitionName, resourceFound);
    }
}

Example 23 with ZKHelixAdmin

use of org.apache.helix.manager.zk.ZKHelixAdmin in project ambry by linkedin.

the class HelixBootstrapUpgradeToolTest method verifyResourceCount.

/**
 * Verify that the number of resources in Helix is as expected.
 * @param hardwareLayout the {@link HardwareLayout} of the static clustermap.
 * @param expectedResourceCount the expected number of resources in Helix.
 */
private void verifyResourceCount(HardwareLayout hardwareLayout, long expectedResourceCount) {
    for (Datacenter dc : hardwareLayout.getDatacenters()) {
        ZkInfo zkInfo = dcsToZkInfo.get(dc.getName());
        ZKHelixAdmin admin = new ZKHelixAdmin("localhost:" + zkInfo.getPort());
        if (!activeDcSet.contains(dc.getName())) {
            Assert.assertFalse("Cluster should not be present, as dc " + dc.getName() + " is not enabled", admin.getClusters().contains(CLUSTER_NAME_PREFIX + CLUSTER_NAME_IN_STATIC_CLUSTER_MAP));
        } else {
            assertEquals("Resource count mismatch", expectedResourceCount, admin.getResourcesInCluster(CLUSTER_NAME_PREFIX + CLUSTER_NAME_IN_STATIC_CLUSTER_MAP).size());
        }
    }
}

Example 24 with ZKHelixAdmin

use of org.apache.helix.manager.zk.ZKHelixAdmin in project ambry by linkedin.

the class HelixBootstrapUpgradeToolTest method testEnablePartitionAdminOp.

/**
 * Test that partition is correctly enabled on given node. The partition is first disabled and then enabled.
 * @throws Exception
 */
@Test
public void testEnablePartitionAdminOp() throws Exception {
    assumeTrue(!dcStr.equals("DC1") && !dcStr.equals("DC0"));
    String clusterName = CLUSTER_NAME_PREFIX + CLUSTER_NAME_IN_STATIC_CLUSTER_MAP;
    // Test regular bootstrap.
    long expectedResourceCount = (testPartitionLayout.getPartitionLayout().getPartitionCount() - 1) / DEFAULT_MAX_PARTITIONS_PER_RESOURCE + 1;
    writeBootstrapOrUpgrade(expectedResourceCount, false);
    int totalPartitionCount = testPartitionLayout.getPartitionCount();
    // Randomly pick a partition to disable/enable
    Partition testPartition = (Partition) testPartitionLayout.getPartitionLayout().getPartitions(null).get(RANDOM.nextInt(totalPartitionCount));
    // Randomly pick a replica from this partition
    List<ReplicaId> replicaIds = testPartition.getReplicaIds();
    DataNodeId dataNodeId = replicaIds.get(RANDOM.nextInt(replicaIds.size())).getDataNodeId();
    // Disable partition on chosen node
    HelixBootstrapUpgradeUtil.controlPartitionState(hardwareLayoutPath, partitionLayoutPath, zkLayoutPath, CLUSTER_NAME_PREFIX, dataNodeId.getDatacenterName(), dataNodeId.getHostname(), dataNodeId.getPort(), DisablePartition, testPartition.toPathString());
    // Verify the InstanceConfig is changed only in MapFields (Disabled partition is added to this field)
    ZkInfo zkInfo = dcsToZkInfo.get(dataNodeId.getDatacenterName());
    ZKHelixAdmin admin = new ZKHelixAdmin("localhost:" + zkInfo.getPort());
    InstanceConfig currentInstanceConfig = admin.getInstanceConfig(clusterName, getInstanceName(dataNodeId));
    assertTrue("There should be additional string in InstanceConfig due to disabling partition", currentInstanceConfig.getRecord().getMapFields().keySet().stream().anyMatch(k -> !k.startsWith("/mnt")));
    // Verify given partition is indeed disabled on specified node
    String resourceName = getResourceNameOfPartition(admin, clusterName, testPartition.toPathString());
    List<String> disabledPartitions = currentInstanceConfig.getDisabledPartitions(resourceName);
    assertEquals("Disabled partition is not expected", Collections.singletonList(testPartition.toPathString()), disabledPartitions);
    // Enable the same partition on same node
    HelixBootstrapUpgradeUtil.controlPartitionState(hardwareLayoutPath, partitionLayoutPath, zkLayoutPath, CLUSTER_NAME_PREFIX, dataNodeId.getDatacenterName(), dataNodeId.getHostname(), dataNodeId.getPort(), EnablePartition, testPartition.toPathString());
    // Verify instanceConfig has been updated (disabled partition is removed)
    currentInstanceConfig = admin.getInstanceConfig(clusterName, getInstanceName(dataNodeId));
    assertFalse("There shouldn't be any additional string in InstanceConfig", currentInstanceConfig.getRecord().getMapFields().keySet().stream().anyMatch(k -> !k.startsWith("/mnt")));
    // Verify there is no disabled partition
    assertNull("There shouldn't be any disabled partition", currentInstanceConfig.getDisabledPartitions(resourceName));
}

Example 25 with ZKHelixAdmin

use of org.apache.helix.manager.zk.ZKHelixAdmin in project ambry by linkedin.

the class HelixBootstrapUpgradeToolTest method clear.

@After
public void clear() {
    for (ZkInfo zkInfo : dcsToZkInfo.values()) {
        ZKHelixAdmin admin = new ZKHelixAdmin("localhost:" + zkInfo.getPort());
        admin.dropCluster(CLUSTER_NAME_PREFIX + CLUSTER_NAME_IN_STATIC_CLUSTER_MAP);
    }
}