Examples with HelixAdmin org.apache.helix.HelixAdmin used on opensource projects

Example 11 with HelixAdmin

use of org.apache.helix.HelixAdmin in project ambry by linkedin.

the class HelixBootstrapUpgradeUtil method verifyDataNodeAndDiskEquivalencyInDc.

/**
 * Verify that the hardware layout information is in sync - which includes the node and disk information. Also verify
 * that the replicas belonging to disks are in sync between the static cluster map and Helix.
 * @param dc the datacenter whose information is to be verified.
 * @param clusterName the cluster to be verified.
 * @param partitionLayout the {@link PartitionLayout} of the static clustermap.
 */
private void verifyDataNodeAndDiskEquivalencyInDc(Datacenter dc, String clusterName, PartitionLayout partitionLayout) throws Exception {
    // The following properties are immaterial for the tool, but the ClusterMapConfig mandates their presence.
    Properties props = new Properties();
    props.setProperty("clustermap.host.name", "localhost");
    props.setProperty("clustermap.cluster.name", clusterName);
    props.setProperty("clustermap.datacenter.name", dc.getName());
    ClusterMapConfig clusterMapConfig = new ClusterMapConfig(new VerifiableProperties(props));
    StaticClusterManager staticClusterMap = (new StaticClusterAgentsFactory(clusterMapConfig, partitionLayout)).getClusterMap();
    HelixAdmin admin = adminForDc.get(dc.getName());
    List<String> allInstancesInHelix = admin.getInstancesInCluster(clusterName);
    for (DataNodeId dataNodeId : dc.getDataNodes()) {
        Map<String, Map<String, String>> mountPathToReplicas = getMountPathToReplicas(staticClusterMap, dataNodeId);
        DataNode dataNode = (DataNode) dataNodeId;
        String instanceName = getInstanceName(dataNode);
        ensureOrThrow(allInstancesInHelix.remove(instanceName), "Instance not present in Helix " + instanceName);
        InstanceConfig instanceConfig = admin.getInstanceConfig(clusterName, instanceName);
        Map<String, Map<String, String>> diskInfos = new HashMap<>(instanceConfig.getRecord().getMapFields());
        for (Disk disk : dataNode.getDisks()) {
            Map<String, String> diskInfoInHelix = diskInfos.remove(disk.getMountPath());
            ensureOrThrow(diskInfoInHelix != null, "Disk not present for instance " + instanceName + " disk " + disk.getMountPath());
            ensureOrThrow(disk.getRawCapacityInBytes() == Long.valueOf(diskInfoInHelix.get(ClusterMapUtils.DISK_CAPACITY_STR)), "Capacity mismatch for instance " + instanceName + " disk " + disk.getMountPath());
            Set<String> replicasInClusterMap;
            Map<String, String> replicaList = mountPathToReplicas.get(disk.getMountPath());
            replicasInClusterMap = new HashSet<>();
            if (replicaList != null) {
                replicasInClusterMap.addAll(replicaList.keySet());
            }
            Set<String> replicasInHelix;
            String replicasStr = diskInfoInHelix.get(ClusterMapUtils.REPLICAS_STR);
            if (replicasStr.isEmpty()) {
                replicasInHelix = new HashSet<>();
            } else {
                replicasInHelix = new HashSet<>();
                List<String> replicaInfoList = Arrays.asList(replicasStr.split(ClusterMapUtils.REPLICAS_DELIM_STR));
                for (String replicaInfo : replicaInfoList) {
                    String[] info = replicaInfo.split(ClusterMapUtils.REPLICAS_STR_SEPARATOR);
                    replicasInHelix.add(info[0]);
                    ensureOrThrow(info[1].equals(replicaList.get(info[0])), "Replica capacity should be the same.");
                }
            }
            ensureOrThrow(replicasInClusterMap.equals(replicasInHelix), "Replica information not consistent for instance " + instanceName + " disk " + disk.getMountPath() + "\n in Helix: " + replicaList + "\n in static clustermap: " + replicasInClusterMap);
        }
        ensureOrThrow(diskInfos.isEmpty(), "Instance " + instanceName + " has extra disks in Helix: " + diskInfos);
        ensureOrThrow(!dataNode.hasSSLPort() || (dataNode.getSSLPort() == Integer.valueOf(instanceConfig.getRecord().getSimpleField(ClusterMapUtils.SSLPORT_STR))), "SSL Port mismatch for instance " + instanceName);
        ensureOrThrow(dataNode.getDatacenterName().equals(instanceConfig.getRecord().getSimpleField(ClusterMapUtils.DATACENTER_STR)), "Datacenter mismatch for instance " + instanceName);
        ensureOrThrow(dataNode.getRackId() == Long.valueOf(instanceConfig.getRecord().getSimpleField(ClusterMapUtils.RACKID_STR)), "Rack Id mismatch for instance " + instanceName);
        Set<String> sealedReplicasInHelix = new HashSet<>(instanceConfig.getRecord().getListField(ClusterMapUtils.SEALED_STR));
        Set<String> sealedReplicasInClusterMap = new HashSet<>();
        for (Replica replica : staticClusterMap.getReplicas(dataNodeId)) {
            if (replica.getPartition().partitionState.equals(PartitionState.READ_ONLY)) {
                sealedReplicasInClusterMap.add(Long.toString(replica.getPartition().getId()));
            }
        }
        ensureOrThrow(sealedReplicasInClusterMap.equals(sealedReplicasInHelix), "Sealed replicas info mismatch for " + "instance " + instanceName);
    }
    ensureOrThrow(allInstancesInHelix.isEmpty(), "Following instances in Helix not found in the clustermap " + allInstancesInHelix);
}

Example 12 with HelixAdmin

use of org.apache.helix.HelixAdmin in project ambry by linkedin.

the class HelixParticipant method setSealedReplicas.

/**
 * Set the list of sealed replicas in the HelixAdmin
 * @param sealedReplicas list of sealed replicas to be set in the HelixAdmin
 * @return whether the operation succeeded or not
 */
private boolean setSealedReplicas(List<String> sealedReplicas) {
    HelixAdmin helixAdmin = manager.getClusterManagmentTool();
    InstanceConfig instanceConfig = helixAdmin.getInstanceConfig(clusterName, instanceName);
    if (instanceConfig == null) {
        throw new IllegalStateException("No instance config found for cluster: \"" + clusterName + "\", instance: \"" + instanceName + "\"");
    }
    instanceConfig.getRecord().setListField(ClusterMapUtils.SEALED_STR, sealedReplicas);
    return helixAdmin.setInstanceConfig(clusterName, instanceName, instanceConfig);
}

Example 13 with HelixAdmin

use of org.apache.helix.HelixAdmin in project ambry by linkedin.

the class HelixParticipant method getSealedReplicas.

/**
 * Get the list of sealed replicas from the HelixAdmin
 * @return list of sealed replicas from HelixAdmin
 */
private List<String> getSealedReplicas() {
    HelixAdmin helixAdmin = manager.getClusterManagmentTool();
    InstanceConfig instanceConfig = helixAdmin.getInstanceConfig(clusterName, instanceName);
    if (instanceConfig == null) {
        throw new IllegalStateException("No instance config found for cluster: \"" + clusterName + "\", instance: \"" + instanceName + "\"");
    }
    return ClusterMapUtils.getSealedReplicas(instanceConfig);
}

Example 14 with HelixAdmin

use of org.apache.helix.HelixAdmin in project ambry by linkedin.

the class HelixParticipantTest method testSetReplicaSealedState.

/**
 * Tests setReplicaSealedState method for {@link HelixParticipant}
 * @throws IOException
 */
@Test
public void testSetReplicaSealedState() throws IOException {
    // setup HelixParticipant and dependencies
    String partitionIdStr = "somePartitionId";
    String partitionIdStr2 = "someOtherPartitionId";
    ReplicaId replicaId = createMockAmbryReplica(partitionIdStr);
    ReplicaId replicaId2 = createMockAmbryReplica(partitionIdStr2);
    String hostname = "localhost";
    int port = 2200;
    String instanceName = ClusterMapUtils.getInstanceName(hostname, port);
    HelixParticipant helixParticipant = new HelixParticipant(new ClusterMapConfig(new VerifiableProperties(props)), helixManagerFactory);
    helixParticipant.initialize(hostname, port, Collections.EMPTY_LIST);
    HelixManager helixManager = helixManagerFactory.getZKHelixManager(null, null, null, null);
    HelixAdmin helixAdmin = helixManager.getClusterManagmentTool();
    InstanceConfig instanceConfig = new InstanceConfig("someInstanceId");
    helixAdmin.setInstanceConfig(clusterName, instanceName, instanceConfig);
    // Make sure the current sealedReplicas list is null
    List<String> sealedReplicas = ClusterMapUtils.getSealedReplicas(instanceConfig);
    assertNull("sealedReplicas is not null", sealedReplicas);
    String listName = "sealedReplicas";
    // Check that invoking setReplicaSealedState with a non-AmbryReplica ReplicaId throws an IllegalArgumentException
    ReplicaId notAmbryReplica = createMockNotAmbryReplica(partitionIdStr);
    try {
        helixParticipant.setReplicaSealedState(notAmbryReplica, true);
        fail("Expected an IllegalArgumentException here");
    } catch (IllegalArgumentException e) {
    // Expected exception
    }
    // Check that invoking setReplicaSealedState adds the partition to the list of sealed replicas
    helixParticipant.setReplicaSealedState(replicaId, true);
    sealedReplicas = ClusterMapUtils.getSealedReplicas(helixAdmin.getInstanceConfig(clusterName, instanceName));
    listIsExpectedSize(sealedReplicas, 1, listName);
    assertTrue(sealedReplicas.contains(partitionIdStr));
    // Seal another replicaId
    helixParticipant.setReplicaSealedState(replicaId2, true);
    sealedReplicas = ClusterMapUtils.getSealedReplicas(helixAdmin.getInstanceConfig(clusterName, instanceName));
    listIsExpectedSize(sealedReplicas, 2, listName);
    assertTrue(sealedReplicas.contains(partitionIdStr2));
    assertTrue(sealedReplicas.contains(partitionIdStr));
    // Check that sealed replica list doesn't take duplicates (and that dups are detected by partitionId comparison, not
    // replicaId object comparison
    ReplicaId dup = createMockAmbryReplica(partitionIdStr);
    helixParticipant.setReplicaSealedState(dup, true);
    helixParticipant.setReplicaSealedState(replicaId2, true);
    sealedReplicas = ClusterMapUtils.getSealedReplicas(helixAdmin.getInstanceConfig(clusterName, instanceName));
    listIsExpectedSize(sealedReplicas, 2, listName);
    assertTrue(sealedReplicas.contains(partitionIdStr2));
    assertTrue(sealedReplicas.contains(partitionIdStr));
    // Check that invoking setReplicaSealedState with isSealed == false removes partition from list of sealed replicas
    helixParticipant.setReplicaSealedState(replicaId, false);
    sealedReplicas = ClusterMapUtils.getSealedReplicas(helixAdmin.getInstanceConfig(clusterName, instanceName));
    listIsExpectedSize(sealedReplicas, 1, listName);
    assertTrue(sealedReplicas.contains(partitionIdStr2));
    assertFalse(sealedReplicas.contains(partitionIdStr));
    // Removing a replicaId that's already been removed doesn't hurt anything
    helixParticipant.setReplicaSealedState(replicaId, false);
    sealedReplicas = ClusterMapUtils.getSealedReplicas(helixAdmin.getInstanceConfig(clusterName, instanceName));
    listIsExpectedSize(sealedReplicas, 1, listName);
    // Removing all replicas yields expected behavior (and removal works by partitionId, not replicaId itself)
    dup = createMockAmbryReplica(partitionIdStr2);
    helixParticipant.setReplicaSealedState(dup, false);
    sealedReplicas = ClusterMapUtils.getSealedReplicas(helixAdmin.getInstanceConfig(clusterName, instanceName));
    listIsExpectedSize(sealedReplicas, 0, listName);
}

Example 15 with HelixAdmin

use of org.apache.helix.HelixAdmin in project helix by apache.

the class TestHelixAdminScenariosRest method testGetInstances.

@Test
public void testGetInstances() throws IOException {
    final String clusterName = "TestTagAwareness_testGetResources";
    final String[] TAGS = { "tag1", "tag2" };
    final String URL_BASE = "http://localhost:" + ADMIN_PORT + "/clusters/" + clusterName + "/instances";
    _gSetupTool.addCluster(clusterName, true);
    HelixAdmin admin = _gSetupTool.getClusterManagementTool();
    // Add 4 participants, each with differint tag characteristics
    InstanceConfig instance1 = new InstanceConfig("localhost_1");
    instance1.addTag(TAGS[0]);
    admin.addInstance(clusterName, instance1);
    InstanceConfig instance2 = new InstanceConfig("localhost_2");
    instance2.addTag(TAGS[1]);
    admin.addInstance(clusterName, instance2);
    InstanceConfig instance3 = new InstanceConfig("localhost_3");
    instance3.addTag(TAGS[0]);
    instance3.addTag(TAGS[1]);
    admin.addInstance(clusterName, instance3);
    InstanceConfig instance4 = new InstanceConfig("localhost_4");
    admin.addInstance(clusterName, instance4);
    // Now make a REST call for all resources
    Reference resourceRef = new Reference(URL_BASE);
    Request request = new Request(Method.GET, resourceRef);
    Response response = _gClient.handle(request);
    ListInstancesWrapper responseWrapper = ClusterRepresentationUtil.JsonToObject(ListInstancesWrapper.class, response.getEntityAsText());
    Map<String, List<String>> tagInfo = responseWrapper.tagInfo;
    // Ensure tag ownership is reported correctly
    Assert.assertTrue(tagInfo.containsKey(TAGS[0]));
    Assert.assertTrue(tagInfo.containsKey(TAGS[1]));
    Assert.assertTrue(tagInfo.get(TAGS[0]).contains("localhost_1"));
    Assert.assertFalse(tagInfo.get(TAGS[0]).contains("localhost_2"));
    Assert.assertTrue(tagInfo.get(TAGS[0]).contains("localhost_3"));
    Assert.assertFalse(tagInfo.get(TAGS[0]).contains("localhost_4"));
    Assert.assertFalse(tagInfo.get(TAGS[1]).contains("localhost_1"));
    Assert.assertTrue(tagInfo.get(TAGS[1]).contains("localhost_2"));
    Assert.assertTrue(tagInfo.get(TAGS[1]).contains("localhost_3"));
    Assert.assertFalse(tagInfo.get(TAGS[1]).contains("localhost_4"));
}