Examples with ZKHelixAdmin org.apache.helix.manager.zk.ZKHelixAdmin used on opensource projects

Example 26 with ZKHelixAdmin

use of org.apache.helix.manager.zk.ZKHelixAdmin in project ambry by linkedin.

the class HelixBootstrapUpgradeToolTest method testListSealedPartitions.

/**
 * Test listing sealed partitions in Helix cluster.
 * @throws Exception
 */
@Test
public void testListSealedPartitions() throws Exception {
    assumeTrue(!dcStr.equals("DC1"));
    String clusterName = CLUSTER_NAME_PREFIX + CLUSTER_NAME_IN_STATIC_CLUSTER_MAP;
    ClusterMapConfig clusterMapConfig = getClusterMapConfig(clusterName, "DC1", null);
    InstanceConfigToDataNodeConfigAdapter.Converter instanceConfigConverter = new InstanceConfigToDataNodeConfigAdapter.Converter(clusterMapConfig);
    // Test regular bootstrap
    long expectedResourceCount = (testPartitionLayout.getPartitionLayout().getPartitionCount() - 1) / DEFAULT_MAX_PARTITIONS_PER_RESOURCE + 1;
    writeBootstrapOrUpgrade(expectedResourceCount, false);
    Set<String> sealedPartitions = HelixBootstrapUpgradeUtil.listSealedPartition(hardwareLayoutPath, partitionLayoutPath, zkLayoutPath, CLUSTER_NAME_PREFIX, dcStr, dataNodeConfigSourceType);
    assertEquals("Sealed partition set should be empty initially", Collections.emptySet(), sealedPartitions);
    // randomly choose 20 partitions to mark as sealed
    int[] intArray = new Random().ints(0, 100).distinct().limit(20).toArray();
    Set<String> selectedSealedPartitionSet = new HashSet<>();
    for (int id : intArray) {
        selectedSealedPartitionSet.add(String.valueOf(id));
    }
    // update the sealed lists in Helix
    for (ZkInfo zkInfo : dcsToZkInfo.values()) {
        ZKHelixAdmin admin = new ZKHelixAdmin("localhost:" + zkInfo.getPort());
        PropertyStoreToDataNodeConfigAdapter propertyStoreAdapter = dataNodeConfigSourceType == DataNodeConfigSourceType.INSTANCE_CONFIG ? null : new PropertyStoreToDataNodeConfigAdapter("localhost:" + zkInfo.getPort(), clusterMapConfig);
        for (String instanceName : admin.getInstancesInCluster(clusterName)) {
            DataNodeConfig dataNodeConfig = dataNodeConfigSourceType == DataNodeConfigSourceType.INSTANCE_CONFIG ? instanceConfigConverter.convert(admin.getInstanceConfig(clusterName, instanceName)) : propertyStoreAdapter.get(instanceName);
            Set<String> localReplicas = new HashSet<>();
            for (DataNodeConfig.DiskConfig diskConfig : dataNodeConfig.getDiskConfigs().values()) {
                localReplicas.addAll(diskConfig.getReplicaConfigs().keySet());
            }
            // derive the intersection of localReplicas set and selectedSealedPartitionSet
            localReplicas.retainAll(selectedSealedPartitionSet);
            if (dataNodeConfigSourceType == DataNodeConfigSourceType.INSTANCE_CONFIG) {
                InstanceConfig instanceConfig = admin.getInstanceConfig(clusterName, instanceName);
                instanceConfig.getRecord().setListField(SEALED_STR, new ArrayList<>(localReplicas));
                admin.setInstanceConfig(clusterName, instanceName, instanceConfig);
            } else {
                dataNodeConfig.getSealedReplicas().addAll(localReplicas);
                propertyStoreAdapter.set(dataNodeConfig);
            }
        }
    }
    // query sealed partition in Helix again
    sealedPartitions = HelixBootstrapUpgradeUtil.listSealedPartition(hardwareLayoutPath, partitionLayoutPath, zkLayoutPath, CLUSTER_NAME_PREFIX, dcStr, dataNodeConfigSourceType);
    assertEquals("Mismatch in sealed partition set", selectedSealedPartitionSet, sealedPartitions);
}

Example 27 with ZKHelixAdmin

use of org.apache.helix.manager.zk.ZKHelixAdmin in project ambry by linkedin.

the class VcrTestUtil method populateZkInfoAndStartController.

/**
 * Populate info on ZooKeeper server and start {@link HelixControllerManager}.
 * @param zkConnectString zk connect string to zk server.
 * @param vcrClusterName the vcr cluster name.
 * @param clusterMap the {@link ClusterMap} to use.
 * @param vcrHelixStateModel the state model to use for helix cluster events.
 * @return the created {@link HelixControllerManager}.
 */
public static HelixControllerManager populateZkInfoAndStartController(String zkConnectString, String vcrClusterName, ClusterMap clusterMap, String vcrHelixStateModel) {
    HelixZkClient zkClient = DedicatedZkClientFactory.getInstance().buildZkClient(new HelixZkClient.ZkConnectionConfig(zkConnectString), new HelixZkClient.ZkClientConfig());
    try {
        zkClient.setZkSerializer(new ZNRecordSerializer());
        ClusterSetup clusterSetup = new ClusterSetup(zkClient);
        clusterSetup.addCluster(vcrClusterName, true);
        HelixAdmin admin = new ZKHelixAdmin(zkClient);
        // set ALLOW_PARTICIPANT_AUTO_JOIN
        HelixConfigScope configScope = new HelixConfigScopeBuilder(HelixConfigScope.ConfigScopeProperty.CLUSTER).forCluster(vcrClusterName).build();
        Map<String, String> helixClusterProperties = new HashMap<>();
        helixClusterProperties.put(ZKHelixManager.ALLOW_PARTICIPANT_AUTO_JOIN, String.valueOf(true));
        admin.setConfig(configScope, helixClusterProperties);
        // set PersistBestPossibleAssignment
        ConfigAccessor configAccessor = new ConfigAccessor(zkClient);
        ClusterConfig clusterConfig = configAccessor.getClusterConfig(vcrClusterName);
        clusterConfig.setPersistBestPossibleAssignment(true);
        configAccessor.setClusterConfig(vcrClusterName, clusterConfig);
        FullAutoModeISBuilder builder = new FullAutoModeISBuilder(helixResource);
        builder.setStateModel(vcrHelixStateModel);
        for (PartitionId partitionId : clusterMap.getAllPartitionIds(null)) {
            builder.add(partitionId.toPathString());
        }
        builder.setMinActiveReplica(MIN_ACTIVE_REPLICAS);
        builder.setRebalanceDelay((int) REBALANCE_DELAY_MS);
        builder.setRebalancerClass(DelayedAutoRebalancer.class.getName());
        builder.setRebalanceStrategy(CrushEdRebalanceStrategy.class.getName());
        IdealState idealState = builder.build();
        admin.addResource(vcrClusterName, helixResource, idealState);
        admin.rebalance(vcrClusterName, helixResource, NUM_REPLICAS, "", "");
        HelixControllerManager helixControllerManager = new HelixControllerManager(zkConnectString, vcrClusterName);
        helixControllerManager.syncStart();
        return helixControllerManager;
    } finally {
        zkClient.close();
    }
}

Example 28 with ZKHelixAdmin

use of org.apache.helix.manager.zk.ZKHelixAdmin in project ambry by linkedin.

the class HelixParticipantTest method testGetAndSetReplicaStoppedState.

/**
 * Tests setReplicaStoppedState method for {@link HelixParticipant}
 * @throws Exception
 */
@Test
public void testGetAndSetReplicaStoppedState() throws Exception {
    // setup HelixParticipant, HelixParticipantDummy and dependencies
    ClusterMapConfig clusterMapConfig = new ClusterMapConfig(new VerifiableProperties(props));
    String instanceName = ClusterMapUtils.getInstanceName("localhost", clusterMapConfig.clusterMapPort);
    HelixParticipant helixParticipant = new HelixParticipant(clusterMapConfig, new HelixFactory(), new MetricRegistry(), getDefaultZkConnectStr(clusterMapConfig), true);
    ZKHelixAdmin helixAdmin = new ZKHelixAdmin("localhost:" + zkInfo.getPort());
    DataNodeConfig dataNodeConfig = getDataNodeConfigInHelix(helixAdmin, instanceName);
    // Make sure the current stoppedReplicas list is empty
    List<String> stoppedReplicas = helixParticipant.getStoppedReplicas();
    assertEquals("stoppedReplicas list should be empty", Collections.emptyList(), stoppedReplicas);
    String listName = "stoppedReplicas list";
    Set<String> localPartitionNames = new HashSet<>();
    dataNodeConfig.getDiskConfigs().values().forEach(diskConfig -> localPartitionNames.addAll(diskConfig.getReplicaConfigs().keySet()));
    String[] partitionIds = new String[3];
    for (int i = 0; i < partitionIds.length; ++i) {
        partitionIds[i] = localPartitionNames.iterator().next();
        localPartitionNames.remove(partitionIds[i]);
    }
    ReplicaId replicaId1 = createMockAmbryReplica(partitionIds[0]);
    ReplicaId replicaId2 = createMockAmbryReplica(partitionIds[1]);
    ReplicaId replicaId3 = createMockAmbryReplica(partitionIds[2]);
    // Check that invoking setReplicaStoppedState with a non-AmbryReplica ReplicaId throws an IllegalArgumentException
    ReplicaId nonAmbryReplica = createMockNotAmbryReplica(partitionIds[1]);
    try {
        helixParticipant.setReplicaStoppedState(Collections.singletonList(nonAmbryReplica), true);
        fail("Expected an IllegalArgumentException here");
    } catch (IllegalArgumentException e) {
    // expected. Nothing to do.
    }
    // Check that invoking setReplicaStoppedState adds the replicaId1, replicaId2 to the list of stopped replicas
    helixParticipant.setReplicaStoppedState(Arrays.asList(replicaId1, replicaId2), true);
    stoppedReplicas = helixParticipant.getStoppedReplicas();
    listIsExpectedSize(stoppedReplicas, 2, listName);
    assertTrue(stoppedReplicas.contains(replicaId1.getPartitionId().toPathString()));
    assertTrue(stoppedReplicas.contains(replicaId2.getPartitionId().toPathString()));
    // Invoke setReplicaStoppedState to add replicaId1, replicaId2 again, should be no-op
    helixParticipant.setReplicaStoppedState(Arrays.asList(replicaId1, replicaId2), true);
    listIsExpectedSize(helixParticipant.getStoppedReplicas(), 2, listName);
    // Add replicaId1 again as well as replicaId3 to ensure new replicaId is correctly added and no duplicates in the stopped list
    helixParticipant.setReplicaStoppedState(Arrays.asList(replicaId1, replicaId3), true);
    stoppedReplicas = helixParticipant.getStoppedReplicas();
    listIsExpectedSize(stoppedReplicas, 3, listName);
    assertTrue(stoppedReplicas.contains(replicaId1.getPartitionId().toPathString()));
    assertTrue(stoppedReplicas.contains(replicaId2.getPartitionId().toPathString()));
    assertTrue(stoppedReplicas.contains(replicaId3.getPartitionId().toPathString()));
    // Check that invoking setReplicaStoppedState with markStop == false removes replicaId1, replicaId2 from stopped list
    helixParticipant.setReplicaStoppedState(Arrays.asList(replicaId1, replicaId2), false);
    stoppedReplicas = helixParticipant.getStoppedReplicas();
    listIsExpectedSize(stoppedReplicas, 1, listName);
    assertTrue(stoppedReplicas.contains(replicaId3.getPartitionId().toPathString()));
    assertFalse(stoppedReplicas.contains(replicaId2.getPartitionId().toPathString()));
    assertFalse(stoppedReplicas.contains(replicaId1.getPartitionId().toPathString()));
    // Removing replicaIds which have already been removed doesn't hurt anything and will not update InstanceConfig in Helix
    helixParticipant.setReplicaStoppedState(Arrays.asList(replicaId1, replicaId2), false);
    stoppedReplicas = helixParticipant.getStoppedReplicas();
    listIsExpectedSize(stoppedReplicas, 1, listName);
    assertTrue(stoppedReplicas.contains(replicaId3.getPartitionId().toPathString()));
    // Removing all replicas (including replica not in the list) yields expected behavior
    helixParticipant.setReplicaStoppedState(Arrays.asList(replicaId2, replicaId3), false);
    stoppedReplicas = helixParticipant.getStoppedReplicas();
    listIsExpectedSize(stoppedReplicas, 0, listName);
    helixAdmin.close();
}

Example 29 with ZKHelixAdmin

use of org.apache.helix.manager.zk.ZKHelixAdmin in project ambry by linkedin.

the class HelixVcrUtil method controlResource.

/**
 * Enable or disable a resource in dest cluster.
 * @param destZkString the cluster's zk string
 * @param destClusterName the cluster's name
 * @param resourceName the resource to enable/disable
 * @param enable enable the resource if true
 */
static void controlResource(String destZkString, String destClusterName, String resourceName, boolean enable) {
    HelixZkClient destZkClient = getHelixZkClient(destZkString);
    HelixAdmin destAdmin = new ZKHelixAdmin(destZkClient);
    destAdmin.enableResource(destClusterName, resourceName, enable);
}

Example 30 with ZKHelixAdmin

use of org.apache.helix.manager.zk.ZKHelixAdmin in project ambry by linkedin.

the class HelixVcrUtil method updateResourceIdealState.

/**
 * Update the resources in the destination cluster with the new IdealState settings.
 * @param destZkString the destination Zookeeper server string.
 * @param destClusterName the destination cluster name.
 * @param dryRun run without actual change.
 */
static void updateResourceIdealState(String destZkString, String destClusterName, VcrHelixConfig config, boolean dryRun) {
    HelixAdmin destAdmin = new ZKHelixAdmin(destZkString);
    Set<String> destResources = new HashSet<>(destAdmin.getResourcesInCluster(destClusterName));
    for (String resource : destResources) {
        IdealState currentIdealState = destAdmin.getResourceIdealState(destClusterName, resource);
        IdealState newIdealState = buildIdealState(resource, currentIdealState.getPartitionSet(), config.getIdealStateConfigFields());
        if (dryRun) {
            logger.info("Will update {} to new ideal state {}", resource, newIdealState.toString());
        } else {
            destAdmin.setResourceIdealState(destClusterName, resource, newIdealState);
            logger.info("Updated the ideal state for resource {} ", resource);
            destAdmin.rebalance(destClusterName, resource, config.getIdealStateConfigFields().getNumReplicas(), "", "");
            logger.info("Rebalanced resource with REPLICA_NUM: {}", resource, config.getIdealStateConfigFields().getNumReplicas());
        }
    }
}