Examples with InstanceConfig org.apache.helix.model.InstanceConfig used on opensource projects

Example 56 with InstanceConfig

use of org.apache.helix.model.InstanceConfig in project helix by apache.

the class TestAlertingRebalancerFailure method setInstanceEnable.

private void setInstanceEnable(String instanceName, boolean enabled, ConfigAccessor configAccessor) {
    InstanceConfig instanceConfig = configAccessor.getInstanceConfig(CLUSTER_NAME, instanceName);
    instanceConfig.setInstanceEnabled(enabled);
    configAccessor.setInstanceConfig(CLUSTER_NAME, instanceName, instanceConfig);
}

Example 57 with InstanceConfig

use of org.apache.helix.model.InstanceConfig in project helix by apache.

the class TestDrop method testDropErrorPartitionFailedAutoIS.

@Test
public void testDropErrorPartitionFailedAutoIS() throws Exception {
    // Logger.getRootLogger().setLevel(Level.INFO);
    String className = TestHelper.getTestClassName();
    String methodName = TestHelper.getTestMethodName();
    String clusterName = className + "_" + methodName;
    final int n = 5;
    System.out.println("START " + clusterName + " at " + new Date(System.currentTimeMillis()));
    MockParticipantManager[] participants = new MockParticipantManager[n];
    // participant port
    TestHelper.setupCluster(// participant port
    clusterName, // participant port
    ZK_ADDR, // participant port
    12918, // participant name prefix
    "localhost", // resource name prefix
    "TestDB", // resources
    1, // partitions per resource
    8, // number of nodes
    n, // replicas
    3, "MasterSlave", // do rebalance
    true);
    // start controller
    ClusterControllerManager controller = new ClusterControllerManager(ZK_ADDR, clusterName, "controller_0");
    controller.syncStart();
    // start participants
    Map<String, Set<String>> errTransitions = new HashMap<String, Set<String>>();
    errTransitions.put("SLAVE-MASTER", TestHelper.setOf("TestDB0_4"));
    errTransitions.put("ERROR-DROPPED", TestHelper.setOf("TestDB0_4"));
    for (int i = 0; i < n; i++) {
        String instanceName = "localhost_" + (12918 + i);
        if (i == 0) {
            participants[i] = new MockParticipantManager(ZK_ADDR, clusterName, instanceName);
            participants[i].setTransition(new ErrTransition(errTransitions));
        } else {
            participants[i] = new MockParticipantManager(ZK_ADDR, clusterName, instanceName);
        }
        participants[i].syncStart();
    }
    Map<String, Map<String, String>> errStateMap = new HashMap<>();
    errStateMap.put("TestDB0", new HashMap<String, String>());
    errStateMap.get("TestDB0").put("TestDB0_4", "localhost_12918");
    HelixClusterVerifier verifier = new BestPossibleExternalViewVerifier.Builder(clusterName).setZkAddr(ZK_ADDR).setErrStates(errStateMap).build();
    Assert.assertTrue(verifier.verify());
    // drop resource containing error partitions should invoke error->dropped transition
    // if error happens during error->dropped transition, partition should be disabled
    ClusterSetup.processCommandLineArgs(new String[] { "--zkSvr", ZK_ADDR, "--dropResource", clusterName, "TestDB0" });
    Thread.sleep(100);
    // make sure TestDB0_4 stay in ERROR state and is disabled
    Assert.assertTrue(verifier.verify());
    ZKHelixDataAccessor accessor = new ZKHelixDataAccessor(clusterName, new ZkBaseDataAccessor<ZNRecord>(_gZkClient));
    PropertyKey.Builder keyBuilder = accessor.keyBuilder();
    InstanceConfig config = accessor.getProperty(keyBuilder.instanceConfig("localhost_12918"));
    List<String> disabledPartitions = config.getDisabledPartitions();
    // System.out.println("disabledPartitions: " + disabledPartitions);
    Assert.assertEquals(disabledPartitions.size(), 1, "TestDB0_4 should be disabled");
    Assert.assertEquals(disabledPartitions.get(0), "TestDB0_4");
    // ExteranlView should have TestDB0_4->localhost_12918_>ERROR
    Thread.sleep(2000);
    ExternalView ev = accessor.getProperty(keyBuilder.externalView("TestDB0"));
    Set<String> partitions = ev.getPartitionSet();
    Assert.assertEquals(partitions.size(), 1, "Should have TestDB0_4->localhost_12918->ERROR");
    String errPartition = partitions.iterator().next();
    Assert.assertEquals(errPartition, "TestDB0_4");
    Map<String, String> stateMap = ev.getStateMap(errPartition);
    Assert.assertEquals(stateMap.size(), 1);
    Assert.assertEquals(stateMap.keySet().iterator().next(), "localhost_12918");
    Assert.assertEquals(stateMap.get("localhost_12918"), HelixDefinedState.ERROR.name());
    // localhost_12918 should have TestDB0_4 in ERROR state
    CurrentState cs = accessor.getProperty(keyBuilder.currentState(participants[0].getInstanceName(), participants[0].getSessionId(), "TestDB0"));
    Map<String, String> partitionStateMap = cs.getPartitionStateMap();
    Assert.assertEquals(partitionStateMap.size(), 1);
    Assert.assertEquals(partitionStateMap.keySet().iterator().next(), "TestDB0_4");
    Assert.assertEquals(partitionStateMap.get("TestDB0_4"), HelixDefinedState.ERROR.name());
    // all other participants should have cleaned up empty current state
    for (int i = 1; i < n; i++) {
        String instanceName = participants[i].getInstanceName();
        String sessionId = participants[i].getSessionId();
        Assert.assertNull(accessor.getProperty(keyBuilder.currentState(instanceName, sessionId, "TestDB0")));
    }
    // clean up
    controller.syncStop();
    for (int i = 0; i < n; i++) {
        participants[i].syncStop();
    }
    System.out.println("END " + clusterName + " at " + new Date(System.currentTimeMillis()));
}

Example 58 with InstanceConfig

use of org.apache.helix.model.InstanceConfig in project helix by apache.

the class TestDelayedAutoRebalanceWithDisabledInstance method enableInstance.

private void enableInstance(String instance, boolean enabled) {
    // Disable one node, no partition should be moved.
    long currentTime = System.currentTimeMillis();
    _setupTool.getClusterManagementTool().enableInstance(CLUSTER_NAME, instance, enabled);
    InstanceConfig instanceConfig = _configAccessor.getInstanceConfig(CLUSTER_NAME, instance);
    Assert.assertEquals(instanceConfig.getInstanceEnabled(), enabled);
    Assert.assertTrue(instanceConfig.getInstanceEnabledTime() >= currentTime);
    Assert.assertTrue(instanceConfig.getInstanceEnabledTime() <= currentTime + 100);
}

Example 59 with InstanceConfig

use of org.apache.helix.model.InstanceConfig in project ambry by linkedin.

the class HelixBootstrapUpgradeToolTest method testEnablePartitionAdminOp.

/**
 * Test that partition is correctly enabled on given node. The partition is first disabled and then enabled.
 * @throws Exception
 */
@Test
public void testEnablePartitionAdminOp() throws Exception {
    assumeTrue(!dcStr.equals("DC1") && !dcStr.equals("DC0"));
    String clusterName = CLUSTER_NAME_PREFIX + CLUSTER_NAME_IN_STATIC_CLUSTER_MAP;
    // Test regular bootstrap.
    long expectedResourceCount = (testPartitionLayout.getPartitionLayout().getPartitionCount() - 1) / DEFAULT_MAX_PARTITIONS_PER_RESOURCE + 1;
    writeBootstrapOrUpgrade(expectedResourceCount, false);
    int totalPartitionCount = testPartitionLayout.getPartitionCount();
    // Randomly pick a partition to disable/enable
    Partition testPartition = (Partition) testPartitionLayout.getPartitionLayout().getPartitions(null).get(RANDOM.nextInt(totalPartitionCount));
    // Randomly pick a replica from this partition
    List<ReplicaId> replicaIds = testPartition.getReplicaIds();
    DataNodeId dataNodeId = replicaIds.get(RANDOM.nextInt(replicaIds.size())).getDataNodeId();
    // Disable partition on chosen node
    HelixBootstrapUpgradeUtil.controlPartitionState(hardwareLayoutPath, partitionLayoutPath, zkLayoutPath, CLUSTER_NAME_PREFIX, dataNodeId.getDatacenterName(), dataNodeId.getHostname(), dataNodeId.getPort(), DisablePartition, testPartition.toPathString());
    // Verify the InstanceConfig is changed only in MapFields (Disabled partition is added to this field)
    ZkInfo zkInfo = dcsToZkInfo.get(dataNodeId.getDatacenterName());
    ZKHelixAdmin admin = new ZKHelixAdmin("localhost:" + zkInfo.getPort());
    InstanceConfig currentInstanceConfig = admin.getInstanceConfig(clusterName, getInstanceName(dataNodeId));
    assertTrue("There should be additional string in InstanceConfig due to disabling partition", currentInstanceConfig.getRecord().getMapFields().keySet().stream().anyMatch(k -> !k.startsWith("/mnt")));
    // Verify given partition is indeed disabled on specified node
    String resourceName = getResourceNameOfPartition(admin, clusterName, testPartition.toPathString());
    List<String> disabledPartitions = currentInstanceConfig.getDisabledPartitions(resourceName);
    assertEquals("Disabled partition is not expected", Collections.singletonList(testPartition.toPathString()), disabledPartitions);
    // Enable the same partition on same node
    HelixBootstrapUpgradeUtil.controlPartitionState(hardwareLayoutPath, partitionLayoutPath, zkLayoutPath, CLUSTER_NAME_PREFIX, dataNodeId.getDatacenterName(), dataNodeId.getHostname(), dataNodeId.getPort(), EnablePartition, testPartition.toPathString());
    // Verify instanceConfig has been updated (disabled partition is removed)
    currentInstanceConfig = admin.getInstanceConfig(clusterName, getInstanceName(dataNodeId));
    assertFalse("There shouldn't be any additional string in InstanceConfig", currentInstanceConfig.getRecord().getMapFields().keySet().stream().anyMatch(k -> !k.startsWith("/mnt")));
    // Verify there is no disabled partition
    assertNull("There shouldn't be any disabled partition", currentInstanceConfig.getDisabledPartitions(resourceName));
}

Example 60 with InstanceConfig

use of org.apache.helix.model.InstanceConfig in project ambry by linkedin.

the class HelixBootstrapUpgradeToolTest method testListSealedPartitions.

/**
 * Test listing sealed partitions in Helix cluster.
 * @throws Exception
 */
@Test
public void testListSealedPartitions() throws Exception {
    assumeTrue(!dcStr.equals("DC1"));
    String clusterName = CLUSTER_NAME_PREFIX + CLUSTER_NAME_IN_STATIC_CLUSTER_MAP;
    ClusterMapConfig clusterMapConfig = getClusterMapConfig(clusterName, "DC1", null);
    InstanceConfigToDataNodeConfigAdapter.Converter instanceConfigConverter = new InstanceConfigToDataNodeConfigAdapter.Converter(clusterMapConfig);
    // Test regular bootstrap
    long expectedResourceCount = (testPartitionLayout.getPartitionLayout().getPartitionCount() - 1) / DEFAULT_MAX_PARTITIONS_PER_RESOURCE + 1;
    writeBootstrapOrUpgrade(expectedResourceCount, false);
    Set<String> sealedPartitions = HelixBootstrapUpgradeUtil.listSealedPartition(hardwareLayoutPath, partitionLayoutPath, zkLayoutPath, CLUSTER_NAME_PREFIX, dcStr, dataNodeConfigSourceType);
    assertEquals("Sealed partition set should be empty initially", Collections.emptySet(), sealedPartitions);
    // randomly choose 20 partitions to mark as sealed
    int[] intArray = new Random().ints(0, 100).distinct().limit(20).toArray();
    Set<String> selectedSealedPartitionSet = new HashSet<>();
    for (int id : intArray) {
        selectedSealedPartitionSet.add(String.valueOf(id));
    }
    // update the sealed lists in Helix
    for (ZkInfo zkInfo : dcsToZkInfo.values()) {
        ZKHelixAdmin admin = new ZKHelixAdmin("localhost:" + zkInfo.getPort());
        PropertyStoreToDataNodeConfigAdapter propertyStoreAdapter = dataNodeConfigSourceType == DataNodeConfigSourceType.INSTANCE_CONFIG ? null : new PropertyStoreToDataNodeConfigAdapter("localhost:" + zkInfo.getPort(), clusterMapConfig);
        for (String instanceName : admin.getInstancesInCluster(clusterName)) {
            DataNodeConfig dataNodeConfig = dataNodeConfigSourceType == DataNodeConfigSourceType.INSTANCE_CONFIG ? instanceConfigConverter.convert(admin.getInstanceConfig(clusterName, instanceName)) : propertyStoreAdapter.get(instanceName);
            Set<String> localReplicas = new HashSet<>();
            for (DataNodeConfig.DiskConfig diskConfig : dataNodeConfig.getDiskConfigs().values()) {
                localReplicas.addAll(diskConfig.getReplicaConfigs().keySet());
            }
            // derive the intersection of localReplicas set and selectedSealedPartitionSet
            localReplicas.retainAll(selectedSealedPartitionSet);
            if (dataNodeConfigSourceType == DataNodeConfigSourceType.INSTANCE_CONFIG) {
                InstanceConfig instanceConfig = admin.getInstanceConfig(clusterName, instanceName);
                instanceConfig.getRecord().setListField(SEALED_STR, new ArrayList<>(localReplicas));
                admin.setInstanceConfig(clusterName, instanceName, instanceConfig);
            } else {
                dataNodeConfig.getSealedReplicas().addAll(localReplicas);
                propertyStoreAdapter.set(dataNodeConfig);
            }
        }
    }
    // query sealed partition in Helix again
    sealedPartitions = HelixBootstrapUpgradeUtil.listSealedPartition(hardwareLayoutPath, partitionLayoutPath, zkLayoutPath, CLUSTER_NAME_PREFIX, dcStr, dataNodeConfigSourceType);
    assertEquals("Mismatch in sealed partition set", selectedSealedPartitionSet, sealedPartitions);
}