Examples with HelixAdmin org.apache.helix.HelixAdmin used on opensource projects

Example 81 with HelixAdmin

use of org.apache.helix.HelixAdmin in project helix by apache.

the class TestAddBuiltInStateModelDef method test.

@Test
public void test() throws Exception {
    String className = TestHelper.getTestClassName();
    String methodName = TestHelper.getTestMethodName();
    String clusterName = className + "_" + methodName;
    System.out.println("START " + clusterName + " at " + new Date(System.currentTimeMillis()));
    HelixAdmin admin = new ZKHelixAdmin(_gZkClient);
    admin.addCluster(clusterName);
    admin.addStateModelDef(clusterName, BuiltInStateModelDefinitions.MasterSlave.getStateModelDefinition().getId(), BuiltInStateModelDefinitions.MasterSlave.getStateModelDefinition());
    ClusterControllerManager controller = new ClusterControllerManager(ZK_ADDR, clusterName);
    controller.syncStart();
    // controller shall create all built-in state model definitions
    final BaseDataAccessor<ZNRecord> baseAccessor = new ZkBaseDataAccessor<ZNRecord>(_gZkClient);
    final PropertyKey.Builder keyBuilder = new PropertyKey.Builder(clusterName);
    boolean ret = TestHelper.verify(new TestHelper.Verifier() {

        @Override
        public boolean verify() throws Exception {
            for (BuiltInStateModelDefinitions def : BuiltInStateModelDefinitions.values()) {
                String path = keyBuilder.stateModelDef(def.getStateModelDefinition().getId()).getPath();
                boolean exist = baseAccessor.exists(path, 0);
                if (!exist) {
                    return false;
                }
                // make sure MasterSlave is not over-written
                if (def == BuiltInStateModelDefinitions.MasterSlave) {
                    Stat stat = new Stat();
                    baseAccessor.get(path, stat, 0);
                    if (stat.getVersion() != 0) {
                        return false;
                    }
                }
            }
            return true;
        }
    }, 10 * 1000);
    Assert.assertTrue(ret);
    controller.syncStop();
    System.out.println("END " + clusterName + " at " + new Date(System.currentTimeMillis()));
}

Example 82 with HelixAdmin

use of org.apache.helix.HelixAdmin in project helix by apache.

the class YAMLClusterSetup method setupCluster.

/**
 * Set up the cluster by parsing a YAML file.
 * @param input InputStream representing the file
 * @return ClusterConfig Java wrapper of the configuration file
 */
public YAMLClusterConfig setupCluster(InputStream input) {
    // parse the YAML
    Yaml yaml = new Yaml();
    YAMLClusterConfig cfg = yaml.loadAs(input, YAMLClusterConfig.class);
    // create the cluster
    HelixAdmin helixAdmin = new ZKHelixAdmin(_zkAddress);
    if (cfg.clusterName == null) {
        throw new HelixException("Cluster name is required!");
    }
    helixAdmin.addCluster(cfg.clusterName);
    // add each participant
    if (cfg.participants != null) {
        for (ParticipantConfig participant : cfg.participants) {
            helixAdmin.addInstance(cfg.clusterName, getInstanceCfg(participant));
        }
    }
    // add each resource
    if (cfg.resources != null) {
        for (ResourceConfig resource : cfg.resources) {
            if (resource.name == null) {
                throw new HelixException("Resources must be named!");
            }
            if (resource.stateModel == null || resource.stateModel.name == null) {
                throw new HelixException("Resource must specify a named state model!");
            }
            // if states is null, assume using a built-in or already-added state model
            if (resource.stateModel.states != null) {
                StateModelDefinition stateModelDef = getStateModelDef(resource.stateModel, resource.constraints);
                helixAdmin.addStateModelDef(cfg.clusterName, resource.stateModel.name, stateModelDef);
            } else {
                StateModelDefinition stateModelDef = null;
                if (resource.stateModel.name.equals("MasterSlave")) {
                    stateModelDef = new StateModelDefinition(StateModelConfigGenerator.generateConfigForMasterSlave());
                } else if (resource.stateModel.name.equals("OnlineOffline")) {
                    stateModelDef = new StateModelDefinition(StateModelConfigGenerator.generateConfigForOnlineOffline());
                } else if (resource.stateModel.name.equals("LeaderStandby")) {
                    stateModelDef = new StateModelDefinition(StateModelConfigGenerator.generateConfigForLeaderStandby());
                }
                if (stateModelDef != null) {
                    try {
                        helixAdmin.addStateModelDef(cfg.clusterName, resource.stateModel.name, stateModelDef);
                    } catch (HelixException e) {
                        LOG.warn("State model definition " + resource.stateModel.name + " could not be added.");
                    }
                }
            }
            int partitions = 1;
            int replicas = 1;
            if (resource.partitions != null) {
                if (resource.partitions.containsKey("count")) {
                    partitions = resource.partitions.get("count");
                }
                if (resource.partitions.containsKey("replicas")) {
                    replicas = resource.partitions.get("replicas");
                }
            }
            if (resource.rebalancer == null || !resource.rebalancer.containsKey("mode")) {
                throw new HelixException("Rebalance mode is required!");
            }
            helixAdmin.addResource(cfg.clusterName, resource.name, partitions, resource.stateModel.name, resource.rebalancer.get("mode"));
            // user-defined rebalancer
            if (resource.rebalancer.containsKey("class") && resource.rebalancer.get("mode").equals(RebalanceMode.USER_DEFINED.toString())) {
                IdealState idealState = helixAdmin.getResourceIdealState(cfg.clusterName, resource.name);
                idealState.setRebalancerClassName(resource.rebalancer.get("class"));
                helixAdmin.setResourceIdealState(cfg.clusterName, resource.name, idealState);
            }
            helixAdmin.rebalance(cfg.clusterName, resource.name, replicas);
        }
    }
    // enable auto join if this option is set
    if (cfg.autoJoinAllowed != null && cfg.autoJoinAllowed) {
        HelixConfigScope scope = new HelixConfigScopeBuilder(ConfigScopeProperty.CLUSTER).forCluster(cfg.clusterName).build();
        Map<String, String> properties = new HashMap<String, String>();
        properties.put(ZKHelixManager.ALLOW_PARTICIPANT_AUTO_JOIN, cfg.autoJoinAllowed.toString());
        helixAdmin.setConfig(scope, properties);
    }
    return cfg;
}

Example 83 with HelixAdmin

use of org.apache.helix.HelixAdmin in project helix by apache.

the class TestRebalancePipeline method testChangeIdealStateWithPendingMsg.

@Test
public void testChangeIdealStateWithPendingMsg() {
    String clusterName = "CLUSTER_" + _className + "_pending";
    System.out.println("START " + clusterName + " at " + new Date(System.currentTimeMillis()));
    HelixDataAccessor accessor = new ZKHelixDataAccessor(clusterName, new ZkBaseDataAccessor<ZNRecord>(_gZkClient));
    HelixManager manager = new DummyClusterManager(clusterName, accessor);
    ClusterEvent event = new ClusterEvent(ClusterEventType.Unknown);
    event.addAttribute(AttributeName.helixmanager.name(), manager);
    ClusterDataCache cache = new ClusterDataCache();
    event.addAttribute(AttributeName.ClusterDataCache.name(), cache);
    refreshClusterConfig(clusterName, accessor);
    final String resourceName = "testResource_pending";
    String[] resourceGroups = new String[] { resourceName };
    // ideal state: node0 is MASTER, node1 is SLAVE
    // replica=2 means 1 master and 1 slave
    setupIdealState(clusterName, new int[] { 0 }, resourceGroups, 1, 1);
    setupLiveInstances(clusterName, new int[] { 0 });
    setupStateModel(clusterName);
    // cluster data cache refresh pipeline
    Pipeline dataRefresh = new Pipeline();
    dataRefresh.addStage(new ReadClusterDataStage());
    // rebalance pipeline
    Pipeline rebalancePipeline = new Pipeline();
    rebalancePipeline.addStage(new ResourceComputationStage());
    rebalancePipeline.addStage(new CurrentStateComputationStage());
    rebalancePipeline.addStage(new BestPossibleStateCalcStage());
    rebalancePipeline.addStage(new IntermediateStateCalcStage());
    rebalancePipeline.addStage(new MessageGenerationPhase());
    rebalancePipeline.addStage(new MessageSelectionStage());
    rebalancePipeline.addStage(new MessageThrottleStage());
    rebalancePipeline.addStage(new TaskAssignmentStage());
    // round1: set node0 currentState to OFFLINE and node1 currentState to SLAVE
    setCurrentState(clusterName, "localhost_0", resourceName, resourceName + "_0", "session_0", "OFFLINE");
    runPipeline(event, dataRefresh);
    runPipeline(event, rebalancePipeline);
    MessageSelectionStageOutput msgSelOutput = event.getAttribute(AttributeName.MESSAGES_SELECTED.name());
    List<Message> messages = msgSelOutput.getMessages(resourceName, new Partition(resourceName + "_0"));
    Assert.assertEquals(messages.size(), 1, "Should output 1 message: OFFLINE-SLAVE for node0");
    Message message = messages.get(0);
    Assert.assertEquals(message.getFromState(), "OFFLINE");
    Assert.assertEquals(message.getToState(), "SLAVE");
    Assert.assertEquals(message.getTgtName(), "localhost_0");
    // round2: drop resource, but keep the
    // message, make sure controller should not send O->DROPPED until O->S is done
    HelixAdmin admin = new ZKHelixAdmin(_gZkClient);
    admin.dropResource(clusterName, resourceName);
    List<IdealState> idealStates = accessor.getChildValues(accessor.keyBuilder().idealStates());
    cache.setIdealStates(idealStates);
    runPipeline(event, dataRefresh);
    cache = event.getAttribute(AttributeName.ClusterDataCache.name());
    cache.setClusterConfig(new ClusterConfig(clusterName));
    runPipeline(event, rebalancePipeline);
    msgSelOutput = event.getAttribute(AttributeName.MESSAGES_SELECTED.name());
    messages = msgSelOutput.getMessages(resourceName, new Partition(resourceName + "_0"));
    Assert.assertEquals(messages.size(), 0, "Should not output only 1 message: OFFLINE->DROPPED for localhost_0");
    // round3: remove O->S message for localhost_0, localhost_0 still in OFFLINE
    // controller should now send O->DROPPED to localhost_0
    Builder keyBuilder = accessor.keyBuilder();
    List<String> msgIds = accessor.getChildNames(keyBuilder.messages("localhost_0"));
    accessor.removeProperty(keyBuilder.message("localhost_0", msgIds.get(0)));
    runPipeline(event, dataRefresh);
    runPipeline(event, rebalancePipeline);
    msgSelOutput = event.getAttribute(AttributeName.MESSAGES_SELECTED.name());
    messages = msgSelOutput.getMessages(resourceName, new Partition(resourceName + "_0"));
    Assert.assertEquals(messages.size(), 1, "Should output 1 message: OFFLINE->DROPPED for localhost_0");
    message = messages.get(0);
    Assert.assertEquals(message.getFromState(), "OFFLINE");
    Assert.assertEquals(message.getToState(), "DROPPED");
    Assert.assertEquals(message.getTgtName(), "localhost_0");
    System.out.println("END " + clusterName + " at " + new Date(System.currentTimeMillis()));
}

Example 84 with HelixAdmin

use of org.apache.helix.HelixAdmin in project helix by apache.

the class TestDisablePartition method testDisableFullAuto.

@Test(dataProvider = "rebalancer", enabled = true)
public void testDisableFullAuto(String rebalancerName) throws Exception {
    final int NUM_PARTITIONS = 8;
    final int NUM_PARTICIPANTS = 2;
    final int NUM_REPLICAS = 1;
    String className = TestHelper.getTestClassName();
    String methodName = TestHelper.getTestMethodName();
    String clusterName = className + "_" + methodName;
    ClusterSetup clusterSetup = new ClusterSetup(ZK_ADDR);
    clusterSetup.addCluster(clusterName, true);
    for (int i = 0; i < NUM_PARTICIPANTS; i++) {
        String instanceName = "localhost_" + (11420 + i);
        clusterSetup.addInstanceToCluster(clusterName, instanceName);
    }
    // Create a known problematic scenario
    HelixAdmin admin = clusterSetup.getClusterManagementTool();
    String resourceName = "MailboxDB";
    IdealState idealState = new IdealState(resourceName + "DR");
    idealState.setRebalanceMode(RebalanceMode.SEMI_AUTO);
    idealState.setStateModelDefRef("LeaderStandby");
    idealState.setReplicas(String.valueOf(NUM_REPLICAS));
    idealState.setNumPartitions(NUM_PARTITIONS);
    for (int i = 0; i < NUM_PARTITIONS; i++) {
        String partitionName = resourceName + '_' + i;
        List<String> assignmentList = Lists.newArrayList();
        if (i < NUM_PARTITIONS / 2) {
            assignmentList.add("localhost_11420");
        } else {
            assignmentList.add("localhost_11421");
        }
        Map<String, String> emptyMap = Maps.newHashMap();
        idealState.getRecord().setListField(partitionName, assignmentList);
        idealState.getRecord().setMapField(partitionName, emptyMap);
    }
    admin.addResource(clusterName, idealState.getResourceName(), idealState);
    // Start everything
    MockParticipantManager[] participants = new MockParticipantManager[NUM_PARTICIPANTS];
    for (int i = 0; i < NUM_PARTICIPANTS; i++) {
        String instanceName = "localhost_" + (11420 + i);
        participants[i] = new MockParticipantManager(ZK_ADDR, clusterName, instanceName);
        participants[i].syncStart();
    }
    ClusterControllerManager controller = new ClusterControllerManager(ZK_ADDR, clusterName, "controller_1");
    controller.syncStart();
    Thread.sleep(1000);
    // Switch to full auto
    idealState.setRebalanceMode(RebalanceMode.FULL_AUTO);
    idealState.setRebalancerClassName(rebalancerName);
    for (int i = 0; i < NUM_PARTITIONS; i++) {
        List<String> emptyList = Collections.emptyList();
        idealState.getRecord().setListField(resourceName + '_' + i, emptyList);
    }
    admin.setResourceIdealState(clusterName, idealState.getResourceName(), idealState);
    Thread.sleep(1000);
    // Get the external view
    HelixDataAccessor accessor = controller.getHelixDataAccessor();
    PropertyKey.Builder keyBuilder = accessor.keyBuilder();
    ExternalView externalView = accessor.getProperty(keyBuilder.externalView(idealState.getResourceName()));
    // Disable the partitions in an order known to cause problems
    int[] pid = { 0, 7 };
    for (int i = 0; i < pid.length; i++) {
        String partitionName = resourceName + '_' + pid[i];
        Map<String, String> stateMap = externalView.getStateMap(partitionName);
        String leader = null;
        for (String participantName : stateMap.keySet()) {
            String state = stateMap.get(participantName);
            if (state.equals("LEADER")) {
                leader = participantName;
            }
        }
        List<String> partitionNames = Lists.newArrayList(partitionName);
        admin.enablePartition(false, clusterName, leader, idealState.getResourceName(), partitionNames);
        Thread.sleep(1000);
    }
    // Ensure that nothing was reassigned and the disabled are offline
    externalView = accessor.getProperty(keyBuilder.externalView(idealState.getResourceName()));
    Map<String, String> p0StateMap = externalView.getStateMap(resourceName + "_0");
    Assert.assertEquals(p0StateMap.size(), 1);
    String p0Participant = p0StateMap.keySet().iterator().next();
    Assert.assertEquals(p0StateMap.get(p0Participant), "OFFLINE");
    Map<String, String> p7StateMap = externalView.getStateMap(resourceName + "_7");
    Assert.assertEquals(p7StateMap.size(), 1);
    String p7Participant = p7StateMap.keySet().iterator().next();
    Assert.assertEquals(p7StateMap.get(p7Participant), "OFFLINE");
    // Cleanup
    controller.syncStop();
    for (MockParticipantManager participant : participants) {
        participant.syncStop();
    }
}

Example 85 with HelixAdmin

use of org.apache.helix.HelixAdmin in project helix by apache.

the class TestDrop method testBasic.

@Test
public void testBasic() throws Exception {
    String className = TestHelper.getTestClassName();
    String methodName = TestHelper.getTestMethodName();
    String clusterName = className + "_" + methodName;
    final int n = 5;
    System.out.println("START " + clusterName + " at " + new Date(System.currentTimeMillis()));
    MockParticipantManager[] participants = new MockParticipantManager[n];
    // participant port
    TestHelper.setupCluster(// participant port
    clusterName, // participant port
    ZK_ADDR, // participant port
    12918, // participant name prefix
    "localhost", // resource name prefix
    "TestDB", // resources
    1, // partitions per resource
    8, // number of nodes
    n, // replicas
    3, "MasterSlave", // do rebalance
    true);
    // start controller
    ClusterControllerManager controller = new ClusterControllerManager(ZK_ADDR, clusterName, "controller");
    controller.syncStart();
    // start participants
    for (int i = 0; i < n; i++) {
        String instanceName = "localhost_" + (12918 + i);
        participants[i] = new MockParticipantManager(ZK_ADDR, clusterName, instanceName);
        participants[i].syncStart();
    }
    HelixClusterVerifier verifier = new BestPossibleExternalViewVerifier.Builder(clusterName).setZkAddr(ZK_ADDR).build();
    Assert.assertTrue(verifier.verify());
    // Drop TestDB0
    HelixAdmin admin = new ZKHelixAdmin(_gZkClient);
    admin.dropResource(clusterName, "TestDB0");
    Thread.sleep(1000);
    assertEmptyCSandEV(clusterName, "TestDB0", participants);
    System.out.println("END " + clusterName + " at " + new Date(System.currentTimeMillis()));
}