Examples with HelixAdmin org.apache.helix.HelixAdmin used on opensource projects

Example 31 with HelixAdmin

use of org.apache.helix.HelixAdmin in project helix by apache.

the class TestEntropyFreeNodeBounce method testBounceAll.

@Test
public void testBounceAll() throws Exception {
    // pick numbers that don't divide evenly
    final int NUM_PARTICIPANTS = 5;
    final int NUM_PARTITIONS = 123;
    final int NUM_REPLICAS = 1;
    final String RESOURCE_PREFIX = "TestDB";
    final String RESOURCE_NAME = RESOURCE_PREFIX + "0";
    // create a cluster name based on this test name
    String className = TestHelper.getTestClassName();
    String methodName = TestHelper.getTestMethodName();
    String clusterName = className + "_" + methodName;
    System.out.println("START " + clusterName + " at " + new Date(System.currentTimeMillis()));
    // Set up cluster
    // participant port
    TestHelper.setupCluster(// participant port
    clusterName, // participant port
    ZK_ADDR, // participant port
    12918, // participant name prefix
    "localhost", // resource name prefix
    "TestDB", // resources
    1, // partitions per resource
    NUM_PARTITIONS, // number of nodes
    NUM_PARTICIPANTS, // replicas
    NUM_REPLICAS, // use FULL_AUTO mode to test node tagging
    "OnlineOffline", // use FULL_AUTO mode to test node tagging
    RebalanceMode.FULL_AUTO, // do rebalance
    true);
    // Start the participants
    HelixManager[] participants = new HelixManager[NUM_PARTICIPANTS];
    for (int i = 0; i < NUM_PARTICIPANTS; i++) {
        final String instanceName = "localhost_" + (12918 + i);
        participants[i] = createParticipant(clusterName, instanceName);
        participants[i].connect();
    }
    // Start the controller
    ClusterControllerManager controller = new ClusterControllerManager(ZK_ADDR, clusterName, "controller");
    controller.syncStart();
    // get an admin and accessor
    HelixAdmin helixAdmin = new ZKHelixAdmin(_gZkClient);
    BaseDataAccessor<ZNRecord> baseAccessor = new ZkBaseDataAccessor<ZNRecord>(_gZkClient);
    HelixDataAccessor accessor = new ZKHelixDataAccessor(clusterName, baseAccessor);
    PropertyKey.Builder keyBuilder = accessor.keyBuilder();
    // do the test
    try {
        Thread.sleep(1000);
        // ensure that the external view coalesces
        boolean result = ClusterStateVerifier.verifyByZkCallback(new BestPossAndExtViewZkVerifier(ZK_ADDR, clusterName));
        Assert.assertTrue(result);
        ExternalView stableExternalView = accessor.getProperty(keyBuilder.externalView(RESOURCE_NAME));
        for (HelixManager participant : participants) {
            // disable the controller, bounce the node, re-enable the controller, verify assignments
            // remained the same
            helixAdmin.enableCluster(clusterName, false);
            participant.disconnect();
            Thread.sleep(1000);
            participant = createParticipant(clusterName, participant.getInstanceName());
            participant.connect();
            Thread.sleep(1000);
            helixAdmin.enableCluster(clusterName, true);
            Thread.sleep(1000);
            result = ClusterStateVerifier.verifyByZkCallback(new MatchingExternalViewVerifier(stableExternalView, clusterName));
            Assert.assertTrue(result);
        }
    } finally {
        // clean up
        controller.syncStop();
        for (HelixManager participant : participants) {
            participant.disconnect();
        }
        System.out.println("END " + clusterName + " at " + new Date(System.currentTimeMillis()));
    }
}

Example 32 with HelixAdmin

use of org.apache.helix.HelixAdmin in project helix by apache.

the class TestMessageThrottle method testMessageThrottle.

@Test()
public void testMessageThrottle() throws Exception {
    // Logger.getRootLogger().setLevel(Level.INFO);
    String clusterName = getShortClassName();
    MockParticipantManager[] participants = new MockParticipantManager[5];
    System.out.println("START " + clusterName + " at " + new Date(System.currentTimeMillis()));
    // participant start
    TestHelper.setupCluster(// participant start
    clusterName, // participant start
    ZK_ADDR, // participant start
    12918, // participant name prefix
    "localhost", // resource name prefix
    "TestDB", // resources
    1, // partitions per resource
    10, // number of nodes
    5, // replicas
    3, "MasterSlave", // do rebalance
    true);
    // setup message constraint
    // "MESSAGE_TYPE=STATE_TRANSITION,TRANSITION=OFFLINE-SLAVE,INSTANCE=.*,CONSTRAINT_VALUE=1";
    HelixAdmin admin = new ZKHelixAdmin(_gZkClient);
    ConstraintItemBuilder builder = new ConstraintItemBuilder();
    builder.addConstraintAttribute("MESSAGE_TYPE", "STATE_TRANSITION").addConstraintAttribute("INSTANCE", ".*").addConstraintAttribute("CONSTRAINT_VALUE", "1");
    // Map<String, String> constraints = new TreeMap<String, String>();
    // constraints.put("MESSAGE_TYPE", "STATE_TRANSITION");
    // // constraints.put("TRANSITION", "OFFLINE-SLAVE");
    // constraints.put("CONSTRAINT_VALUE", "1");
    // constraints.put("INSTANCE", ".*");
    admin.setConstraint(clusterName, ConstraintType.MESSAGE_CONSTRAINT, "constraint1", builder.build());
    final ZKHelixDataAccessor accessor = new ZKHelixDataAccessor(clusterName, new ZkBaseDataAccessor<ZNRecord>(_gZkClient));
    // make sure we never see more than 1 state transition message for each participant
    final AtomicBoolean success = new AtomicBoolean(true);
    for (int i = 0; i < 5; i++) {
        String instanceName = "localhost_" + (12918 + i);
        String msgPath = PropertyPathBuilder.instanceMessage(clusterName, instanceName);
        _gZkClient.subscribeChildChanges(msgPath, new IZkChildListener() {

            @Override
            public void handleChildChange(String parentPath, List<String> currentChilds) throws Exception {
                if (currentChilds != null && currentChilds.size() > 1) {
                    List<ZNRecord> records = accessor.getBaseDataAccessor().getChildren(parentPath, null, 0);
                    int transitionMsgCount = 0;
                    for (ZNRecord record : records) {
                        Message msg = new Message(record);
                        if (msg.getMsgType().equals(Message.MessageType.STATE_TRANSITION.name())) {
                            transitionMsgCount++;
                        }
                    }
                    if (transitionMsgCount > 1) {
                        success.set(false);
                        Assert.fail("Should not see more than 1 message");
                    }
                }
            }
        });
    }
    ClusterControllerManager controller = new ClusterControllerManager(ZK_ADDR, clusterName, "controller_0");
    controller.syncStart();
    // start participants
    for (int i = 0; i < 5; i++) {
        String instanceName = "localhost_" + (12918 + i);
        participants[i] = new MockParticipantManager(ZK_ADDR, clusterName, instanceName);
        participants[i].syncStart();
    }
    boolean result = ClusterStateVerifier.verifyByZkCallback(new MasterNbInExtViewVerifier(ZK_ADDR, clusterName));
    Assert.assertTrue(result);
    result = ClusterStateVerifier.verifyByZkCallback(new BestPossAndExtViewZkVerifier(ZK_ADDR, clusterName));
    Assert.assertTrue(result);
    Assert.assertTrue(success.get());
    // clean up
    controller.syncStop();
    for (int i = 0; i < 5; i++) {
        participants[i].syncStop();
    }
    System.out.println("END " + clusterName + " at " + new Date(System.currentTimeMillis()));
}

Example 33 with HelixAdmin

use of org.apache.helix.HelixAdmin in project helix by apache.

the class TestMessageThrottle2 method startAdmin.

void startAdmin() throws Exception {
    HelixAdmin admin = new ZKHelixAdmin(ZK_ADDR);
    // create cluster
    System.out.println("Creating cluster: " + clusterName);
    admin.addCluster(clusterName, true);
    // add MasterSlave state mode definition
    admin.addStateModelDef(clusterName, "MasterSlave", new StateModelDefinition(generateConfigForMasterSlave()));
    // ideal-state znrecord
    ZNRecord record = new ZNRecord(resourceName);
    record.setSimpleField("IDEAL_STATE_MODE", "AUTO");
    record.setSimpleField("NUM_PARTITIONS", "1");
    record.setSimpleField("REPLICAS", "2");
    record.setSimpleField("STATE_MODEL_DEF_REF", "MasterSlave");
    record.setListField(resourceName, Arrays.asList("node1", "node2"));
    admin.setResourceIdealState(clusterName, resourceName, new IdealState(record));
    ConstraintItemBuilder builder = new ConstraintItemBuilder();
    // limit one transition message at a time across the entire cluster
    builder.addConstraintAttribute("MESSAGE_TYPE", "STATE_TRANSITION").addConstraintAttribute("CONSTRAINT_VALUE", "1");
    admin.setConstraint(clusterName, ClusterConstraints.ConstraintType.MESSAGE_CONSTRAINT, "constraint1", builder.build());
}

Example 34 with HelixAdmin

use of org.apache.helix.HelixAdmin in project helix by apache.

the class TestZkReconnect method testZKReconnect.

@Test(enabled = false)
public void testZKReconnect() throws Exception {
    final AtomicReference<ZkServer> zkServerRef = new AtomicReference<ZkServer>();
    final int zkPort = TestHelper.getRandomPort();
    final String zkAddr = String.format("localhost:%d", zkPort);
    ZkServer zkServer = TestHelper.startZkServer(zkAddr);
    zkServerRef.set(zkServer);
    String className = TestHelper.getTestClassName();
    String methodName = TestHelper.getTestMethodName();
    String clusterName = className + "_" + methodName;
    // Setup cluster
    LOG.info("Setup clusters");
    ClusterSetup clusterSetup = new ClusterSetup(zkAddr);
    clusterSetup.addCluster(clusterName, true);
    // Registers and starts controller
    LOG.info("Starts controller");
    HelixManager controller = HelixManagerFactory.getZKHelixManager(clusterName, null, InstanceType.CONTROLLER, zkAddr);
    controller.connect();
    // Registers and starts participant
    LOG.info("Starts participant");
    String hostname = "localhost";
    String instanceId = String.format("%s_%d", hostname, 1);
    clusterSetup.addInstanceToCluster(clusterName, instanceId);
    HelixManager participant = HelixManagerFactory.getZKHelixManager(clusterName, instanceId, InstanceType.PARTICIPANT, zkAddr);
    participant.connect();
    LOG.info("Register state machine");
    final CountDownLatch latch = new CountDownLatch(1);
    participant.getStateMachineEngine().registerStateModelFactory("OnlineOffline", new StateModelFactory<StateModel>() {

        @Override
        public StateModel createNewStateModel(String resource, String stateUnitKey) {
            return new SimpleStateModel(latch);
        }
    }, "test");
    String resourceName = "test-resource";
    LOG.info("Ideal state assignment");
    HelixAdmin helixAdmin = participant.getClusterManagmentTool();
    helixAdmin.addResource(clusterName, resourceName, 1, "OnlineOffline", IdealState.RebalanceMode.CUSTOMIZED.toString());
    IdealState idealState = helixAdmin.getResourceIdealState(clusterName, resourceName);
    idealState.setReplicas("1");
    idealState.setStateModelFactoryName("test");
    idealState.setPartitionState(resourceName + "_0", instanceId, "ONLINE");
    LOG.info("Shutdown ZK server");
    TestHelper.stopZkServer(zkServerRef.get());
    Executors.newSingleThreadScheduledExecutor().schedule(new Runnable() {

        @Override
        public void run() {
            try {
                LOG.info("Restart ZK server");
                // zkServer.set(TestUtils.startZookeeper(zkDir, zkPort));
                zkServerRef.set(TestHelper.startZkServer(zkAddr, null, false));
            } catch (Exception e) {
                LOG.error(e.getMessage(), e);
            }
        }
    }, 2L, TimeUnit.SECONDS);
    // future.get();
    LOG.info("Before update ideal state");
    helixAdmin.setResourceIdealState(clusterName, resourceName, idealState);
    LOG.info("After update ideal state");
    LOG.info("Wait for OFFLINE->ONLINE state transition");
    try {
        Assert.assertTrue(latch.await(15, TimeUnit.SECONDS));
        // wait until stable state
        boolean result = ClusterStateVerifier.verifyByZkCallback(new BestPossAndExtViewZkVerifier(zkAddr, clusterName));
        Assert.assertTrue(result);
    } finally {
        participant.disconnect();
        zkServerRef.get().shutdown();
    }
}

Example 35 with HelixAdmin

use of org.apache.helix.HelixAdmin in project helix by apache.

the class TestAutoRebalanceWithDisabledInstance method getPartitionsAssignedtoInstance.

private Set<String> getPartitionsAssignedtoInstance(String cluster, String dbName, String instance) {
    HelixAdmin admin = _setupTool.getClusterManagementTool();
    Set<String> partitionSet = new HashSet<String>();
    IdealState is = admin.getResourceIdealState(cluster, dbName);
    for (String partition : is.getRecord().getListFields().keySet()) {
        List<String> assignments = is.getRecord().getListField(partition);
        for (String ins : assignments) {
            if (ins.equals(instance)) {
                partitionSet.add(partition);
            }
        }
    }
    return partitionSet;
}