Search in sources :

Example 41 with ZKHelixDataAccessor

use of org.apache.helix.manager.zk.ZKHelixDataAccessor in project helix by apache.

the class TestMessageThrottle method testMessageThrottle.

@Test()
public void testMessageThrottle() throws Exception {
    // Logger.getRootLogger().setLevel(Level.INFO);
    String clusterName = getShortClassName();
    MockParticipantManager[] participants = new MockParticipantManager[5];
    System.out.println("START " + clusterName + " at " + new Date(System.currentTimeMillis()));
    // participant start
    TestHelper.setupCluster(// participant start
    clusterName, // participant start
    ZK_ADDR, // participant start
    12918, // participant name prefix
    "localhost", // resource name prefix
    "TestDB", // resources
    1, // partitions per resource
    10, // number of nodes
    5, // replicas
    3, "MasterSlave", // do rebalance
    true);
    // setup message constraint
    // "MESSAGE_TYPE=STATE_TRANSITION,TRANSITION=OFFLINE-SLAVE,INSTANCE=.*,CONSTRAINT_VALUE=1";
    HelixAdmin admin = new ZKHelixAdmin(_gZkClient);
    ConstraintItemBuilder builder = new ConstraintItemBuilder();
    builder.addConstraintAttribute("MESSAGE_TYPE", "STATE_TRANSITION").addConstraintAttribute("INSTANCE", ".*").addConstraintAttribute("CONSTRAINT_VALUE", "1");
    // Map<String, String> constraints = new TreeMap<String, String>();
    // constraints.put("MESSAGE_TYPE", "STATE_TRANSITION");
    // // constraints.put("TRANSITION", "OFFLINE-SLAVE");
    // constraints.put("CONSTRAINT_VALUE", "1");
    // constraints.put("INSTANCE", ".*");
    admin.setConstraint(clusterName, ConstraintType.MESSAGE_CONSTRAINT, "constraint1", builder.build());
    final ZKHelixDataAccessor accessor = new ZKHelixDataAccessor(clusterName, new ZkBaseDataAccessor<ZNRecord>(_gZkClient));
    // make sure we never see more than 1 state transition message for each participant
    final AtomicBoolean success = new AtomicBoolean(true);
    for (int i = 0; i < 5; i++) {
        String instanceName = "localhost_" + (12918 + i);
        String msgPath = PropertyPathBuilder.instanceMessage(clusterName, instanceName);
        _gZkClient.subscribeChildChanges(msgPath, new IZkChildListener() {

            @Override
            public void handleChildChange(String parentPath, List<String> currentChilds) throws Exception {
                if (currentChilds != null && currentChilds.size() > 1) {
                    List<ZNRecord> records = accessor.getBaseDataAccessor().getChildren(parentPath, null, 0);
                    int transitionMsgCount = 0;
                    for (ZNRecord record : records) {
                        Message msg = new Message(record);
                        if (msg.getMsgType().equals(Message.MessageType.STATE_TRANSITION.name())) {
                            transitionMsgCount++;
                        }
                    }
                    if (transitionMsgCount > 1) {
                        success.set(false);
                        Assert.fail("Should not see more than 1 message");
                    }
                }
            }
        });
    }
    ClusterControllerManager controller = new ClusterControllerManager(ZK_ADDR, clusterName, "controller_0");
    controller.syncStart();
    // start participants
    for (int i = 0; i < 5; i++) {
        String instanceName = "localhost_" + (12918 + i);
        participants[i] = new MockParticipantManager(ZK_ADDR, clusterName, instanceName);
        participants[i].syncStart();
    }
    boolean result = ClusterStateVerifier.verifyByZkCallback(new MasterNbInExtViewVerifier(ZK_ADDR, clusterName));
    Assert.assertTrue(result);
    result = ClusterStateVerifier.verifyByZkCallback(new BestPossAndExtViewZkVerifier(ZK_ADDR, clusterName));
    Assert.assertTrue(result);
    Assert.assertTrue(success.get());
    // clean up
    controller.syncStop();
    for (int i = 0; i < 5; i++) {
        participants[i].syncStop();
    }
    System.out.println("END " + clusterName + " at " + new Date(System.currentTimeMillis()));
}
Also used : MasterNbInExtViewVerifier(org.apache.helix.tools.ClusterStateVerifier.MasterNbInExtViewVerifier) MockParticipantManager(org.apache.helix.integration.manager.MockParticipantManager) Message(org.apache.helix.model.Message) ConstraintItemBuilder(org.apache.helix.model.builder.ConstraintItemBuilder) HelixAdmin(org.apache.helix.HelixAdmin) ZKHelixAdmin(org.apache.helix.manager.zk.ZKHelixAdmin) BestPossAndExtViewZkVerifier(org.apache.helix.tools.ClusterStateVerifier.BestPossAndExtViewZkVerifier) Date(java.util.Date) ClusterControllerManager(org.apache.helix.integration.manager.ClusterControllerManager) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) ZKHelixAdmin(org.apache.helix.manager.zk.ZKHelixAdmin) IZkChildListener(org.I0Itec.zkclient.IZkChildListener) List(java.util.List) ZNRecord(org.apache.helix.ZNRecord) ZKHelixDataAccessor(org.apache.helix.manager.zk.ZKHelixDataAccessor) Test(org.testng.annotations.Test)

Example 42 with ZKHelixDataAccessor

use of org.apache.helix.manager.zk.ZKHelixDataAccessor in project helix by apache.

the class TestMessageThrottle2 method test.

@Test
public void test() throws Exception {
    System.out.println("START " + clusterName + " at " + new Date(System.currentTimeMillis()));
    startAdmin();
    startController();
    // start node2 first
    Node.main(new String[] { "2" });
    // wait for node2 becoming MASTER
    final Builder keyBuilder = new Builder(clusterName);
    final HelixDataAccessor accessor = new ZKHelixDataAccessor(clusterName, new ZkBaseDataAccessor<ZNRecord>(_gZkClient));
    TestHelper.verify(new TestHelper.Verifier() {

        @Override
        public boolean verify() throws Exception {
            ExternalView view = accessor.getProperty(keyBuilder.externalView(resourceName));
            String state = null;
            if (view != null) {
                Map<String, String> map = view.getStateMap(resourceName);
                if (map != null) {
                    state = map.get("node2");
                }
            }
            return state != null && state.equals("MASTER");
        }
    }, 10 * 1000);
    // start node 1
    Node.main(new String[] { "1" });
    boolean result = ClusterStateVerifier.verifyByZkCallback(new BestPossAndExtViewZkVerifier(ZK_ADDR, clusterName));
    Assert.assertTrue(result);
    System.out.println("END " + clusterName + " at " + new Date(System.currentTimeMillis()));
}
Also used : ExternalView(org.apache.helix.model.ExternalView) ConstraintItemBuilder(org.apache.helix.model.builder.ConstraintItemBuilder) Builder(org.apache.helix.PropertyKey.Builder) BestPossAndExtViewZkVerifier(org.apache.helix.tools.ClusterStateVerifier.BestPossAndExtViewZkVerifier) Date(java.util.Date) TestHelper(org.apache.helix.TestHelper) ZKHelixDataAccessor(org.apache.helix.manager.zk.ZKHelixDataAccessor) HelixDataAccessor(org.apache.helix.HelixDataAccessor) HashMap(java.util.HashMap) Map(java.util.Map) ZNRecord(org.apache.helix.ZNRecord) ZKHelixDataAccessor(org.apache.helix.manager.zk.ZKHelixDataAccessor) Test(org.testng.annotations.Test)

Example 43 with ZKHelixDataAccessor

use of org.apache.helix.manager.zk.ZKHelixDataAccessor in project helix by apache.

the class TestAutoRebalance method testAutoRebalance.

@Test()
public void testAutoRebalance() throws Exception {
    // kill 1 node
    _participants[0].syncStop();
    boolean result = ClusterStateVerifier.verifyByZkCallback(new ExternalViewBalancedVerifier(_gZkClient, CLUSTER_NAME, TEST_DB));
    Assert.assertTrue(result);
    // add 2 nodes
    for (int i = 0; i < 2; i++) {
        String storageNodeName = PARTICIPANT_PREFIX + "_" + (1000 + i);
        _setupTool.addInstanceToCluster(CLUSTER_NAME, storageNodeName);
        MockParticipantManager participant = new MockParticipantManager(ZK_ADDR, CLUSTER_NAME, storageNodeName.replace(':', '_'));
        participant.syncStart();
    }
    Thread.sleep(1000);
    result = ClusterStateVerifier.verifyByZkCallback(new ExternalViewBalancedVerifier(_gZkClient, CLUSTER_NAME, TEST_DB));
    Assert.assertTrue(result);
    result = ClusterStateVerifier.verifyByZkCallback(new ExternalViewBalancedVerifier(_gZkClient, CLUSTER_NAME, db2));
    Assert.assertTrue(result);
    HelixDataAccessor accessor = new ZKHelixDataAccessor(CLUSTER_NAME, new ZkBaseDataAccessor<ZNRecord>(_gZkClient));
    Builder keyBuilder = accessor.keyBuilder();
    ExternalView ev = accessor.getProperty(keyBuilder.externalView(db2));
    Set<String> instancesSet = new HashSet<String>();
    for (String partitionName : ev.getRecord().getMapFields().keySet()) {
        Map<String, String> assignmentMap = ev.getRecord().getMapField(partitionName);
        for (String instance : assignmentMap.keySet()) {
            instancesSet.add(instance);
        }
    }
    Assert.assertEquals(instancesSet.size(), 2);
}
Also used : ExternalView(org.apache.helix.model.ExternalView) MockParticipantManager(org.apache.helix.integration.manager.MockParticipantManager) Builder(org.apache.helix.PropertyKey.Builder) ZKHelixDataAccessor(org.apache.helix.manager.zk.ZKHelixDataAccessor) HelixDataAccessor(org.apache.helix.HelixDataAccessor) ZNRecord(org.apache.helix.ZNRecord) ZKHelixDataAccessor(org.apache.helix.manager.zk.ZKHelixDataAccessor) HashSet(java.util.HashSet) Test(org.testng.annotations.Test)

Example 44 with ZKHelixDataAccessor

use of org.apache.helix.manager.zk.ZKHelixDataAccessor in project helix by apache.

the class TestClusterInMaintenanceModeWhenReachingMaxPartition method beforeClass.

@BeforeClass
public void beforeClass() throws Exception {
    System.out.println("START " + CLASS_NAME + " at " + new Date(System.currentTimeMillis()));
    String namespace = "/" + CLUSTER_NAME;
    if (_gZkClient.exists(namespace)) {
        _gZkClient.deleteRecursively(namespace);
    }
    _gSetupTool.addCluster(CLUSTER_NAME, true);
    for (int i = 0; i < NUM_NODE; i++) {
        String storageNodeName = PARTICIPANT_PREFIX + "_" + (START_PORT + i);
        _gSetupTool.addInstanceToCluster(CLUSTER_NAME, storageNodeName);
        // start dummy participants
        MockParticipantManager participant = new MockParticipantManager(ZK_ADDR, CLUSTER_NAME, storageNodeName);
        participant.syncStart();
        _participants.add(participant);
    }
    // start controller
    String controllerName = CONTROLLER_PREFIX + "_0";
    _controller = new ClusterControllerManager(ZK_ADDR, CLUSTER_NAME, controllerName);
    _controller.syncStart();
    _clusterVerifier = new BestPossibleExternalViewVerifier.Builder(CLUSTER_NAME).setZkAddr(ZK_ADDR).build();
    enablePersistBestPossibleAssignment(_gZkClient, CLUSTER_NAME, true);
    _dataAccessor = new ZKHelixDataAccessor(CLUSTER_NAME, _baseAccessor);
}
Also used : ClusterControllerManager(org.apache.helix.integration.manager.ClusterControllerManager) MockParticipantManager(org.apache.helix.integration.manager.MockParticipantManager) BestPossibleExternalViewVerifier(org.apache.helix.tools.ClusterVerifiers.BestPossibleExternalViewVerifier) Date(java.util.Date) ZKHelixDataAccessor(org.apache.helix.manager.zk.ZKHelixDataAccessor) BeforeClass(org.testng.annotations.BeforeClass)

Example 45 with ZKHelixDataAccessor

use of org.apache.helix.manager.zk.ZKHelixDataAccessor in project helix by apache.

the class TestClusterInMaintenanceModeWhenReachingOfflineInstancesLimit method beforeClass.

@BeforeClass
public void beforeClass() throws Exception {
    System.out.println("START " + CLASS_NAME + " at " + new Date(System.currentTimeMillis()));
    String namespace = "/" + CLUSTER_NAME;
    if (_gZkClient.exists(namespace)) {
        _gZkClient.deleteRecursively(namespace);
    }
    _gSetupTool.addCluster(CLUSTER_NAME, true);
    for (int i = 0; i < NUM_NODE; i++) {
        String instanceName = PARTICIPANT_PREFIX + "_" + (START_PORT + i);
        _gSetupTool.addInstanceToCluster(CLUSTER_NAME, instanceName);
        // start dummy participants
        MockParticipantManager participant = new MockParticipantManager(ZK_ADDR, CLUSTER_NAME, instanceName);
        participant.syncStart();
        _participants.add(participant);
    }
    // start controller
    String controllerName = CONTROLLER_PREFIX + "_0";
    _controller = new ClusterControllerManager(ZK_ADDR, CLUSTER_NAME, controllerName);
    _controller.syncStart();
    _clusterVerifier = new BestPossibleExternalViewVerifier.Builder(CLUSTER_NAME).setZkAddr(ZK_ADDR).build();
    enablePersistBestPossibleAssignment(_gZkClient, CLUSTER_NAME, true);
    _dataAccessor = new ZKHelixDataAccessor(CLUSTER_NAME, _baseAccessor);
    ConfigAccessor configAccessor = new ConfigAccessor(_gZkClient);
    ClusterConfig clusterConfig = configAccessor.getClusterConfig(CLUSTER_NAME);
    clusterConfig.setMaxOfflineInstancesAllowed(_maxOfflineInstancesAllowed);
    configAccessor.setClusterConfig(CLUSTER_NAME, clusterConfig);
    for (int i = 0; i < 3; i++) {
        String db = "Test-DB-" + i++;
        createResourceWithDelayedRebalance(CLUSTER_NAME, db, BuiltInStateModelDefinitions.MasterSlave.name(), _PARTITIONS, 3, 3, -1);
    }
    Thread.sleep(100);
    Assert.assertTrue(_clusterVerifier.verify());
}
Also used : ClusterControllerManager(org.apache.helix.integration.manager.ClusterControllerManager) MockParticipantManager(org.apache.helix.integration.manager.MockParticipantManager) BestPossibleExternalViewVerifier(org.apache.helix.tools.ClusterVerifiers.BestPossibleExternalViewVerifier) ConfigAccessor(org.apache.helix.ConfigAccessor) Date(java.util.Date) ZKHelixDataAccessor(org.apache.helix.manager.zk.ZKHelixDataAccessor) ClusterConfig(org.apache.helix.model.ClusterConfig) BeforeClass(org.testng.annotations.BeforeClass)

Aggregations

ZKHelixDataAccessor (org.apache.helix.manager.zk.ZKHelixDataAccessor)104 ZNRecord (org.apache.helix.ZNRecord)78 Date (java.util.Date)66 Test (org.testng.annotations.Test)66 Builder (org.apache.helix.PropertyKey.Builder)47 MockParticipantManager (org.apache.helix.integration.manager.MockParticipantManager)41 HelixDataAccessor (org.apache.helix.HelixDataAccessor)38 ClusterControllerManager (org.apache.helix.integration.manager.ClusterControllerManager)37 IdealState (org.apache.helix.model.IdealState)33 ZkBaseDataAccessor (org.apache.helix.manager.zk.ZkBaseDataAccessor)30 PropertyKey (org.apache.helix.PropertyKey)29 ExternalView (org.apache.helix.model.ExternalView)21 BestPossAndExtViewZkVerifier (org.apache.helix.tools.ClusterStateVerifier.BestPossAndExtViewZkVerifier)21 LiveInstance (org.apache.helix.model.LiveInstance)20 ClusterStateVerifier (org.apache.helix.tools.ClusterStateVerifier)16 HelixManager (org.apache.helix.HelixManager)15 Message (org.apache.helix.model.Message)12 ZKHelixAdmin (org.apache.helix.manager.zk.ZKHelixAdmin)11 HashMap (java.util.HashMap)10 ZkClient (org.apache.helix.manager.zk.ZkClient)10