Examples with MockParticipantManager org.apache.helix.integration.manager.MockParticipantManager used on opensource projects

Example 41 with MockParticipantManager

use of org.apache.helix.integration.manager.MockParticipantManager in project helix by apache.

the class TestStateTransitionCancellation method testRebalancingCauseCancellation.

@Test
public void testRebalancingCauseCancellation() throws InterruptedException {
    // Enable cancellation
    ClusterConfig clusterConfig = _configAccessor.getClusterConfig(CLUSTER_NAME);
    clusterConfig.stateTransitionCancelEnabled(true);
    clusterConfig.setPersistBestPossibleAssignment(true);
    _configAccessor.setClusterConfig(CLUSTER_NAME, clusterConfig);
    // Reenable resource
    stateCleanUp();
    _setupTool.getClusterManagementTool().enableResource(CLUSTER_NAME, WorkflowGenerator.DEFAULT_TGT_DB, true);
    // Wait for assignment done
    Thread.sleep(2000);
    int numNodesToStart = 10;
    for (int i = 0; i < numNodesToStart; i++) {
        String storageNodeName = PARTICIPANT_PREFIX + "_" + (_startPort + _numNodes + i);
        _setupTool.addInstanceToCluster(CLUSTER_NAME, storageNodeName);
    }
    MockParticipantManager[] newParticipants = new MockParticipantManager[numNodesToStart];
    registerParticipants(newParticipants, numNodesToStart, _startPort + _numNodes, 1000, -3000000L);
    // Wait for pipeline reaching final stage
    Thread.sleep(2000L);
    int numOfMasters = 0;
    ExternalView externalView = _setupTool.getClusterManagementTool().getResourceExternalView(CLUSTER_NAME, WorkflowGenerator.DEFAULT_TGT_DB);
    for (String partition : externalView.getPartitionSet()) {
        if (externalView.getStateMap(partition).values().contains("MASTER")) {
            numOfMasters++;
        }
    }
    for (MockParticipantManager participant : newParticipants) {
        participant.syncStop();
    }
    // Either partial of state transitions have been cancelled or all the Slave -> Master
    // reassigned to other cluster
    Assert.assertTrue((numOfMasters > 0 && numOfMasters <= _numParitions));
}

Example 42 with MockParticipantManager

use of org.apache.helix.integration.manager.MockParticipantManager in project helix by apache.

the class TestStateTransitionThrottle method testTransitionThrottleOnErrorPartition.

@Test
public void testTransitionThrottleOnErrorPartition() throws Exception {
    String clusterName = getShortClassName() + "testMaxErrorPartition";
    MockParticipantManager[] participants = new MockParticipantManager[participantCount];
    System.out.println("START " + clusterName + " at " + new Date(System.currentTimeMillis()));
    final ZKHelixDataAccessor accessor = new ZKHelixDataAccessor(clusterName, new ZkBaseDataAccessor<ZNRecord>(_gZkClient));
    setupCluster(clusterName, accessor);
    // Set throttle config to enable throttling
    PropertyKey.Builder keyBuilder = accessor.keyBuilder();
    ClusterConfig clusterConfig = accessor.getProperty(accessor.keyBuilder().clusterConfig());
    clusterConfig.setResourcePriorityField("Name");
    List<StateTransitionThrottleConfig> throttleConfigs = new ArrayList<>();
    throttleConfigs.add(new StateTransitionThrottleConfig(StateTransitionThrottleConfig.RebalanceType.LOAD_BALANCE, StateTransitionThrottleConfig.ThrottleScope.CLUSTER, 100));
    throttleConfigs.add(new StateTransitionThrottleConfig(StateTransitionThrottleConfig.RebalanceType.RECOVERY_BALANCE, StateTransitionThrottleConfig.ThrottleScope.CLUSTER, 100));
    clusterConfig.setStateTransitionThrottleConfigs(throttleConfigs);
    accessor.setProperty(keyBuilder.clusterConfig(), clusterConfig);
    // set one partition to be always Error, so load balance won't be triggered
    Map<String, Set<String>> errPartitions = new HashMap<>();
    errPartitions.put("OFFLINE-SLAVE", TestHelper.setOf(resourceName + "_0"));
    // start part of participants
    for (int i = 0; i < participantCount - 1; i++) {
        participants[i] = new MockParticipantManager(ZK_ADDR, clusterName, "localhost_" + (12918 + i));
        if (i == 0) {
            participants[i].setTransition(new ErrTransition(errPartitions));
        }
        participants[i].syncStart();
    }
    ClusterControllerManager controller = new ClusterControllerManager(ZK_ADDR, clusterName, "controller_0");
    controller.syncStart();
    BestPossibleExternalViewVerifier verifier = new BestPossibleExternalViewVerifier.Builder(clusterName).setZkClient(_gZkClient).build();
    Assert.assertTrue(verifier.verify(3000));
    // Adding one more participant.
    participants[participantCount - 1] = new MockParticipantManager(ZK_ADDR, clusterName, "localhost_" + (12918 + participantCount - 1));
    participants[participantCount - 1].syncStart();
    // Since error partition exists, no load balance transition will be done
    Assert.assertFalse(pollForPartitionAssignment(accessor, participants[participantCount - 1], resourceName, 5000));
    // Update cluster config to tolerate error partition, so load balance transition will be done
    clusterConfig = accessor.getProperty(accessor.keyBuilder().clusterConfig());
    clusterConfig.setErrorPartitionThresholdForLoadBalance(1);
    accessor.setProperty(keyBuilder.clusterConfig(), clusterConfig);
    _gSetupTool.rebalanceResource(clusterName, resourceName, 3);
    Assert.assertTrue(pollForPartitionAssignment(accessor, participants[participantCount - 1], resourceName, 3000));
    // clean up
    controller.syncStop();
    for (int i = 0; i < participantCount; i++) {
        participants[i].syncStop();
    }
    System.out.println("END " + clusterName + " at " + new Date(System.currentTimeMillis()));
}

Example 43 with MockParticipantManager

use of org.apache.helix.integration.manager.MockParticipantManager in project helix by apache.

the class TestSyncSessionToController method testSyncSessionToController.

@Test
public void testSyncSessionToController() throws Exception {
    System.out.println("START testSyncSessionToController at " + new Date(System.currentTimeMillis()));
    String clusterName = getShortClassName();
    MockParticipantManager[] participants = new MockParticipantManager[5];
    int resourceNb = 10;
    // participant port
    TestHelper.setupCluster(// participant port
    clusterName, // participant port
    ZK_ADDR, // participant port
    12918, // participant name prefix
    "localhost", // resource name prefix
    "TestDB", // resources
    resourceNb, // partitions per resource
    1, // number of nodes
    5, // replicas
    1, "MasterSlave", // do rebalance
    true);
    ClusterControllerManager controller = new ClusterControllerManager(ZK_ADDR, clusterName, "controller_0");
    controller.syncStart();
    // start participants
    for (int i = 0; i < 5; i++) {
        String instanceName = "localhost_" + (12918 + i);
        participants[i] = new MockParticipantManager(ZK_ADDR, clusterName, instanceName);
        participants[i].syncStart();
    }
    ZKHelixManager zkHelixManager = new ZKHelixManager(clusterName, "controllerMessageListener", InstanceType.CONTROLLER, ZK_ADDR);
    zkHelixManager.connect();
    MockMessageListener mockMessageListener = new MockMessageListener();
    zkHelixManager.addControllerMessageListener(mockMessageListener);
    PropertyKey.Builder keyBuilder = new PropertyKey.Builder(clusterName);
    ZkBaseDataAccessor<ZNRecord> accessor = new ZkBaseDataAccessor<ZNRecord>(_gZkClient);
    String path = keyBuilder.liveInstance("localhost_12918").getPath();
    Stat stat = new Stat();
    ZNRecord data = accessor.get(path, stat, 2);
    data.getSimpleFields().put("SESSION_ID", "invalid-id");
    accessor.set(path, data, 2);
    Thread.sleep(2000);
    Assert.assertTrue(mockMessageListener.isSessionSyncMessageSent());
}

Example 44 with MockParticipantManager

use of org.apache.helix.integration.manager.MockParticipantManager in project helix by apache.

the class TestZkCallbackHandlerLeak method testCbHandlerLeakOnParticipantSessionExpiry.

@Test
public void testCbHandlerLeakOnParticipantSessionExpiry() throws Exception {
    // Logger.getRootLogger().setLevel(Level.INFO);
    String className = TestHelper.getTestClassName();
    String methodName = TestHelper.getTestMethodName();
    String clusterName = className + "_" + methodName;
    final int n = 2;
    final int r = 2;
    System.out.println("START " + clusterName + " at " + new Date(System.currentTimeMillis()));
    // participant port
    TestHelper.setupCluster(// participant port
    clusterName, // participant port
    ZK_ADDR, // participant port
    12918, // participant name prefix
    "localhost", // resource name prefix
    "TestDB", // resources
    r, // partitions per resource
    32, // number of nodes
    n, // replicas
    2, "MasterSlave", // do rebalance
    true);
    final ClusterControllerManager controller = new ClusterControllerManager(ZK_ADDR, clusterName, "controller_0");
    controller.syncStart();
    // start participants
    MockParticipantManager[] participants = new MockParticipantManager[n];
    for (int i = 0; i < n; i++) {
        String instanceName = "localhost_" + (12918 + i);
        participants[i] = new MockParticipantManager(ZK_ADDR, clusterName, instanceName);
        participants[i].syncStart();
    }
    HelixClusterVerifier verifier = new BestPossibleExternalViewVerifier.Builder(clusterName).setZkAddr(ZK_ADDR).build();
    Assert.assertTrue(verifier.verify());
    final MockParticipantManager participantManagerToExpire = participants[1];
    // check controller zk-watchers
    boolean result = TestHelper.verify(new TestHelper.Verifier() {

        @Override
        public boolean verify() throws Exception {
            Map<String, Set<String>> watchers = ZkTestHelper.getListenersBySession(ZK_ADDR);
            // Set<String> watchPaths = watchers.get("0x" + controllerManager.getSessionId());
            Set<String> watchPaths = watchers.get("0x" + controller.getSessionId());
            // where n is number of nodes and r is number of resources
            return watchPaths.size() == (7 + r + (4 + r) * n);
        }
    }, 2000);
    Assert.assertTrue(result, "Controller has incorrect number of zk-watchers.");
    // check participant zk-watchers
    result = TestHelper.verify(new TestHelper.Verifier() {

        @Override
        public boolean verify() throws Exception {
            Map<String, Set<String>> watchers = ZkTestHelper.getListenersBySession(ZK_ADDR);
            Set<String> watchPaths = watchers.get("0x" + participantManagerToExpire.getSessionId());
            // participant should have 1 zk-watcher: 1 for MESSAGE
            return watchPaths.size() == 1;
        }
    }, 2000);
    Assert.assertTrue(result, "Participant should have 1 zk-watcher. MESSAGES->HelixTaskExecutor");
    // check HelixManager#_handlers
    // printHandlers(controllerManager);
    // printHandlers(participantManagerToExpire);
    int controllerHandlerNb = controller.getHandlers().size();
    int particHandlerNb = participantManagerToExpire.getHandlers().size();
    Assert.assertEquals(controllerHandlerNb, 11, "HelixController should have 10 (5+2n) callback handlers for 2 (n) participant");
    Assert.assertEquals(particHandlerNb, 1, "HelixParticipant should have 1 (msg->HelixTaskExecutor) callback handlers");
    // expire the session of participant
    System.out.println("Expiring participant session...");
    String oldSessionId = participantManagerToExpire.getSessionId();
    ZkTestHelper.expireSession(participantManagerToExpire.getZkClient());
    String newSessionId = participantManagerToExpire.getSessionId();
    System.out.println("Expried participant session. oldSessionId: " + oldSessionId + ", newSessionId: " + newSessionId);
    result = ClusterStateVerifier.verifyByPolling(new ClusterStateVerifier.BestPossAndExtViewZkVerifier(ZK_ADDR, clusterName));
    Assert.assertTrue(result);
    // check controller zk-watchers
    result = TestHelper.verify(new TestHelper.Verifier() {

        @Override
        public boolean verify() throws Exception {
            Map<String, Set<String>> watchers = ZkTestHelper.getListenersBySession(ZK_ADDR);
            Set<String> watchPaths = watchers.get("0x" + controller.getSessionId());
            // where n is number of nodes and r is number of resources
            return watchPaths.size() == (7 + r + (4 + r) * n);
        }
    }, 2000);
    Assert.assertTrue(result, "Controller has incorrect number of zk-watchers after session expiry.");
    // check participant zk-watchers
    result = TestHelper.verify(new TestHelper.Verifier() {

        @Override
        public boolean verify() throws Exception {
            Map<String, Set<String>> watchers = ZkTestHelper.getListenersBySession(ZK_ADDR);
            Set<String> watchPaths = watchers.get("0x" + participantManagerToExpire.getSessionId());
            // participant should have 1 zk-watcher: 1 for MESSAGE
            return watchPaths.size() == 1;
        }
    }, 2000);
    Assert.assertTrue(result, "Participant should have 1 zk-watcher after session expiry.");
    // check handlers
    // printHandlers(controllerManager);
    // printHandlers(participantManagerToExpire);
    int handlerNb = controller.getHandlers().size();
    Assert.assertEquals(handlerNb, controllerHandlerNb, "controller callback handlers should not increase after participant session expiry");
    handlerNb = participantManagerToExpire.getHandlers().size();
    Assert.assertEquals(handlerNb, particHandlerNb, "participant callback handlers should not increase after participant session expiry");
    // clean up
    controller.syncStop();
    for (int i = 0; i < n; i++) {
        participants[i].syncStop();
    }
    System.out.println("END " + clusterName + " at " + new Date(System.currentTimeMillis()));
}

Example 45 with MockParticipantManager

use of org.apache.helix.integration.manager.MockParticipantManager in project helix by apache.

the class TestZkCallbackHandlerLeak method testCbHandlerLeakOnControllerSessionExpiry.

@Test
public void testCbHandlerLeakOnControllerSessionExpiry() throws Exception {
    // Logger.getRootLogger().setLevel(Level.INFO);
    String className = TestHelper.getTestClassName();
    String methodName = TestHelper.getTestMethodName();
    String clusterName = className + "_" + methodName;
    final int n = 2;
    final int r = 1;
    System.out.println("START " + clusterName + " at " + new Date(System.currentTimeMillis()));
    // participant port
    TestHelper.setupCluster(// participant port
    clusterName, // participant port
    ZK_ADDR, // participant port
    12918, // participant name prefix
    "localhost", // resource name prefix
    "TestDB", // resources
    r, // partitions per resource
    32, // number of nodes
    n, // replicas
    2, "MasterSlave", // do rebalance
    true);
    final ClusterControllerManager controller = new ClusterControllerManager(ZK_ADDR, clusterName, "controller_0");
    controller.syncStart();
    // start participants
    MockParticipantManager[] participants = new MockParticipantManager[n];
    for (int i = 0; i < n; i++) {
        String instanceName = "localhost_" + (12918 + i);
        participants[i] = new MockParticipantManager(ZK_ADDR, clusterName, instanceName);
        participants[i].syncStart();
    }
    HelixClusterVerifier verifier = new BestPossibleExternalViewVerifier.Builder(clusterName).setZkAddr(ZK_ADDR).build();
    Assert.assertTrue(verifier.verify());
    final MockParticipantManager participantManager = participants[0];
    // wait until we get all the listeners registered
    TestHelper.verify(new TestHelper.Verifier() {

        @Override
        public boolean verify() throws Exception {
            int controllerHandlerNb = controller.getHandlers().size();
            int particHandlerNb = participantManager.getHandlers().size();
            if (controllerHandlerNb == 10 && particHandlerNb == 2)
                return true;
            else
                return false;
        }
    }, 1000);
    int controllerHandlerNb = controller.getHandlers().size();
    int particHandlerNb = participantManager.getHandlers().size();
    Assert.assertEquals(controllerHandlerNb, 11, "HelixController should have 10 (6+2n) callback handlers for 2 participant, but was " + controllerHandlerNb + ", " + printHandlers(controller));
    Assert.assertEquals(particHandlerNb, 1, "HelixParticipant should have 1 (msg->HelixTaskExecutor) callback handler, but was " + particHandlerNb + ", " + printHandlers(participantManager));
    // expire controller
    System.out.println("Expiring controller session...");
    String oldSessionId = controller.getSessionId();
    ZkTestHelper.expireSession(controller.getZkClient());
    String newSessionId = controller.getSessionId();
    System.out.println("Expired controller session. oldSessionId: " + oldSessionId + ", newSessionId: " + newSessionId);
    Assert.assertTrue(verifier.verify());
    // check controller zk-watchers
    boolean result = TestHelper.verify(new TestHelper.Verifier() {

        @Override
        public boolean verify() throws Exception {
            Map<String, Set<String>> watchers = ZkTestHelper.getListenersBySession(ZK_ADDR);
            Set<String> watchPaths = watchers.get("0x" + controller.getSessionId());
            System.err.println("controller watch paths after session expiry: " + watchPaths.size());
            // where r is number of resources and n is number of nodes
            int expected = (7 + r + (4 + r) * n);
            return watchPaths.size() == expected;
        }
    }, 2000);
    Assert.assertTrue(result, "Controller has incorrect zk-watchers after session expiry.");
    // check participant zk-watchers
    result = TestHelper.verify(new TestHelper.Verifier() {

        @Override
        public boolean verify() throws Exception {
            Map<String, Set<String>> watchers = ZkTestHelper.getListenersBySession(ZK_ADDR);
            Set<String> watchPaths = watchers.get("0x" + participantManager.getSessionId());
            // participant should have 1 zk-watcher: 1 for MESSAGE
            return watchPaths.size() == 1;
        }
    }, 2000);
    Assert.assertTrue(result, "Participant should have 1 zk-watcher after session expiry.");
    // check HelixManager#_handlers
    // printHandlers(controllerManager);
    int handlerNb = controller.getHandlers().size();
    Assert.assertEquals(handlerNb, controllerHandlerNb, "controller callback handlers should not increase after participant session expiry, but was " + printHandlers(controller));
    handlerNb = participantManager.getHandlers().size();
    Assert.assertEquals(handlerNb, particHandlerNb, "participant callback handlers should not increase after participant session expiry, but was " + printHandlers(participantManager));
    // clean up
    controller.syncStop();
    for (int i = 0; i < n; i++) {
        participants[i].syncStop();
    }
    System.out.println("END " + clusterName + " at " + new Date(System.currentTimeMillis()));
}