Examples with HelixClusterVerifier org.apache.helix.tools.ClusterVerifiers.HelixClusterVerifier used on opensource projects

Example 31 with HelixClusterVerifier

use of org.apache.helix.tools.ClusterVerifiers.HelixClusterVerifier in project helix by apache.

the class TestZkCallbackHandlerLeak method testRemoveUserCbHdlrOnPathRemoval.

@Test
public void testRemoveUserCbHdlrOnPathRemoval() throws Exception {
    String className = TestHelper.getTestClassName();
    String methodName = TestHelper.getTestMethodName();
    String clusterName = className + "_" + methodName;
    final int n = 3;
    final String zkAddr = ZK_ADDR;
    System.out.println("START " + clusterName + " at " + new Date(System.currentTimeMillis()));
    // resource
    TestHelper.setupCluster(// resource
    clusterName, // resource
    zkAddr, // resource
    12918, // resource
    "localhost", // resource
    "TestDB", // resource
    1, // partitions
    32, // nodes
    n, // replicas
    2, "MasterSlave", true);
    final ClusterControllerManager controller = new ClusterControllerManager(zkAddr, clusterName, "controller");
    controller.syncStart();
    MockParticipantManager[] participants = new MockParticipantManager[n];
    for (int i = 0; i < n; i++) {
        String instanceName = "localhost_" + (12918 + i);
        participants[i] = new MockParticipantManager(zkAddr, clusterName, instanceName);
        participants[i].syncStart();
        // register a controller listener on participant_0
        if (i == 0) {
            MockParticipantManager manager = participants[0];
            manager.addCurrentStateChangeListener(new CurrentStateChangeListener() {

                @Override
                public void onStateChange(String instanceName, List<CurrentState> statesInfo, NotificationContext changeContext) {
                // To change body of implemented methods use File | Settings | File Templates.
                // System.out.println(instanceName + " on current-state change, type: " +
                // changeContext.getType());
                }
            }, manager.getInstanceName(), manager.getSessionId());
        }
    }
    HelixClusterVerifier verifier = new BestPossibleExternalViewVerifier.Builder(clusterName).setZkAddr(zkAddr).build();
    Assert.assertTrue(verifier.verify());
    MockParticipantManager participantToExpire = participants[0];
    String oldSessionId = participantToExpire.getSessionId();
    PropertyKey.Builder keyBuilder = new PropertyKey.Builder(clusterName);
    // check manager#hanlders
    Assert.assertEquals(participantToExpire.getHandlers().size(), 2, "Should have 2 handlers: CURRENTSTATE/{sessionId}, and MESSAGES");
    // check zkclient#listeners
    Map<String, Set<IZkDataListener>> dataListeners = ZkTestHelper.getZkDataListener(participantToExpire.getZkClient());
    Map<String, Set<IZkChildListener>> childListeners = ZkTestHelper.getZkChildListener(participantToExpire.getZkClient());
    // printZkListeners(participantToExpire.getZkClient());
    Assert.assertEquals(dataListeners.size(), 1, "Should have 1 path (CURRENTSTATE/{sessionId}/TestDB0) which has 1 data-listeners");
    String path = keyBuilder.currentState(participantToExpire.getInstanceName(), oldSessionId, "TestDB0").getPath();
    Assert.assertEquals(dataListeners.get(path).size(), 1, "Should have 1 data-listeners on path: " + path);
    Assert.assertEquals(childListeners.size(), 2, "Should have 2 paths (CURRENTSTATE/{sessionId}, and MESSAGES) each of which has 1 child-listener");
    path = keyBuilder.currentStates(participantToExpire.getInstanceName(), oldSessionId).getPath();
    Assert.assertEquals(childListeners.get(path).size(), 1, "Should have 1 child-listener on path: " + path);
    path = keyBuilder.messages(participantToExpire.getInstanceName()).getPath();
    Assert.assertEquals(childListeners.get(path).size(), 1, "Should have 1 child-listener on path: " + path);
    path = keyBuilder.controller().getPath();
    Assert.assertNull(childListeners.get(path), "Should have no child-listener on path: " + path);
    // check zookeeper#watches on client side
    Map<String, List<String>> watchPaths = ZkTestHelper.getZkWatch(participantToExpire.getZkClient());
    LOG.debug("localhost_12918 zk-client side watchPaths: " + watchPaths);
    Assert.assertEquals(watchPaths.get("dataWatches").size(), 3, "Should have 3 data-watches: CURRENTSTATE/{sessionId}, CURRENTSTATE/{sessionId}/TestDB, MESSAGES");
    Assert.assertEquals(watchPaths.get("childWatches").size(), 2, "Should have 2 child-watches: MESSAGES, and CURRENTSTATE/{sessionId}");
    // expire localhost_12918
    System.out.println("Expire participant: " + participantToExpire.getInstanceName() + ", session: " + participantToExpire.getSessionId());
    ZkTestHelper.expireSession(participantToExpire.getZkClient());
    String newSessionId = participantToExpire.getSessionId();
    System.out.println(participantToExpire.getInstanceName() + " oldSessionId: " + oldSessionId + ", newSessionId: " + newSessionId);
    Assert.assertTrue(verifier.verify());
    // check manager#hanlders
    Assert.assertEquals(participantToExpire.getHandlers().size(), 1, "Should have 1 handler: MESSAGES. CURRENTSTATE/{sessionId} handler should be removed by CallbackHandler#handleChildChange()");
    // check zkclient#listeners
    dataListeners = ZkTestHelper.getZkDataListener(participantToExpire.getZkClient());
    childListeners = ZkTestHelper.getZkChildListener(participantToExpire.getZkClient());
    // printZkListeners(participantToExpire.getZkClient());
    Assert.assertTrue(dataListeners.isEmpty(), "Should have no data-listeners");
    Assert.assertEquals(childListeners.size(), 2, "Should have 2 paths (CURRENTSTATE/{oldSessionId}, and MESSAGES). " + "MESSAGE has 1 child-listener each. CURRENTSTATE/{oldSessionId} doesn't have listener (ZkClient doesn't remove empty childListener set. probably a ZkClient bug. see ZkClient#unsubscribeChildChange())");
    path = keyBuilder.currentStates(participantToExpire.getInstanceName(), oldSessionId).getPath();
    Assert.assertEquals(childListeners.get(path).size(), 0, "Should have no child-listener on path: " + path);
    path = keyBuilder.messages(participantToExpire.getInstanceName()).getPath();
    Assert.assertEquals(childListeners.get(path).size(), 1, "Should have 1 child-listener on path: " + path);
    path = keyBuilder.controller().getPath();
    Assert.assertNull(childListeners.get(path), "Should have no child-listener on path: " + path);
    // check zookeeper#watches on client side
    watchPaths = ZkTestHelper.getZkWatch(participantToExpire.getZkClient());
    LOG.debug("localhost_12918 zk-client side watchPaths: " + watchPaths);
    Assert.assertEquals(watchPaths.get("dataWatches").size(), 1, "Should have 1 data-watch: MESSAGES");
    Assert.assertEquals(watchPaths.get("childWatches").size(), 1, "Should have 1 child-watch: MESSAGES");
    Assert.assertEquals(watchPaths.get("existWatches").size(), 2, "Should have 2 exist-watches: CURRENTSTATE/{oldSessionId} and CURRENTSTATE/{oldSessionId}/TestDB0");
    // another session expiry on localhost_12918 should clear the two exist-watches on
    // CURRENTSTATE/{oldSessionId}
    System.out.println("Expire participant: " + participantToExpire.getInstanceName() + ", session: " + participantToExpire.getSessionId());
    ZkTestHelper.expireSession(participantToExpire.getZkClient());
    Assert.assertTrue(verifier.verify());
    // check zookeeper#watches on client side
    watchPaths = ZkTestHelper.getZkWatch(participantToExpire.getZkClient());
    LOG.debug("localhost_12918 zk-client side watchPaths: " + watchPaths);
    Assert.assertEquals(watchPaths.get("dataWatches").size(), 1, "Should have 1 data-watch: MESSAGES");
    Assert.assertEquals(watchPaths.get("childWatches").size(), 1, "Should have 1 child-watch: MESSAGES");
    Assert.assertEquals(watchPaths.get("existWatches").size(), 0, "Should have no exist-watches. exist-watches on CURRENTSTATE/{oldSessionId} and CURRENTSTATE/{oldSessionId}/TestDB0 should be cleared during handleNewSession");
    // Thread.sleep(1000);
    System.out.println("END " + clusterName + " at " + new Date(System.currentTimeMillis()));
}

Example 32 with HelixClusterVerifier

use of org.apache.helix.tools.ClusterVerifiers.HelixClusterVerifier in project helix by apache.

the class TestRebalanceRunningTask method testFixedTargetTaskAndDisabledRebalanceAndNodeAdded.

/**
 * Task type: fixed target
 * Rebalance raunning task: disabled
 * Story: new node added
 */
@Test
public void testFixedTargetTaskAndDisabledRebalanceAndNodeAdded() throws InterruptedException {
    WORKFLOW = TestHelper.getTestMethodName();
    JobConfig.Builder jobBuilder = new JobConfig.Builder().setWorkflow(WORKFLOW).setTargetResource(DATABASE).setTargetPartitionStates(Sets.newHashSet(MasterSlaveSMD.States.MASTER.name())).setNumConcurrentTasksPerInstance(100).setFailureThreshold(2).setMaxAttemptsPerTask(2).setCommand(MockTask.TASK_COMMAND).setJobCommandConfigMap(// task stuck
    ImmutableMap.of(MockTask.TIMEOUT_CONFIG, "99999999"));
    Workflow.Builder workflowBuilder = new Workflow.Builder(WORKFLOW).addJob(JOB, jobBuilder);
    _driver.start(workflowBuilder.build());
    // All tasks stuck on the same instance
    Assert.assertTrue(checkTasksOnSameInstances());
    // Add a new instance, partition is rebalanced
    startParticipant(_initialNumNodes);
    HelixClusterVerifier clusterVerifier = new BestPossibleExternalViewVerifier.Builder(CLUSTER_NAME).setZkClient(_gZkClient).setResources(Sets.newHashSet(DATABASE)).build();
    Assert.assertTrue(clusterVerifier.verify(10 * 1000));
    // Running tasks are also rebalanced, even though RebalanceRunningTask is disabled
    Assert.assertTrue(checkTasksOnDifferentInstances());
}

Example 33 with HelixClusterVerifier

use of org.apache.helix.tools.ClusterVerifiers.HelixClusterVerifier in project helix by apache.

the class TestDisable method testDisableNodeCustomIS.

@Test
public void testDisableNodeCustomIS() throws Exception {
    // Logger.getRootLogger().setLevel(Level.INFO);
    String className = TestHelper.getTestClassName();
    String methodName = TestHelper.getTestMethodName();
    String clusterName = className + "_" + methodName;
    final int n = 5;
    String disableNode = "localhost_12918";
    System.out.println("START " + clusterName + " at " + new Date(System.currentTimeMillis()));
    MockParticipantManager[] participants = new MockParticipantManager[n];
    // participant port
    TestHelper.setupCluster(// participant port
    clusterName, // participant port
    ZK_ADDR, // participant port
    12918, // participant name prefix
    "localhost", // resource name prefix
    "TestDB", // resources
    1, // partitions per resource
    8, // number of nodes
    n, // replicas
    3, "MasterSlave", // do rebalance
    true);
    // set ideal state to customized mode
    ZkBaseDataAccessor<ZNRecord> baseAccessor = new ZkBaseDataAccessor<ZNRecord>(_gZkClient);
    ZKHelixDataAccessor accessor = new ZKHelixDataAccessor(clusterName, baseAccessor);
    Builder keyBuilder = accessor.keyBuilder();
    IdealState idealState = accessor.getProperty(keyBuilder.idealStates("TestDB0"));
    idealState.setRebalanceMode(RebalanceMode.CUSTOMIZED);
    accessor.setProperty(keyBuilder.idealStates("TestDB0"), idealState);
    // start controller
    ClusterControllerManager controller = new ClusterControllerManager(ZK_ADDR, clusterName, "controller_0");
    controller.syncStart();
    // start participants
    for (int i = 0; i < n; i++) {
        String instanceName = "localhost_" + (12918 + i);
        participants[i] = new MockParticipantManager(ZK_ADDR, clusterName, instanceName);
        participants[i].syncStart();
    }
    HelixClusterVerifier _clusterVerifier = new BestPossibleExternalViewVerifier.Builder(clusterName).setZkAddr(ZK_ADDR).build();
    Assert.assertTrue(_clusterVerifier.verify());
    // disable localhost_12918
    String command = "--zkSvr " + ZK_ADDR + " --enableInstance " + clusterName + " " + disableNode + " false";
    ClusterSetup.processCommandLineArgs(command.split("\\s+"));
    Assert.assertTrue(_clusterVerifier.verify());
    // make sure localhost_12918 is in OFFLINE state
    Map<String, Map<String, String>> expectStateMap = new HashMap<String, Map<String, String>>();
    Map<String, String> expectInstanceStateMap = new HashMap<String, String>();
    expectInstanceStateMap.put(disableNode, "OFFLINE");
    expectStateMap.put(".*", expectInstanceStateMap);
    boolean result = ZkTestHelper.verifyState(_gZkClient, clusterName, "TestDB0", expectStateMap, "==");
    Assert.assertTrue(result, disableNode + " should be in OFFLINE");
    // re-enable localhost_12918
    command = "--zkSvr " + ZK_ADDR + " --enableInstance " + clusterName + " " + disableNode + " true";
    ClusterSetup.processCommandLineArgs(command.split("\\s+"));
    Assert.assertTrue(_clusterVerifier.verify());
    // make sure localhost_12918 is NOT in OFFLINE state
    result = ZkTestHelper.verifyState(_gZkClient, clusterName, "TestDB0", expectStateMap, "!=");
    Assert.assertTrue(result, disableNode + " should NOT be in OFFLINE");
    // clean up
    // wait for all zk callbacks done
    controller.syncStop();
    for (int i = 0; i < 5; i++) {
        participants[i].syncStop();
    }
    System.out.println("END " + clusterName + " at " + new Date(System.currentTimeMillis()));
}

Example 34 with HelixClusterVerifier

use of org.apache.helix.tools.ClusterVerifiers.HelixClusterVerifier in project helix by apache.

the class TestDrop method testDropSchemataResource.

@Test
public void testDropSchemataResource() throws Exception {
    // Logger.getRootLogger().setLevel(Level.INFO);
    String className = TestHelper.getTestClassName();
    String methodName = TestHelper.getTestMethodName();
    String clusterName = className + "_" + methodName;
    final int n = 5;
    System.out.println("START " + clusterName + " at " + new Date(System.currentTimeMillis()));
    MockParticipantManager[] participants = new MockParticipantManager[n];
    // participant port
    TestHelper.setupCluster(// participant port
    clusterName, // participant port
    ZK_ADDR, // participant port
    12918, // participant name prefix
    "localhost", // resource name prefix
    "TestDB", // resources
    1, // partitions per resource
    8, // number of nodes
    n, // replicas
    3, "MasterSlave", // do rebalance
    true);
    // start controller
    ClusterControllerManager controller = new ClusterControllerManager(ZK_ADDR, clusterName, "controller_0");
    controller.syncStart();
    // start participants
    for (int i = 0; i < n; i++) {
        String instanceName = "localhost_" + (12918 + i);
        participants[i] = new MockParticipantManager(ZK_ADDR, clusterName, instanceName);
        participants[i].syncStart();
    }
    HelixClusterVerifier verifier = new BestPossibleExternalViewVerifier.Builder(clusterName).setZkAddr(ZK_ADDR).build();
    Assert.assertTrue(verifier.verify());
    // add schemata resource group
    String command = "--zkSvr " + ZK_ADDR + " --addResource " + clusterName + " schemata 1 STORAGE_DEFAULT_SM_SCHEMATA";
    ClusterSetup.processCommandLineArgs(command.split("\\s+"));
    command = "--zkSvr " + ZK_ADDR + " --rebalance " + clusterName + " schemata " + n;
    ClusterSetup.processCommandLineArgs(command.split("\\s+"));
    Assert.assertTrue(verifier.verify());
    // drop schemata resource group
    // System.out.println("Dropping schemata resource group...");
    command = "--zkSvr " + ZK_ADDR + " --dropResource " + clusterName + " schemata";
    ClusterSetup.processCommandLineArgs(command.split("\\s+"));
    Assert.assertTrue(verifier.verify());
    Thread.sleep(2000);
    assertEmptyCSandEV(clusterName, "schemata", participants);
    // clean up
    controller.syncStop();
    for (int i = 0; i < n; i++) {
        participants[i].syncStop();
    }
    System.out.println("END " + clusterName + " at " + new Date(System.currentTimeMillis()));
}

Example 35 with HelixClusterVerifier

use of org.apache.helix.tools.ClusterVerifiers.HelixClusterVerifier in project helix by apache.

the class TestDrop method testDropErrorPartitionCustomIS.

@Test
public void testDropErrorPartitionCustomIS() throws Exception {
    // Logger.getRootLogger().setLevel(Level.INFO);
    String className = TestHelper.getTestClassName();
    String methodName = TestHelper.getTestMethodName();
    String clusterName = className + "_" + methodName;
    final int n = 2;
    System.out.println("START " + clusterName + " at " + new Date(System.currentTimeMillis()));
    MockParticipantManager[] participants = new MockParticipantManager[n];
    // participant port
    TestHelper.setupCluster(// participant port
    clusterName, // participant port
    ZK_ADDR, // participant port
    12918, // participant name prefix
    "localhost", // resource name prefix
    "TestDB", // resources
    1, // partitions per resource
    2, // number of nodes
    n, // replicas
    2, "MasterSlave", // do rebalance
    false);
    // set custom ideal-state
    CustomModeISBuilder isBuilder = new CustomModeISBuilder("TestDB0");
    isBuilder.setNumPartitions(2);
    isBuilder.setNumReplica(2);
    isBuilder.setStateModel("MasterSlave");
    isBuilder.assignInstanceAndState("TestDB0_0", "localhost_12918", "MASTER");
    isBuilder.assignInstanceAndState("TestDB0_0", "localhost_12919", "SLAVE");
    isBuilder.assignInstanceAndState("TestDB0_1", "localhost_12919", "MASTER");
    isBuilder.assignInstanceAndState("TestDB0_1", "localhost_12918", "SLAVE");
    HelixDataAccessor accessor = new ZKHelixDataAccessor(clusterName, new ZkBaseDataAccessor<ZNRecord>(_gZkClient));
    PropertyKey.Builder keyBuilder = accessor.keyBuilder();
    accessor.setProperty(keyBuilder.idealStates("TestDB0"), isBuilder.build());
    // start controller
    ClusterControllerManager controller = new ClusterControllerManager(ZK_ADDR, clusterName, "controller_0");
    controller.syncStart();
    // start participants
    Map<String, Set<String>> errTransitions = new HashMap<String, Set<String>>();
    errTransitions.put("SLAVE-MASTER", TestHelper.setOf("TestDB0_0"));
    for (int i = 0; i < n; i++) {
        String instanceName = "localhost_" + (12918 + i);
        if (i == 0) {
            participants[i] = new MockParticipantManager(ZK_ADDR, clusterName, instanceName);
            participants[i].setTransition(new ErrTransition(errTransitions));
        } else {
            participants[i] = new MockParticipantManager(ZK_ADDR, clusterName, instanceName);
        }
        participants[i].syncStart();
    }
    Map<String, Map<String, String>> errStateMap = new HashMap<String, Map<String, String>>();
    errStateMap.put("TestDB0", new HashMap<String, String>());
    errStateMap.get("TestDB0").put("TestDB0_0", "localhost_12918");
    HelixClusterVerifier verifier = new BestPossibleExternalViewVerifier.Builder(clusterName).setZkAddr(ZK_ADDR).setErrStates(errStateMap).build();
    Assert.assertTrue(verifier.verify());
    // drop resource containing error partitions should drop the partition successfully
    ClusterSetup.processCommandLineArgs(new String[] { "--zkSvr", ZK_ADDR, "--dropResource", clusterName, "TestDB0" });
    // make sure TestDB0_0 partition is dropped
    verifier = new BestPossibleExternalViewVerifier.Builder(clusterName).setZkAddr(ZK_ADDR).build();
    Assert.assertTrue(verifier.verify(), "Should be empty exeternal-view");
    assertEmptyCSandEV(clusterName, "TestDB0", participants);
    // clean up
    controller.syncStop();
    for (int i = 0; i < n; i++) {
        participants[i].syncStop();
    }
    System.out.println("END " + clusterName + " at " + new Date(System.currentTimeMillis()));
}