Examples with BestPossAndExtViewZkVerifier org.apache.helix.tools.ClusterStateVerifier.BestPossAndExtViewZkVerifier used on opensource projects

Example 41 with BestPossAndExtViewZkVerifier

use of org.apache.helix.tools.ClusterStateVerifier.BestPossAndExtViewZkVerifier in project helix by apache.

the class TestDistributedControllerManager method simpleIntegrationTest.

@Test
public void simpleIntegrationTest() throws Exception {
    // Logger.getRootLogger().setLevel(Level.INFO);
    String className = TestHelper.getTestClassName();
    String methodName = TestHelper.getTestMethodName();
    String clusterName = className + "_" + methodName;
    int n = 2;
    System.out.println("START " + clusterName + " at " + new Date(System.currentTimeMillis()));
    // participant port
    TestHelper.setupCluster(// participant port
    clusterName, // participant port
    ZK_ADDR, // participant port
    12918, // participant name prefix
    "localhost", // resource name prefix
    "TestDB", // resources
    1, // partitions per resource
    4, // number of nodes
    n, // replicas
    2, "MasterSlave", // do rebalance
    true);
    HelixManager[] distributedControllers = new HelixManager[n];
    for (int i = 0; i < n; i++) {
        int port = 12918 + i;
        distributedControllers[i] = new ZKHelixManager(clusterName, "localhost_" + port, InstanceType.CONTROLLER_PARTICIPANT, ZK_ADDR);
        distributedControllers[i].getStateMachineEngine().registerStateModelFactory("MasterSlave", new MockMSModelFactory());
        distributedControllers[i].connect();
    }
    boolean result = ClusterStateVerifier.verifyByZkCallback(new BestPossAndExtViewZkVerifier(ZK_ADDR, clusterName));
    Assert.assertTrue(result);
    // disconnect first distributed-controller, and verify second takes leadership
    distributedControllers[0].disconnect();
    // verify leader changes to localhost_12919
    Thread.sleep(100);
    result = ClusterStateVerifier.verifyByZkCallback(new BestPossAndExtViewZkVerifier(ZK_ADDR, clusterName));
    Assert.assertTrue(result);
    ZKHelixDataAccessor accessor = new ZKHelixDataAccessor(clusterName, new ZkBaseDataAccessor<ZNRecord>(_gZkClient));
    PropertyKey.Builder keyBuilder = accessor.keyBuilder();
    Assert.assertNull(accessor.getProperty(keyBuilder.liveInstance("localhost_12918")));
    LiveInstance leader = accessor.getProperty(keyBuilder.controllerLeader());
    Assert.assertNotNull(leader);
    Assert.assertEquals(leader.getId(), "localhost_12919");
    // clean up
    distributedControllers[1].disconnect();
    Assert.assertNull(accessor.getProperty(keyBuilder.liveInstance("localhost_12919")));
    Assert.assertNull(accessor.getProperty(keyBuilder.controllerLeader()));
    System.out.println("START " + clusterName + " at " + new Date(System.currentTimeMillis()));
}

Example 42 with BestPossAndExtViewZkVerifier

use of org.apache.helix.tools.ClusterStateVerifier.BestPossAndExtViewZkVerifier in project helix by apache.

the class TestEnableCompression method testEnableCompressionResource.

@Test()
public void testEnableCompressionResource() throws Exception {
    // Logger.getRootLogger().setLevel(Level.INFO);
    String className = TestHelper.getTestClassName();
    String methodName = TestHelper.getTestMethodName();
    String clusterName = className + "_" + methodName;
    System.out.println("START " + clusterName + " at " + new Date(System.currentTimeMillis()));
    MockParticipantManager[] participants = new MockParticipantManager[5];
    // ClusterSetup setupTool = new ClusterSetup(ZK_ADDR);
    int numNodes = 10;
    // participant port
    TestHelper.setupCluster(// participant port
    clusterName, // participant port
    ZK_ADDR, // participant port
    12918, // participant name prefix
    "localhost", // resource name prefix
    "TestDB", // no resources, will be added later
    0, // partitions per resource
    0, // number of nodes
    numNodes, // replicas
    0, "OnlineOffline", // dont rebalance
    false);
    List<String> instancesInCluster = _gSetupTool.getClusterManagementTool().getInstancesInCluster(clusterName);
    String resourceName = "TestResource";
    CustomModeISBuilder customModeISBuilder = new CustomModeISBuilder(resourceName);
    int numPartitions = 10000;
    int numReplica = 3;
    customModeISBuilder.setNumPartitions(numPartitions);
    customModeISBuilder.setNumReplica(numReplica);
    customModeISBuilder.setStateModel("OnlineOffline");
    for (int p = 0; p < numPartitions; p++) {
        String partitionName = resourceName + "_" + p;
        customModeISBuilder.add(partitionName);
        for (int r = 0; r < numReplica; r++) {
            String instanceName = instancesInCluster.get((p % numNodes + r) % numNodes);
            customModeISBuilder.assignInstanceAndState(partitionName, instanceName, "ONLINE");
        }
    }
    IdealState idealstate = customModeISBuilder.build();
    idealstate.getRecord().setBooleanField("enableCompression", true);
    _gSetupTool.getClusterManagementTool().addResource(clusterName, resourceName, idealstate);
    ZkClient zkClient = new ZkClient(ZK_ADDR, 60 * 1000, 60 * 1000, new BytesPushThroughSerializer());
    zkClient.waitUntilConnected(10, TimeUnit.SECONDS);
    ClusterControllerManager controller = new ClusterControllerManager(ZK_ADDR, clusterName, "controller_0");
    controller.syncStart();
    // start participants
    for (int i = 0; i < 5; i++) {
        String instanceName = "localhost_" + (12918 + i);
        participants[i] = new MockParticipantManager(ZK_ADDR, clusterName, instanceName);
        participants[i].syncStart();
    }
    boolean result = ClusterStateVerifier.verifyByZkCallback(new BestPossAndExtViewZkVerifier(ZK_ADDR, clusterName), 120000);
    Assert.assertTrue(result);
    List<String> compressedPaths = new ArrayList<String>();
    findCompressedZNodes(zkClient, "/", compressedPaths);
    System.out.println("compressed paths:" + compressedPaths);
    // ONLY IDEALSTATE and EXTERNAL VIEW must be compressed
    Assert.assertEquals(compressedPaths.size(), 2);
    String idealstatePath = PropertyPathBuilder.idealState(clusterName, resourceName);
    String externalViewPath = PropertyPathBuilder.externalView(clusterName, resourceName);
    Assert.assertTrue(compressedPaths.contains(idealstatePath));
    Assert.assertTrue(compressedPaths.contains(externalViewPath));
    // clean up
    controller.syncStop();
    for (int i = 0; i < 5; i++) {
        participants[i].syncStop();
    }
    System.out.println("END " + clusterName + " at " + new Date(System.currentTimeMillis()));
}

Example 43 with BestPossAndExtViewZkVerifier

use of org.apache.helix.tools.ClusterStateVerifier.BestPossAndExtViewZkVerifier in project helix by apache.

the class TestExternalViewUpdates method testExternalViewUpdates.

@Test
public void testExternalViewUpdates() throws Exception {
    System.out.println("START testExternalViewUpdates at " + new Date(System.currentTimeMillis()));
    String clusterName = getShortClassName();
    MockParticipantManager[] participants = new MockParticipantManager[5];
    int resourceNb = 10;
    // participant port
    TestHelper.setupCluster(// participant port
    clusterName, // participant port
    ZK_ADDR, // participant port
    12918, // participant name prefix
    "localhost", // resource name prefix
    "TestDB", // resources
    resourceNb, // partitions per resource
    1, // number of nodes
    5, // replicas
    1, "MasterSlave", // do rebalance
    true);
    // start participants
    for (int i = 0; i < 5; i++) {
        String instanceName = "localhost_" + (12918 + i);
        participants[i] = new MockParticipantManager(ZK_ADDR, clusterName, instanceName);
        participants[i].syncStart();
    }
    // start controller after participants to trigger rebalance immediate after the controller is ready
    ClusterControllerManager controller = new ClusterControllerManager(ZK_ADDR, clusterName, "controller_0");
    controller.syncStart();
    boolean result = ClusterStateVerifier.verifyByZkCallback(new MasterNbInExtViewVerifier(ZK_ADDR, clusterName));
    Assert.assertTrue(result);
    result = ClusterStateVerifier.verifyByZkCallback(new BestPossAndExtViewZkVerifier(ZK_ADDR, clusterName));
    Assert.assertTrue(result);
    // need to verify that each ExternalView's version number is 2
    Builder keyBuilder = new Builder(clusterName);
    ZkBaseDataAccessor<ZNRecord> accessor = new ZkBaseDataAccessor<ZNRecord>(_gZkClient);
    String parentPath = keyBuilder.externalViews().getPath();
    List<String> childNames = accessor.getChildNames(parentPath, 0);
    List<String> paths = new ArrayList<String>();
    for (String name : childNames) {
        paths.add(parentPath + "/" + name);
    }
    // Stat[] stats = accessor.getStats(paths);
    for (String path : paths) {
        Stat stat = accessor.getStat(path, 0);
        Assert.assertTrue(stat.getVersion() <= 2, "ExternalView should be updated at most 2 times");
    }
    // clean up
    controller.syncStop();
    for (int i = 0; i < 5; i++) {
        participants[i].syncStop();
    }
    System.out.println("END testExternalViewUpdates at " + new Date(System.currentTimeMillis()));
}

Example 44 with BestPossAndExtViewZkVerifier

use of org.apache.helix.tools.ClusterStateVerifier.BestPossAndExtViewZkVerifier in project helix by apache.

the class TestInvalidAutoIdealState method testInvalidReplica2.

// TODO Disable this test, need refactor it for testing message generation based on state priority
// @Test
void testInvalidReplica2() throws Exception {
    HelixAdmin admin = new ZKHelixAdmin(ZK_ADDR);
    // create cluster
    String className = TestHelper.getTestClassName();
    String methodName = TestHelper.getTestMethodName();
    String clusterName = className + "_" + methodName;
    String db = "TestDB";
    System.out.println("START " + clusterName + " at " + new Date(System.currentTimeMillis()));
    // System.out.println("Creating cluster: " + clusterName);
    admin.addCluster(clusterName, true);
    // add MasterSlave state mode definition
    admin.addStateModelDef(clusterName, "MasterSlave", new StateModelDefinition(StateModelConfigGenerator.generateConfigForMasterSlave()));
    // Add nodes to the cluster
    int n = 3;
    System.out.println("Adding " + n + " participants to the cluster");
    for (int i = 0; i < n; i++) {
        int port = 12918 + i;
        InstanceConfig instanceConfig = new InstanceConfig("localhost_" + port);
        instanceConfig.setHostName("localhost");
        instanceConfig.setPort("" + port);
        instanceConfig.setInstanceEnabled(true);
        admin.addInstance(clusterName, instanceConfig);
    // System.out.println("\t Added participant: " + instanceConfig.getInstanceName());
    }
    // construct ideal-state manually
    IdealState idealState = new IdealState(db);
    idealState.setRebalanceMode(RebalanceMode.SEMI_AUTO);
    idealState.setNumPartitions(2);
    // should be 3
    idealState.setReplicas("" + 2);
    idealState.setStateModelDefRef("MasterSlave");
    idealState.getRecord().setListField("TestDB_0", Arrays.asList("localhost_12918", "localhost_12919", "localhost_12920"));
    idealState.getRecord().setListField("TestDB_1", Arrays.asList("localhost_12919", "localhost_12918", "localhost_12920"));
    admin.setResourceIdealState(clusterName, "TestDB", idealState);
    // start participants
    MockParticipantManager[] participants = new MockParticipantManager[n];
    for (int i = 0; i < n; i++) {
        String instanceName = "localhost_" + (12918 + i);
        participants[i] = new MockParticipantManager(ZK_ADDR, clusterName, instanceName);
        participants[i].syncStart();
    }
    ClusterControllerManager controller = new ClusterControllerManager(ZK_ADDR, clusterName, "controller_0");
    controller.syncStart();
    boolean result = ClusterStateVerifier.verifyByZkCallback(new BestPossAndExtViewZkVerifier(ZK_ADDR, clusterName));
    Assert.assertTrue(result);
    // make sure localhost_12919 is master on TestDB_1
    HelixDataAccessor accessor = controller.getHelixDataAccessor();
    Builder keyBuilder = accessor.keyBuilder();
    ExternalView extView = accessor.getProperty(keyBuilder.externalView(db));
    Map<String, String> stateMap = extView.getStateMap(db + "_1");
    Assert.assertEquals(stateMap.get("localhost_12919"), "MASTER", "localhost_12919 should be MASTER even though replicas is set to 2, since we generate message based on target-state priority");
    System.out.println("END " + clusterName + " at " + new Date(System.currentTimeMillis()));
}

Example 45 with BestPossAndExtViewZkVerifier

use of org.apache.helix.tools.ClusterStateVerifier.BestPossAndExtViewZkVerifier in project helix by apache.

the class TestHelixAdminCli method testExpandCluster.

@Test
public void testExpandCluster() throws Exception {
    String className = TestHelper.getTestClassName();
    String methodName = TestHelper.getTestMethodName();
    String clusterName = className + "_" + methodName;
    String grandClusterName = clusterName + "_grand";
    final int n = 6;
    System.out.println("START " + clusterName + " at " + new Date(System.currentTimeMillis()));
    MockParticipantManager[] participants = new MockParticipantManager[n];
    ClusterDistributedController[] controllers = new ClusterDistributedController[2];
    setupCluster(clusterName, grandClusterName, n, participants, controllers);
    String command = "-zkSvr " + ZK_ADDR + " -activateCluster " + clusterName + " " + grandClusterName + " true";
    ClusterSetup.processCommandLineArgs(command.split("\\s+"));
    Thread.sleep(500);
    command = "-zkSvr " + ZK_ADDR + " -addNode " + clusterName + " localhost:12331;localhost:12341;localhost:12351;localhost:12361";
    ClusterSetup.processCommandLineArgs(command.split("\\s+"));
    command = "-zkSvr localhost:2183 -expandCluster " + clusterName;
    ClusterSetup.processCommandLineArgs(command.split("\\s+"));
    MockParticipantManager[] newParticipants = new MockParticipantManager[4];
    for (int i = 3; i <= 6; i++) {
        String instanceName = "localhost_123" + i + "1";
        newParticipants[i - 3] = new MockParticipantManager(ZK_ADDR, clusterName, instanceName);
        newParticipants[i - 3].syncStart();
    }
    boolean verifyResult = ClusterStateVerifier.verifyByZkCallback(new MasterNbInExtViewVerifier(ZK_ADDR, clusterName));
    Assert.assertTrue(verifyResult);
    verifyResult = ClusterStateVerifier.verifyByZkCallback(new BestPossAndExtViewZkVerifier(ZK_ADDR, clusterName));
    Assert.assertTrue(verifyResult);
    // clean up
    for (int i = 0; i < controllers.length; i++) {
        controllers[i].syncStop();
    }
    for (int i = 0; i < participants.length; i++) {
        participants[i].syncStop();
    }
    for (int i = 0; i < newParticipants.length; i++) {
        newParticipants[i].syncStop();
    }
    System.out.println("END " + clusterName + " at " + new Date(System.currentTimeMillis()));
}