Search in sources :

Example 51 with MockParticipantManager

use of org.apache.helix.integration.manager.MockParticipantManager in project helix by apache.

the class TestDrop method testDropErrorPartitionFailedAutoIS.

@Test
public void testDropErrorPartitionFailedAutoIS() throws Exception {
    // Logger.getRootLogger().setLevel(Level.INFO);
    String className = TestHelper.getTestClassName();
    String methodName = TestHelper.getTestMethodName();
    String clusterName = className + "_" + methodName;
    final int n = 5;
    System.out.println("START " + clusterName + " at " + new Date(System.currentTimeMillis()));
    MockParticipantManager[] participants = new MockParticipantManager[n];
    // participant port
    TestHelper.setupCluster(// participant port
    clusterName, // participant port
    ZK_ADDR, // participant port
    12918, // participant name prefix
    "localhost", // resource name prefix
    "TestDB", // resources
    1, // partitions per resource
    8, // number of nodes
    n, // replicas
    3, "MasterSlave", // do rebalance
    true);
    // start controller
    ClusterControllerManager controller = new ClusterControllerManager(ZK_ADDR, clusterName, "controller_0");
    controller.syncStart();
    // start participants
    Map<String, Set<String>> errTransitions = new HashMap<String, Set<String>>();
    errTransitions.put("SLAVE-MASTER", TestHelper.setOf("TestDB0_4"));
    errTransitions.put("ERROR-DROPPED", TestHelper.setOf("TestDB0_4"));
    for (int i = 0; i < n; i++) {
        String instanceName = "localhost_" + (12918 + i);
        if (i == 0) {
            participants[i] = new MockParticipantManager(ZK_ADDR, clusterName, instanceName);
            participants[i].setTransition(new ErrTransition(errTransitions));
        } else {
            participants[i] = new MockParticipantManager(ZK_ADDR, clusterName, instanceName);
        }
        participants[i].syncStart();
    }
    Map<String, Map<String, String>> errStateMap = new HashMap<>();
    errStateMap.put("TestDB0", new HashMap<String, String>());
    errStateMap.get("TestDB0").put("TestDB0_4", "localhost_12918");
    HelixClusterVerifier verifier = new BestPossibleExternalViewVerifier.Builder(clusterName).setZkAddr(ZK_ADDR).setErrStates(errStateMap).build();
    Assert.assertTrue(verifier.verify());
    // drop resource containing error partitions should invoke error->dropped transition
    // if error happens during error->dropped transition, partition should be disabled
    ClusterSetup.processCommandLineArgs(new String[] { "--zkSvr", ZK_ADDR, "--dropResource", clusterName, "TestDB0" });
    Thread.sleep(100);
    // make sure TestDB0_4 stay in ERROR state and is disabled
    Assert.assertTrue(verifier.verify());
    ZKHelixDataAccessor accessor = new ZKHelixDataAccessor(clusterName, new ZkBaseDataAccessor<ZNRecord>(_gZkClient));
    PropertyKey.Builder keyBuilder = accessor.keyBuilder();
    InstanceConfig config = accessor.getProperty(keyBuilder.instanceConfig("localhost_12918"));
    List<String> disabledPartitions = config.getDisabledPartitions();
    // System.out.println("disabledPartitions: " + disabledPartitions);
    Assert.assertEquals(disabledPartitions.size(), 1, "TestDB0_4 should be disabled");
    Assert.assertEquals(disabledPartitions.get(0), "TestDB0_4");
    // ExteranlView should have TestDB0_4->localhost_12918_>ERROR
    Thread.sleep(2000);
    ExternalView ev = accessor.getProperty(keyBuilder.externalView("TestDB0"));
    Set<String> partitions = ev.getPartitionSet();
    Assert.assertEquals(partitions.size(), 1, "Should have TestDB0_4->localhost_12918->ERROR");
    String errPartition = partitions.iterator().next();
    Assert.assertEquals(errPartition, "TestDB0_4");
    Map<String, String> stateMap = ev.getStateMap(errPartition);
    Assert.assertEquals(stateMap.size(), 1);
    Assert.assertEquals(stateMap.keySet().iterator().next(), "localhost_12918");
    Assert.assertEquals(stateMap.get("localhost_12918"), HelixDefinedState.ERROR.name());
    // localhost_12918 should have TestDB0_4 in ERROR state
    CurrentState cs = accessor.getProperty(keyBuilder.currentState(participants[0].getInstanceName(), participants[0].getSessionId(), "TestDB0"));
    Map<String, String> partitionStateMap = cs.getPartitionStateMap();
    Assert.assertEquals(partitionStateMap.size(), 1);
    Assert.assertEquals(partitionStateMap.keySet().iterator().next(), "TestDB0_4");
    Assert.assertEquals(partitionStateMap.get("TestDB0_4"), HelixDefinedState.ERROR.name());
    // all other participants should have cleaned up empty current state
    for (int i = 1; i < n; i++) {
        String instanceName = participants[i].getInstanceName();
        String sessionId = participants[i].getSessionId();
        Assert.assertNull(accessor.getProperty(keyBuilder.currentState(instanceName, sessionId, "TestDB0")));
    }
    // clean up
    controller.syncStop();
    for (int i = 0; i < n; i++) {
        participants[i].syncStop();
    }
    System.out.println("END " + clusterName + " at " + new Date(System.currentTimeMillis()));
}
Also used : ExternalView(org.apache.helix.model.ExternalView) Set(java.util.Set) MockParticipantManager(org.apache.helix.integration.manager.MockParticipantManager) HashMap(java.util.HashMap) InstanceConfig(org.apache.helix.model.InstanceConfig) CurrentState(org.apache.helix.model.CurrentState) BestPossibleExternalViewVerifier(org.apache.helix.tools.ClusterVerifiers.BestPossibleExternalViewVerifier) ZNRecord(org.apache.helix.ZNRecord) ZKHelixDataAccessor(org.apache.helix.manager.zk.ZKHelixDataAccessor) HelixClusterVerifier(org.apache.helix.tools.ClusterVerifiers.HelixClusterVerifier) ErrTransition(org.apache.helix.mock.participant.ErrTransition) Date(java.util.Date) ClusterControllerManager(org.apache.helix.integration.manager.ClusterControllerManager) HashMap(java.util.HashMap) Map(java.util.Map) PropertyKey(org.apache.helix.PropertyKey) Test(org.testng.annotations.Test)

Example 52 with MockParticipantManager

use of org.apache.helix.integration.manager.MockParticipantManager in project helix by apache.

the class TestDrop method testDropErrorPartitionAutoIS.

@Test
public void testDropErrorPartitionAutoIS() throws Exception {
    // Logger.getRootLogger().setLevel(Level.INFO);
    String className = TestHelper.getTestClassName();
    String methodName = TestHelper.getTestMethodName();
    String clusterName = className + "_" + methodName;
    final int n = 5;
    System.out.println("START " + clusterName + " at " + new Date(System.currentTimeMillis()));
    MockParticipantManager[] participants = new MockParticipantManager[n];
    // participant port
    TestHelper.setupCluster(// participant port
    clusterName, // participant port
    ZK_ADDR, // participant port
    12918, // participant name prefix
    "localhost", // resource name prefix
    "TestDB", // resources
    1, // partitions per resource
    10, // number of nodes
    n, // replicas
    3, "MasterSlave", // do rebalance
    true);
    // start controller
    ClusterControllerManager controller = new ClusterControllerManager(ZK_ADDR, clusterName, "controller_0");
    controller.syncStart();
    // start participants
    Map<String, Set<String>> errTransitions = new HashMap<String, Set<String>>();
    errTransitions.put("SLAVE-MASTER", TestHelper.setOf("TestDB0_4"));
    errTransitions.put("OFFLINE-SLAVE", TestHelper.setOf("TestDB0_8"));
    for (int i = 0; i < n; i++) {
        String instanceName = "localhost_" + (12918 + i);
        if (i == 0) {
            participants[i] = new MockParticipantManager(ZK_ADDR, clusterName, instanceName);
            participants[i].setTransition(new ErrTransition(errTransitions));
        } else {
            participants[i] = new MockParticipantManager(ZK_ADDR, clusterName, instanceName);
        }
        participants[i].syncStart();
    }
    Map<String, Map<String, String>> errStateMap = new HashMap<String, Map<String, String>>();
    errStateMap.put("TestDB0", new HashMap<String, String>());
    errStateMap.get("TestDB0").put("TestDB0_4", "localhost_12918");
    errStateMap.get("TestDB0").put("TestDB0_8", "localhost_12918");
    HelixClusterVerifier verifier = new BestPossibleExternalViewVerifier.Builder(clusterName).setZkAddr(ZK_ADDR).setErrStates(errStateMap).build();
    Assert.assertTrue(verifier.verify());
    // drop resource containing error partitions should drop the partition successfully
    ClusterSetup.processCommandLineArgs(new String[] { "--zkSvr", ZK_ADDR, "--dropResource", clusterName, "TestDB0" });
    // make sure TestDB0_4 and TestDB0_8 partitions are dropped
    verifier = new BestPossibleExternalViewVerifier.Builder(clusterName).setZkAddr(ZK_ADDR).build();
    Assert.assertTrue(verifier.verify());
    assertEmptyCSandEV(className, "TestDB0", participants);
    // clean up
    controller.syncStop();
    for (int i = 0; i < n; i++) {
        participants[i].syncStop();
    }
    System.out.println("END " + clusterName + " at " + new Date(System.currentTimeMillis()));
}
Also used : HelixClusterVerifier(org.apache.helix.tools.ClusterVerifiers.HelixClusterVerifier) Set(java.util.Set) MockParticipantManager(org.apache.helix.integration.manager.MockParticipantManager) HashMap(java.util.HashMap) ErrTransition(org.apache.helix.mock.participant.ErrTransition) Date(java.util.Date) ClusterControllerManager(org.apache.helix.integration.manager.ClusterControllerManager) BestPossibleExternalViewVerifier(org.apache.helix.tools.ClusterVerifiers.BestPossibleExternalViewVerifier) HashMap(java.util.HashMap) Map(java.util.Map) Test(org.testng.annotations.Test)

Example 53 with MockParticipantManager

use of org.apache.helix.integration.manager.MockParticipantManager in project helix by apache.

the class TestErrorPartition method testErrorPartition.

@Test()
public void testErrorPartition() throws Exception {
    String clusterName = getShortClassName();
    MockParticipantManager[] participants = new MockParticipantManager[5];
    System.out.println("START testErrorPartition() at " + new Date(System.currentTimeMillis()));
    ZKHelixAdmin tool = new ZKHelixAdmin(_gZkClient);
    TestHelper.setupCluster(clusterName, ZK_ADDR, 12918, "localhost", "TestDB", 1, 10, 5, 3, "MasterSlave", true);
    ClusterControllerManager controller = new ClusterControllerManager(ZK_ADDR, clusterName, "controller_0");
    controller.syncStart();
    for (int i = 0; i < 5; i++) {
        String instanceName = "localhost_" + (12918 + i);
        if (i == 0) {
            Map<String, Set<String>> errPartitions = new HashMap<String, Set<String>>() {

                {
                    put("SLAVE-MASTER", TestHelper.setOf("TestDB0_4"));
                }
            };
            participants[i] = new MockParticipantManager(ZK_ADDR, clusterName, instanceName);
            participants[i].setTransition(new ErrTransition(errPartitions));
        } else {
            participants[i] = new MockParticipantManager(ZK_ADDR, clusterName, instanceName);
        }
        participants[i].syncStart();
    }
    Map<String, Map<String, String>> errStates = new HashMap<String, Map<String, String>>();
    errStates.put("TestDB0", new HashMap<String, String>());
    errStates.get("TestDB0").put("TestDB0_4", "localhost_12918");
    boolean result = ClusterStateVerifier.verifyByPolling(new ClusterStateVerifier.BestPossAndExtViewZkVerifier(ZK_ADDR, clusterName, errStates));
    Assert.assertTrue(result);
    Map<String, Set<String>> errorStateMap = new HashMap<String, Set<String>>() {

        {
            put("TestDB0_4", TestHelper.setOf("localhost_12918"));
        }
    };
    // verify "TestDB0_0", "localhost_12918" is in ERROR state
    TestHelper.verifyState(clusterName, ZK_ADDR, errorStateMap, "ERROR");
    // disable a partition on a node with error state
    tool.enablePartition(false, clusterName, "localhost_12918", "TestDB0", Arrays.asList("TestDB0_4"));
    result = ClusterStateVerifier.verifyByPolling(new ClusterStateVerifier.BestPossAndExtViewZkVerifier(ZK_ADDR, clusterName, errStates));
    Assert.assertTrue(result);
    TestHelper.verifyState(clusterName, ZK_ADDR, errorStateMap, "ERROR");
    // disable a node with error state
    tool.enableInstance(clusterName, "localhost_12918", false);
    result = ClusterStateVerifier.verifyByPolling(new ClusterStateVerifier.BestPossAndExtViewZkVerifier(ZK_ADDR, clusterName, errStates));
    Assert.assertTrue(result);
    // make sure after restart stale ERROR state is gone
    tool.enablePartition(true, clusterName, "localhost_12918", "TestDB0", Arrays.asList("TestDB0_4"));
    tool.enableInstance(clusterName, "localhost_12918", true);
    participants[0].syncStop();
    result = ClusterStateVerifier.verifyByPolling(new ClusterStateVerifier.BestPossAndExtViewZkVerifier(ZK_ADDR, clusterName));
    Assert.assertTrue(result);
    participants[0] = new MockParticipantManager(ZK_ADDR, clusterName, "localhost_12918");
    new Thread(participants[0]).start();
    result = ClusterStateVerifier.verifyByPolling(new ClusterStateVerifier.BestPossAndExtViewZkVerifier(ZK_ADDR, clusterName));
    Assert.assertTrue(result);
    // clean up
    controller.syncStop();
    for (int i = 0; i < 5; i++) {
        participants[i].syncStop();
    }
    System.out.println("END testErrorPartition() at " + new Date(System.currentTimeMillis()));
}
Also used : Set(java.util.Set) MockParticipantManager(org.apache.helix.integration.manager.MockParticipantManager) HashMap(java.util.HashMap) ClusterStateVerifier(org.apache.helix.tools.ClusterStateVerifier) ErrTransition(org.apache.helix.mock.participant.ErrTransition) Date(java.util.Date) ClusterControllerManager(org.apache.helix.integration.manager.ClusterControllerManager) ZKHelixAdmin(org.apache.helix.manager.zk.ZKHelixAdmin) HashMap(java.util.HashMap) Map(java.util.Map) Test(org.testng.annotations.Test)

Example 54 with MockParticipantManager

use of org.apache.helix.integration.manager.MockParticipantManager in project helix by apache.

the class TestBatchMessage method testBasic.

@Test
public void testBasic() throws Exception {
    // Logger.getRootLogger().setLevel(Level.INFO);
    String className = TestHelper.getTestClassName();
    String methodName = TestHelper.getTestMethodName();
    String clusterName = className + "_" + methodName;
    int n = 2;
    System.out.println("START " + clusterName + " at " + new Date(System.currentTimeMillis()));
    // participant port
    TestHelper.setupCluster(// participant port
    clusterName, // participant port
    ZK_ADDR, // participant port
    12918, // participant name prefix
    "localhost", // resource name prefix
    "TestDB", // resources
    1, // partitions per resource
    32, // number of nodes
    n, // replicas
    2, "MasterSlave", // do rebalance
    true);
    // enable batch message
    ZKHelixDataAccessor accessor = new ZKHelixDataAccessor(clusterName, new ZkBaseDataAccessor<ZNRecord>(_gZkClient));
    Builder keyBuilder = accessor.keyBuilder();
    IdealState idealState = accessor.getProperty(keyBuilder.idealStates("TestDB0"));
    idealState.setBatchMessageMode(true);
    accessor.setProperty(keyBuilder.idealStates("TestDB0"), idealState);
    // register a message listener so we know how many message generated
    TestZkChildListener listener = new TestZkChildListener();
    _gZkClient.subscribeChildChanges(keyBuilder.messages("localhost_12918").getPath(), listener);
    ClusterControllerManager controller = new ClusterControllerManager(ZK_ADDR, clusterName, "controller_0");
    controller.syncStart();
    // start participants
    MockParticipantManager[] participants = new MockParticipantManager[n];
    for (int i = 0; i < n; i++) {
        String instanceName = "localhost_" + (12918 + i);
        participants[i] = new MockParticipantManager(ZK_ADDR, clusterName, instanceName);
        participants[i].syncStart();
    }
    boolean result = ClusterStateVerifier.verifyByZkCallback(new BestPossAndExtViewZkVerifier(ZK_ADDR, clusterName));
    Assert.assertTrue(result);
    // Change to three is because there is an extra factory registered
    // So one extra NO_OP message send
    Assert.assertTrue(listener._maxNbOfChilds <= 3, "Should get no more than 2 messages (O->S and S->M)");
    // clean up
    // wait for all zk callbacks done
    controller.syncStop();
    for (int i = 0; i < n; i++) {
        participants[i].syncStop();
    }
    System.out.println("END " + clusterName + " at " + new Date(System.currentTimeMillis()));
}
Also used : MockParticipantManager(org.apache.helix.integration.manager.MockParticipantManager) Builder(org.apache.helix.PropertyKey.Builder) BestPossAndExtViewZkVerifier(org.apache.helix.tools.ClusterStateVerifier.BestPossAndExtViewZkVerifier) Date(java.util.Date) IdealState(org.apache.helix.model.IdealState) ClusterControllerManager(org.apache.helix.integration.manager.ClusterControllerManager) ZNRecord(org.apache.helix.ZNRecord) ZKHelixDataAccessor(org.apache.helix.manager.zk.ZKHelixDataAccessor) Test(org.testng.annotations.Test)

Example 55 with MockParticipantManager

use of org.apache.helix.integration.manager.MockParticipantManager in project helix by apache.

the class TestBatchMessage method testChangeBatchMessageMode.

// a non-batch-message run followed by a batch-message-enabled run
@Test
public void testChangeBatchMessageMode() throws Exception {
    // Logger.getRootLogger().setLevel(Level.INFO);
    String className = TestHelper.getTestClassName();
    String methodName = TestHelper.getTestMethodName();
    String clusterName = className + "_" + methodName;
    int n = 2;
    System.out.println("START " + clusterName + " at " + new Date(System.currentTimeMillis()));
    // participant port
    TestHelper.setupCluster(// participant port
    clusterName, // participant port
    ZK_ADDR, // participant port
    12918, // participant name prefix
    "localhost", // resource name prefix
    "TestDB", // resources
    1, // partitions per resource
    32, // number of nodes
    n, // replicas
    2, "MasterSlave", // do rebalance
    true);
    ClusterControllerManager controller = new ClusterControllerManager(ZK_ADDR, clusterName, "controller_0");
    controller.syncStart();
    // start participants
    MockParticipantManager[] participants = new MockParticipantManager[n];
    for (int i = 0; i < n; i++) {
        String instanceName = "localhost_" + (12918 + i);
        participants[i] = new MockParticipantManager(ZK_ADDR, clusterName, instanceName);
        participants[i].syncStart();
    }
    boolean result = ClusterStateVerifier.verifyByZkCallback(new BestPossAndExtViewZkVerifier(ZK_ADDR, clusterName));
    Assert.assertTrue(result);
    // stop all participants
    for (int i = 0; i < n; i++) {
        participants[i].syncStop();
    }
    // enable batch message
    ZKHelixDataAccessor accessor = new ZKHelixDataAccessor(clusterName, new ZkBaseDataAccessor<ZNRecord>(_gZkClient));
    Builder keyBuilder = accessor.keyBuilder();
    IdealState idealState = accessor.getProperty(keyBuilder.idealStates("TestDB0"));
    idealState.setBatchMessageMode(true);
    accessor.setProperty(keyBuilder.idealStates("TestDB0"), idealState);
    // registry a message listener so we know how many message generated
    TestZkChildListener listener = new TestZkChildListener();
    _gZkClient.subscribeChildChanges(keyBuilder.messages("localhost_12918").getPath(), listener);
    // restart all participants
    for (int i = 0; i < n; i++) {
        String instanceName = "localhost_" + (12918 + i);
        participants[i] = new MockParticipantManager(ZK_ADDR, clusterName, instanceName);
        participants[i].syncStart();
    }
    result = ClusterStateVerifier.verifyByZkCallback(new BestPossAndExtViewZkVerifier(ZK_ADDR, clusterName));
    Assert.assertTrue(result);
    // Change to three is because there is an extra factory registered
    // So one extra NO_OP message send
    Assert.assertTrue(listener._maxNbOfChilds <= 3, "Should get no more than 2 messages (O->S and S->M)");
    // clean up
    // wait for all zk callbacks done
    controller.syncStop();
    for (int i = 0; i < n; i++) {
        participants[i].syncStop();
    }
    System.out.println("END " + clusterName + " at " + new Date(System.currentTimeMillis()));
}
Also used : MockParticipantManager(org.apache.helix.integration.manager.MockParticipantManager) Builder(org.apache.helix.PropertyKey.Builder) BestPossAndExtViewZkVerifier(org.apache.helix.tools.ClusterStateVerifier.BestPossAndExtViewZkVerifier) Date(java.util.Date) IdealState(org.apache.helix.model.IdealState) ClusterControllerManager(org.apache.helix.integration.manager.ClusterControllerManager) ZNRecord(org.apache.helix.ZNRecord) ZKHelixDataAccessor(org.apache.helix.manager.zk.ZKHelixDataAccessor) Test(org.testng.annotations.Test)

Aggregations

MockParticipantManager (org.apache.helix.integration.manager.MockParticipantManager)167 Test (org.testng.annotations.Test)113 Date (java.util.Date)101 ClusterControllerManager (org.apache.helix.integration.manager.ClusterControllerManager)101 ZKHelixDataAccessor (org.apache.helix.manager.zk.ZKHelixDataAccessor)43 ZNRecord (org.apache.helix.ZNRecord)40 IdealState (org.apache.helix.model.IdealState)35 BestPossAndExtViewZkVerifier (org.apache.helix.tools.ClusterStateVerifier.BestPossAndExtViewZkVerifier)35 ClusterStateVerifier (org.apache.helix.tools.ClusterStateVerifier)34 HashMap (java.util.HashMap)33 HelixDataAccessor (org.apache.helix.HelixDataAccessor)30 ClusterSetup (org.apache.helix.tools.ClusterSetup)28 BeforeClass (org.testng.annotations.BeforeClass)27 ExternalView (org.apache.helix.model.ExternalView)24 BestPossibleExternalViewVerifier (org.apache.helix.tools.ClusterVerifiers.BestPossibleExternalViewVerifier)20 HelixClusterVerifier (org.apache.helix.tools.ClusterVerifiers.HelixClusterVerifier)19 Map (java.util.Map)18 PropertyKey (org.apache.helix.PropertyKey)16 Builder (org.apache.helix.PropertyKey.Builder)15 ClusterDistributedController (org.apache.helix.integration.manager.ClusterDistributedController)15