use of org.apache.helix.manager.zk.ZkBaseDataAccessor in project helix by apache.
the class TestEntropyFreeNodeBounce method testBounceAll.
@Test
public void testBounceAll() throws Exception {
// pick numbers that don't divide evenly
final int NUM_PARTICIPANTS = 5;
final int NUM_PARTITIONS = 123;
final int NUM_REPLICAS = 1;
final String RESOURCE_PREFIX = "TestDB";
final String RESOURCE_NAME = RESOURCE_PREFIX + "0";
// create a cluster name based on this test name
String className = TestHelper.getTestClassName();
String methodName = TestHelper.getTestMethodName();
String clusterName = className + "_" + methodName;
System.out.println("START " + clusterName + " at " + new Date(System.currentTimeMillis()));
// Set up cluster
// participant port
TestHelper.setupCluster(// participant port
clusterName, // participant port
ZK_ADDR, // participant port
12918, // participant name prefix
"localhost", // resource name prefix
"TestDB", // resources
1, // partitions per resource
NUM_PARTITIONS, // number of nodes
NUM_PARTICIPANTS, // replicas
NUM_REPLICAS, // use FULL_AUTO mode to test node tagging
"OnlineOffline", // use FULL_AUTO mode to test node tagging
RebalanceMode.FULL_AUTO, // do rebalance
true);
// Start the participants
HelixManager[] participants = new HelixManager[NUM_PARTICIPANTS];
for (int i = 0; i < NUM_PARTICIPANTS; i++) {
final String instanceName = "localhost_" + (12918 + i);
participants[i] = createParticipant(clusterName, instanceName);
participants[i].connect();
}
// Start the controller
ClusterControllerManager controller = new ClusterControllerManager(ZK_ADDR, clusterName, "controller");
controller.syncStart();
// get an admin and accessor
HelixAdmin helixAdmin = new ZKHelixAdmin(_gZkClient);
BaseDataAccessor<ZNRecord> baseAccessor = new ZkBaseDataAccessor<ZNRecord>(_gZkClient);
HelixDataAccessor accessor = new ZKHelixDataAccessor(clusterName, baseAccessor);
PropertyKey.Builder keyBuilder = accessor.keyBuilder();
// do the test
try {
Thread.sleep(1000);
// ensure that the external view coalesces
boolean result = ClusterStateVerifier.verifyByZkCallback(new BestPossAndExtViewZkVerifier(ZK_ADDR, clusterName));
Assert.assertTrue(result);
ExternalView stableExternalView = accessor.getProperty(keyBuilder.externalView(RESOURCE_NAME));
for (HelixManager participant : participants) {
// disable the controller, bounce the node, re-enable the controller, verify assignments
// remained the same
helixAdmin.enableCluster(clusterName, false);
participant.disconnect();
Thread.sleep(1000);
participant = createParticipant(clusterName, participant.getInstanceName());
participant.connect();
Thread.sleep(1000);
helixAdmin.enableCluster(clusterName, true);
Thread.sleep(1000);
result = ClusterStateVerifier.verifyByZkCallback(new MatchingExternalViewVerifier(stableExternalView, clusterName));
Assert.assertTrue(result);
}
} finally {
// clean up
controller.syncStop();
for (HelixManager participant : participants) {
participant.disconnect();
}
System.out.println("END " + clusterName + " at " + new Date(System.currentTimeMillis()));
}
}
use of org.apache.helix.manager.zk.ZkBaseDataAccessor in project helix by apache.
the class ZkTestHelper method verifyState.
/*
* stateMap: partition->instance->state
*/
public static boolean verifyState(ZkClient zkclient, String clusterName, String resourceName, Map<String, Map<String, String>> expectStateMap, String op) {
boolean result = true;
ZkBaseDataAccessor<ZNRecord> baseAccessor = new ZkBaseDataAccessor<ZNRecord>(zkclient);
ZKHelixDataAccessor accessor = new ZKHelixDataAccessor(clusterName, baseAccessor);
Builder keyBuilder = accessor.keyBuilder();
ExternalView extView = accessor.getProperty(keyBuilder.externalView(resourceName));
Map<String, Map<String, String>> actualStateMap = extView.getRecord().getMapFields();
for (String partition : actualStateMap.keySet()) {
for (String expectPartiton : expectStateMap.keySet()) {
if (!partition.matches(expectPartiton)) {
continue;
}
Map<String, String> actualInstanceStateMap = actualStateMap.get(partition);
Map<String, String> expectInstanceStateMap = expectStateMap.get(expectPartiton);
for (String instance : actualInstanceStateMap.keySet()) {
for (String expectInstance : expectStateMap.get(expectPartiton).keySet()) {
if (!instance.matches(expectInstance)) {
continue;
}
String actualState = actualInstanceStateMap.get(instance);
String expectState = expectInstanceStateMap.get(expectInstance);
boolean equals = expectState.equals(actualState);
if (op.equals("==") && !equals || op.equals("!=") && equals) {
System.out.println(partition + "/" + instance + " state mismatch. actual state: " + actualState + ", but expect: " + expectState + ", op: " + op);
result = false;
}
}
}
}
}
return result;
}
use of org.apache.helix.manager.zk.ZkBaseDataAccessor in project helix by apache.
the class TestMessageThrottleStage method testMsgThrottleConstraints.
@Test()
public void testMsgThrottleConstraints() throws Exception {
String clusterName = "CLUSTER_" + _className + "_constraints";
System.out.println("START " + clusterName + " at " + new Date(System.currentTimeMillis()));
HelixDataAccessor accessor = new ZKHelixDataAccessor(clusterName, new ZkBaseDataAccessor(_gZkClient));
HelixManager manager = new DummyClusterManager(clusterName, accessor);
// ideal state: node0 is MASTER, node1 is SLAVE
// replica=2 means 1 master and 1 slave
setupIdealState(clusterName, new int[] { 0, 1 }, new String[] { "TestDB" }, 1, 2);
setupLiveInstances(clusterName, new int[] { 0, 1 });
setupStateModel(clusterName);
// setup constraints
ZNRecord record = new ZNRecord(ConstraintType.MESSAGE_CONSTRAINT.toString());
// constraint0:
// "MESSAGE_TYPE=STATE_TRANSITION,CONSTRAINT_VALUE=ANY"
record.setMapField("constraint0", new TreeMap<String, String>());
record.getMapField("constraint0").put("MESSAGE_TYPE", "STATE_TRANSITION");
record.getMapField("constraint0").put("CONSTRAINT_VALUE", "ANY");
ConstraintItem constraint0 = new ConstraintItem(record.getMapField("constraint0"));
// constraint1:
// "MESSAGE_TYPE=STATE_TRANSITION,TRANSITION=OFFLINE-SLAVE,CONSTRAINT_VALUE=ANY"
record.setMapField("constraint1", new TreeMap<String, String>());
record.getMapField("constraint1").put("MESSAGE_TYPE", "STATE_TRANSITION");
record.getMapField("constraint1").put("TRANSITION", "OFFLINE-SLAVE");
record.getMapField("constraint1").put("CONSTRAINT_VALUE", "50");
ConstraintItem constraint1 = new ConstraintItem(record.getMapField("constraint1"));
// constraint2:
// "MESSAGE_TYPE=STATE_TRANSITION,TRANSITION=OFFLINE-SLAVE,INSTANCE=.*,RESOURCE=TestDB,CONSTRAINT_VALUE=2";
record.setMapField("constraint2", new TreeMap<String, String>());
record.getMapField("constraint2").put("MESSAGE_TYPE", "STATE_TRANSITION");
record.getMapField("constraint2").put("TRANSITION", "OFFLINE-SLAVE");
record.getMapField("constraint2").put("INSTANCE", ".*");
record.getMapField("constraint2").put("RESOURCE", "TestDB");
record.getMapField("constraint2").put("CONSTRAINT_VALUE", "2");
ConstraintItem constraint2 = new ConstraintItem(record.getMapField("constraint2"));
// constraint3:
// "MESSAGE_TYPE=STATE_TRANSITION,TRANSITION=OFFLINE-SLAVE,INSTANCE=localhost_12918,RESOURCE=.*,CONSTRAINT_VALUE=1";
record.setMapField("constraint3", new TreeMap<String, String>());
record.getMapField("constraint3").put("MESSAGE_TYPE", "STATE_TRANSITION");
record.getMapField("constraint3").put("TRANSITION", "OFFLINE-SLAVE");
record.getMapField("constraint3").put("INSTANCE", "localhost_1");
record.getMapField("constraint3").put("RESOURCE", ".*");
record.getMapField("constraint3").put("CONSTRAINT_VALUE", "1");
ConstraintItem constraint3 = new ConstraintItem(record.getMapField("constraint3"));
// constraint4:
// "MESSAGE_TYPE=STATE_TRANSITION,TRANSITION=OFFLINE-SLAVE,INSTANCE=.*,RESOURCE=.*,CONSTRAINT_VALUE=10"
record.setMapField("constraint4", new TreeMap<String, String>());
record.getMapField("constraint4").put("MESSAGE_TYPE", "STATE_TRANSITION");
record.getMapField("constraint4").put("TRANSITION", "OFFLINE-SLAVE");
record.getMapField("constraint4").put("INSTANCE", ".*");
record.getMapField("constraint4").put("RESOURCE", ".*");
record.getMapField("constraint4").put("CONSTRAINT_VALUE", "10");
ConstraintItem constraint4 = new ConstraintItem(record.getMapField("constraint4"));
// constraint5:
// "MESSAGE_TYPE=STATE_TRANSITION,TRANSITION=OFFLINE-SLAVE,INSTANCE=localhost_12918,RESOURCE=TestDB,CONSTRAINT_VALUE=5"
record.setMapField("constraint5", new TreeMap<String, String>());
record.getMapField("constraint5").put("MESSAGE_TYPE", "STATE_TRANSITION");
record.getMapField("constraint5").put("TRANSITION", "OFFLINE-SLAVE");
record.getMapField("constraint5").put("INSTANCE", "localhost_0");
record.getMapField("constraint5").put("RESOURCE", "TestDB");
record.getMapField("constraint5").put("CONSTRAINT_VALUE", "3");
ConstraintItem constraint5 = new ConstraintItem(record.getMapField("constraint5"));
Builder keyBuilder = accessor.keyBuilder();
accessor.setProperty(keyBuilder.constraint(ConstraintType.MESSAGE_CONSTRAINT.toString()), new ClusterConstraints(record));
// ClusterConstraints constraint =
// accessor.getProperty(ClusterConstraints.class,
// PropertyType.CONFIGS,
// ConfigScopeProperty.CONSTRAINT.toString(),
// ConstraintType.MESSAGE_CONSTRAINT.toString());
ClusterConstraints constraint = accessor.getProperty(keyBuilder.constraint(ConstraintType.MESSAGE_CONSTRAINT.toString()));
MessageThrottleStage throttleStage = new MessageThrottleStage();
// test constraintSelection
// message1: hit contraintSelection rule1 and rule2
Message msg1 = createMessage(MessageType.STATE_TRANSITION, "msgId-001", "OFFLINE", "SLAVE", "TestDB", "localhost_0");
Map<ConstraintAttribute, String> msgAttr = ClusterConstraints.toConstraintAttributes(msg1);
Set<ConstraintItem> matches = constraint.match(msgAttr);
System.out.println(msg1 + " matches(" + matches.size() + "): " + matches);
Assert.assertEquals(matches.size(), 5);
Assert.assertTrue(containsConstraint(matches, constraint0));
Assert.assertTrue(containsConstraint(matches, constraint1));
Assert.assertTrue(containsConstraint(matches, constraint2));
Assert.assertTrue(containsConstraint(matches, constraint4));
Assert.assertTrue(containsConstraint(matches, constraint5));
matches = throttleStage.selectConstraints(matches, msgAttr);
System.out.println(msg1 + " matches(" + matches.size() + "): " + matches);
Assert.assertEquals(matches.size(), 2);
Assert.assertTrue(containsConstraint(matches, constraint1));
Assert.assertTrue(containsConstraint(matches, constraint5));
// message2: hit contraintSelection rule1, rule2, and rule3
Message msg2 = createMessage(MessageType.STATE_TRANSITION, "msgId-002", "OFFLINE", "SLAVE", "TestDB", "localhost_1");
msgAttr = ClusterConstraints.toConstraintAttributes(msg2);
matches = constraint.match(msgAttr);
System.out.println(msg2 + " matches(" + matches.size() + "): " + matches);
Assert.assertEquals(matches.size(), 5);
Assert.assertTrue(containsConstraint(matches, constraint0));
Assert.assertTrue(containsConstraint(matches, constraint1));
Assert.assertTrue(containsConstraint(matches, constraint2));
Assert.assertTrue(containsConstraint(matches, constraint3));
Assert.assertTrue(containsConstraint(matches, constraint4));
matches = throttleStage.selectConstraints(matches, msgAttr);
System.out.println(msg2 + " matches(" + matches.size() + "): " + matches);
Assert.assertEquals(matches.size(), 2);
Assert.assertTrue(containsConstraint(matches, constraint1));
Assert.assertTrue(containsConstraint(matches, constraint3));
// test messageThrottleStage
ClusterEvent event = new ClusterEvent(ClusterEventType.Unknown);
ClusterDataCache cache = new ClusterDataCache(clusterName);
event.addAttribute(AttributeName.helixmanager.name(), manager);
event.addAttribute(AttributeName.ClusterDataCache.name(), cache);
Pipeline dataRefresh = new Pipeline();
dataRefresh.addStage(new ReadClusterDataStage());
runPipeline(event, dataRefresh);
runStage(event, new ResourceComputationStage());
MessageSelectionStageOutput msgSelectOutput = new MessageSelectionStageOutput();
Message msg3 = createMessage(MessageType.STATE_TRANSITION, "msgId-003", "OFFLINE", "SLAVE", "TestDB", "localhost_0");
Message msg4 = createMessage(MessageType.STATE_TRANSITION, "msgId-004", "OFFLINE", "SLAVE", "TestDB", "localhost_0");
Message msg5 = createMessage(MessageType.STATE_TRANSITION, "msgId-005", "OFFLINE", "SLAVE", "TestDB", "localhost_0");
Message msg6 = createMessage(MessageType.STATE_TRANSITION, "msgId-006", "OFFLINE", "SLAVE", "TestDB", "localhost_1");
List<Message> selectMessages = new ArrayList<Message>();
selectMessages.add(msg1);
selectMessages.add(msg2);
selectMessages.add(msg3);
selectMessages.add(msg4);
// should be throttled
selectMessages.add(msg5);
// should be throttled
selectMessages.add(msg6);
msgSelectOutput.addMessages("TestDB", new Partition("TestDB_0"), selectMessages);
event.addAttribute(AttributeName.MESSAGES_SELECTED.name(), msgSelectOutput);
runStage(event, throttleStage);
MessageThrottleStageOutput msgThrottleOutput = event.getAttribute(AttributeName.MESSAGES_THROTTLE.name());
List<Message> throttleMessages = msgThrottleOutput.getMessages("TestDB", new Partition("TestDB_0"));
Assert.assertEquals(throttleMessages.size(), 4);
Assert.assertTrue(throttleMessages.contains(msg1));
Assert.assertTrue(throttleMessages.contains(msg2));
Assert.assertTrue(throttleMessages.contains(msg3));
Assert.assertTrue(throttleMessages.contains(msg4));
System.out.println("END " + clusterName + " at " + new Date(System.currentTimeMillis()));
}
use of org.apache.helix.manager.zk.ZkBaseDataAccessor in project helix by apache.
the class TestStatusUpdate method testParticipantStatusUpdates.
// For now write participant StatusUpdates to log4j.
// TODO: Need to investigate another data channel to report to controller and re-enable
// this test
// @Test
public void testParticipantStatusUpdates() throws Exception {
ZkClient zkClient = new ZkClient(ZkIntegrationTestBase.ZK_ADDR);
zkClient.setZkSerializer(new ZNRecordSerializer());
ZKHelixDataAccessor accessor = new ZKHelixDataAccessor(CLUSTER_NAME, new ZkBaseDataAccessor(zkClient));
Builder keyBuilder = accessor.keyBuilder();
List<ExternalView> extViews = accessor.getChildValues(keyBuilder.externalViews());
Assert.assertNotNull(extViews);
for (ExternalView extView : extViews) {
String resourceName = extView.getResourceName();
Set<String> partitionSet = extView.getPartitionSet();
for (String partition : partitionSet) {
Map<String, String> stateMap = extView.getStateMap(partition);
for (String instance : stateMap.keySet()) {
String state = stateMap.get(instance);
StatusUpdateUtil.StatusUpdateContents statusUpdates = StatusUpdateUtil.StatusUpdateContents.getStatusUpdateContents(accessor, instance, resourceName, partition);
Map<String, StatusUpdateUtil.TaskStatus> taskMessages = statusUpdates.getTaskMessages();
List<StatusUpdateUtil.Transition> transitions = statusUpdates.getTransitions();
if (state.equals("MASTER")) {
Assert.assertEquals(transitions.size() >= 2, true, "Invalid number of transitions");
StatusUpdateUtil.Transition lastTransition = transitions.get(transitions.size() - 1);
StatusUpdateUtil.Transition prevTransition = transitions.get(transitions.size() - 2);
Assert.assertEquals(taskMessages.get(lastTransition.getMsgID()), StatusUpdateUtil.TaskStatus.COMPLETED, "Incomplete transition");
Assert.assertEquals(taskMessages.get(prevTransition.getMsgID()), StatusUpdateUtil.TaskStatus.COMPLETED, "Incomplete transition");
Assert.assertEquals(lastTransition.getFromState(), "SLAVE", "Invalid State");
Assert.assertEquals(lastTransition.getToState(), "MASTER", "Invalid State");
} else if (state.equals("SLAVE")) {
Assert.assertEquals(transitions.size() >= 1, true, "Invalid number of transitions");
StatusUpdateUtil.Transition lastTransition = transitions.get(transitions.size() - 1);
Assert.assertEquals(lastTransition.getFromState().equals("MASTER") || lastTransition.getFromState().equals("OFFLINE"), true, "Invalid transition");
Assert.assertEquals(lastTransition.getToState(), "SLAVE", "Invalid State");
}
}
}
}
}
use of org.apache.helix.manager.zk.ZkBaseDataAccessor in project helix by apache.
the class TestDisable method testDisableNodeCustomIS.
@Test
public void testDisableNodeCustomIS() throws Exception {
// Logger.getRootLogger().setLevel(Level.INFO);
String className = TestHelper.getTestClassName();
String methodName = TestHelper.getTestMethodName();
String clusterName = className + "_" + methodName;
final int n = 5;
String disableNode = "localhost_12918";
System.out.println("START " + clusterName + " at " + new Date(System.currentTimeMillis()));
MockParticipantManager[] participants = new MockParticipantManager[n];
// participant port
TestHelper.setupCluster(// participant port
clusterName, // participant port
ZK_ADDR, // participant port
12918, // participant name prefix
"localhost", // resource name prefix
"TestDB", // resources
1, // partitions per resource
8, // number of nodes
n, // replicas
3, "MasterSlave", // do rebalance
true);
// set ideal state to customized mode
ZkBaseDataAccessor<ZNRecord> baseAccessor = new ZkBaseDataAccessor<ZNRecord>(_gZkClient);
ZKHelixDataAccessor accessor = new ZKHelixDataAccessor(clusterName, baseAccessor);
Builder keyBuilder = accessor.keyBuilder();
IdealState idealState = accessor.getProperty(keyBuilder.idealStates("TestDB0"));
idealState.setRebalanceMode(RebalanceMode.CUSTOMIZED);
accessor.setProperty(keyBuilder.idealStates("TestDB0"), idealState);
// start controller
ClusterControllerManager controller = new ClusterControllerManager(ZK_ADDR, clusterName, "controller_0");
controller.syncStart();
// start participants
for (int i = 0; i < n; i++) {
String instanceName = "localhost_" + (12918 + i);
participants[i] = new MockParticipantManager(ZK_ADDR, clusterName, instanceName);
participants[i].syncStart();
}
HelixClusterVerifier _clusterVerifier = new BestPossibleExternalViewVerifier.Builder(clusterName).setZkAddr(ZK_ADDR).build();
Assert.assertTrue(_clusterVerifier.verify());
// disable localhost_12918
String command = "--zkSvr " + ZK_ADDR + " --enableInstance " + clusterName + " " + disableNode + " false";
ClusterSetup.processCommandLineArgs(command.split("\\s+"));
Assert.assertTrue(_clusterVerifier.verify());
// make sure localhost_12918 is in OFFLINE state
Map<String, Map<String, String>> expectStateMap = new HashMap<String, Map<String, String>>();
Map<String, String> expectInstanceStateMap = new HashMap<String, String>();
expectInstanceStateMap.put(disableNode, "OFFLINE");
expectStateMap.put(".*", expectInstanceStateMap);
boolean result = ZkTestHelper.verifyState(_gZkClient, clusterName, "TestDB0", expectStateMap, "==");
Assert.assertTrue(result, disableNode + " should be in OFFLINE");
// re-enable localhost_12918
command = "--zkSvr " + ZK_ADDR + " --enableInstance " + clusterName + " " + disableNode + " true";
ClusterSetup.processCommandLineArgs(command.split("\\s+"));
Assert.assertTrue(_clusterVerifier.verify());
// make sure localhost_12918 is NOT in OFFLINE state
result = ZkTestHelper.verifyState(_gZkClient, clusterName, "TestDB0", expectStateMap, "!=");
Assert.assertTrue(result, disableNode + " should NOT be in OFFLINE");
// clean up
// wait for all zk callbacks done
controller.syncStop();
for (int i = 0; i < 5; i++) {
participants[i].syncStop();
}
System.out.println("END " + clusterName + " at " + new Date(System.currentTimeMillis()));
}
Aggregations