use of org.apache.helix.model.ClusterConstraints.ConstraintAttribute in project helix by apache.
the class ConstraintItemBuilder method addConstraintAttribute.
/**
* add an attribute to constraint-item, overwrite if already exists
* @param attribute
* @param value
*/
public ConstraintItemBuilder addConstraintAttribute(String attribute, String value) {
// make sure constraint attribute is valid
try {
ConstraintAttribute attr = ConstraintAttribute.valueOf(attribute.toUpperCase());
if (attr == ConstraintAttribute.CONSTRAINT_VALUE) {
// make sure constraint-value is valid
try {
ConstraintValue.valueOf(value);
if (_constraintValue == null) {
LOG.info("overwrite existing constraint-value. old-value: " + _constraintValue + ", new-value: " + value);
}
_constraintValue = value;
} catch (IllegalArgumentException e) {
try {
Integer.parseInt(value);
if (_constraintValue == null) {
LOG.info("overwrite existing constraint-value. old-value: " + _constraintValue + ", new-value: " + value);
}
_constraintValue = value;
} catch (NumberFormatException ne) {
LOG.error("fail to add constraint attribute. Invalid constraintValue. " + attribute + ": " + attribute + ", value: " + value);
}
}
} else {
if (_attributes.containsKey(attr)) {
LOG.info("overwrite existing constraint attribute. attribute: " + attribute + ", old-value: " + _attributes.get(attr) + ", new-value: " + value);
}
_attributes.put(attr, value);
}
} catch (IllegalArgumentException e) {
LOG.error("fail to add constraint attribute. Invalid attribute type. attribute: " + attribute + ", value: " + value);
}
return this;
}
use of org.apache.helix.model.ClusterConstraints.ConstraintAttribute in project helix by apache.
the class TestMessageThrottleStage method testMsgThrottleConstraints.
@Test()
public void testMsgThrottleConstraints() throws Exception {
String clusterName = "CLUSTER_" + _className + "_constraints";
System.out.println("START " + clusterName + " at " + new Date(System.currentTimeMillis()));
HelixDataAccessor accessor = new ZKHelixDataAccessor(clusterName, new ZkBaseDataAccessor(_gZkClient));
HelixManager manager = new DummyClusterManager(clusterName, accessor);
// ideal state: node0 is MASTER, node1 is SLAVE
// replica=2 means 1 master and 1 slave
setupIdealState(clusterName, new int[] { 0, 1 }, new String[] { "TestDB" }, 1, 2);
setupLiveInstances(clusterName, new int[] { 0, 1 });
setupStateModel(clusterName);
// setup constraints
ZNRecord record = new ZNRecord(ConstraintType.MESSAGE_CONSTRAINT.toString());
// constraint0:
// "MESSAGE_TYPE=STATE_TRANSITION,CONSTRAINT_VALUE=ANY"
record.setMapField("constraint0", new TreeMap<String, String>());
record.getMapField("constraint0").put("MESSAGE_TYPE", "STATE_TRANSITION");
record.getMapField("constraint0").put("CONSTRAINT_VALUE", "ANY");
ConstraintItem constraint0 = new ConstraintItem(record.getMapField("constraint0"));
// constraint1:
// "MESSAGE_TYPE=STATE_TRANSITION,TRANSITION=OFFLINE-SLAVE,CONSTRAINT_VALUE=ANY"
record.setMapField("constraint1", new TreeMap<String, String>());
record.getMapField("constraint1").put("MESSAGE_TYPE", "STATE_TRANSITION");
record.getMapField("constraint1").put("TRANSITION", "OFFLINE-SLAVE");
record.getMapField("constraint1").put("CONSTRAINT_VALUE", "50");
ConstraintItem constraint1 = new ConstraintItem(record.getMapField("constraint1"));
// constraint2:
// "MESSAGE_TYPE=STATE_TRANSITION,TRANSITION=OFFLINE-SLAVE,INSTANCE=.*,RESOURCE=TestDB,CONSTRAINT_VALUE=2";
record.setMapField("constraint2", new TreeMap<String, String>());
record.getMapField("constraint2").put("MESSAGE_TYPE", "STATE_TRANSITION");
record.getMapField("constraint2").put("TRANSITION", "OFFLINE-SLAVE");
record.getMapField("constraint2").put("INSTANCE", ".*");
record.getMapField("constraint2").put("RESOURCE", "TestDB");
record.getMapField("constraint2").put("CONSTRAINT_VALUE", "2");
ConstraintItem constraint2 = new ConstraintItem(record.getMapField("constraint2"));
// constraint3:
// "MESSAGE_TYPE=STATE_TRANSITION,TRANSITION=OFFLINE-SLAVE,INSTANCE=localhost_12918,RESOURCE=.*,CONSTRAINT_VALUE=1";
record.setMapField("constraint3", new TreeMap<String, String>());
record.getMapField("constraint3").put("MESSAGE_TYPE", "STATE_TRANSITION");
record.getMapField("constraint3").put("TRANSITION", "OFFLINE-SLAVE");
record.getMapField("constraint3").put("INSTANCE", "localhost_1");
record.getMapField("constraint3").put("RESOURCE", ".*");
record.getMapField("constraint3").put("CONSTRAINT_VALUE", "1");
ConstraintItem constraint3 = new ConstraintItem(record.getMapField("constraint3"));
// constraint4:
// "MESSAGE_TYPE=STATE_TRANSITION,TRANSITION=OFFLINE-SLAVE,INSTANCE=.*,RESOURCE=.*,CONSTRAINT_VALUE=10"
record.setMapField("constraint4", new TreeMap<String, String>());
record.getMapField("constraint4").put("MESSAGE_TYPE", "STATE_TRANSITION");
record.getMapField("constraint4").put("TRANSITION", "OFFLINE-SLAVE");
record.getMapField("constraint4").put("INSTANCE", ".*");
record.getMapField("constraint4").put("RESOURCE", ".*");
record.getMapField("constraint4").put("CONSTRAINT_VALUE", "10");
ConstraintItem constraint4 = new ConstraintItem(record.getMapField("constraint4"));
// constraint5:
// "MESSAGE_TYPE=STATE_TRANSITION,TRANSITION=OFFLINE-SLAVE,INSTANCE=localhost_12918,RESOURCE=TestDB,CONSTRAINT_VALUE=5"
record.setMapField("constraint5", new TreeMap<String, String>());
record.getMapField("constraint5").put("MESSAGE_TYPE", "STATE_TRANSITION");
record.getMapField("constraint5").put("TRANSITION", "OFFLINE-SLAVE");
record.getMapField("constraint5").put("INSTANCE", "localhost_0");
record.getMapField("constraint5").put("RESOURCE", "TestDB");
record.getMapField("constraint5").put("CONSTRAINT_VALUE", "3");
ConstraintItem constraint5 = new ConstraintItem(record.getMapField("constraint5"));
Builder keyBuilder = accessor.keyBuilder();
accessor.setProperty(keyBuilder.constraint(ConstraintType.MESSAGE_CONSTRAINT.toString()), new ClusterConstraints(record));
// ClusterConstraints constraint =
// accessor.getProperty(ClusterConstraints.class,
// PropertyType.CONFIGS,
// ConfigScopeProperty.CONSTRAINT.toString(),
// ConstraintType.MESSAGE_CONSTRAINT.toString());
ClusterConstraints constraint = accessor.getProperty(keyBuilder.constraint(ConstraintType.MESSAGE_CONSTRAINT.toString()));
MessageThrottleStage throttleStage = new MessageThrottleStage();
// test constraintSelection
// message1: hit contraintSelection rule1 and rule2
Message msg1 = createMessage(MessageType.STATE_TRANSITION, "msgId-001", "OFFLINE", "SLAVE", "TestDB", "localhost_0");
Map<ConstraintAttribute, String> msgAttr = ClusterConstraints.toConstraintAttributes(msg1);
Set<ConstraintItem> matches = constraint.match(msgAttr);
System.out.println(msg1 + " matches(" + matches.size() + "): " + matches);
Assert.assertEquals(matches.size(), 5);
Assert.assertTrue(containsConstraint(matches, constraint0));
Assert.assertTrue(containsConstraint(matches, constraint1));
Assert.assertTrue(containsConstraint(matches, constraint2));
Assert.assertTrue(containsConstraint(matches, constraint4));
Assert.assertTrue(containsConstraint(matches, constraint5));
matches = throttleStage.selectConstraints(matches, msgAttr);
System.out.println(msg1 + " matches(" + matches.size() + "): " + matches);
Assert.assertEquals(matches.size(), 2);
Assert.assertTrue(containsConstraint(matches, constraint1));
Assert.assertTrue(containsConstraint(matches, constraint5));
// message2: hit contraintSelection rule1, rule2, and rule3
Message msg2 = createMessage(MessageType.STATE_TRANSITION, "msgId-002", "OFFLINE", "SLAVE", "TestDB", "localhost_1");
msgAttr = ClusterConstraints.toConstraintAttributes(msg2);
matches = constraint.match(msgAttr);
System.out.println(msg2 + " matches(" + matches.size() + "): " + matches);
Assert.assertEquals(matches.size(), 5);
Assert.assertTrue(containsConstraint(matches, constraint0));
Assert.assertTrue(containsConstraint(matches, constraint1));
Assert.assertTrue(containsConstraint(matches, constraint2));
Assert.assertTrue(containsConstraint(matches, constraint3));
Assert.assertTrue(containsConstraint(matches, constraint4));
matches = throttleStage.selectConstraints(matches, msgAttr);
System.out.println(msg2 + " matches(" + matches.size() + "): " + matches);
Assert.assertEquals(matches.size(), 2);
Assert.assertTrue(containsConstraint(matches, constraint1));
Assert.assertTrue(containsConstraint(matches, constraint3));
// test messageThrottleStage
ClusterEvent event = new ClusterEvent(ClusterEventType.Unknown);
ClusterDataCache cache = new ClusterDataCache(clusterName);
event.addAttribute(AttributeName.helixmanager.name(), manager);
event.addAttribute(AttributeName.ClusterDataCache.name(), cache);
Pipeline dataRefresh = new Pipeline();
dataRefresh.addStage(new ReadClusterDataStage());
runPipeline(event, dataRefresh);
runStage(event, new ResourceComputationStage());
MessageSelectionStageOutput msgSelectOutput = new MessageSelectionStageOutput();
Message msg3 = createMessage(MessageType.STATE_TRANSITION, "msgId-003", "OFFLINE", "SLAVE", "TestDB", "localhost_0");
Message msg4 = createMessage(MessageType.STATE_TRANSITION, "msgId-004", "OFFLINE", "SLAVE", "TestDB", "localhost_0");
Message msg5 = createMessage(MessageType.STATE_TRANSITION, "msgId-005", "OFFLINE", "SLAVE", "TestDB", "localhost_0");
Message msg6 = createMessage(MessageType.STATE_TRANSITION, "msgId-006", "OFFLINE", "SLAVE", "TestDB", "localhost_1");
List<Message> selectMessages = new ArrayList<Message>();
selectMessages.add(msg1);
selectMessages.add(msg2);
selectMessages.add(msg3);
selectMessages.add(msg4);
// should be throttled
selectMessages.add(msg5);
// should be throttled
selectMessages.add(msg6);
msgSelectOutput.addMessages("TestDB", new Partition("TestDB_0"), selectMessages);
event.addAttribute(AttributeName.MESSAGES_SELECTED.name(), msgSelectOutput);
runStage(event, throttleStage);
MessageThrottleStageOutput msgThrottleOutput = event.getAttribute(AttributeName.MESSAGES_THROTTLE.name());
List<Message> throttleMessages = msgThrottleOutput.getMessages("TestDB", new Partition("TestDB_0"));
Assert.assertEquals(throttleMessages.size(), 4);
Assert.assertTrue(throttleMessages.contains(msg1));
Assert.assertTrue(throttleMessages.contains(msg2));
Assert.assertTrue(throttleMessages.contains(msg3));
Assert.assertTrue(throttleMessages.contains(msg4));
System.out.println("END " + clusterName + " at " + new Date(System.currentTimeMillis()));
}
use of org.apache.helix.model.ClusterConstraints.ConstraintAttribute in project helix by apache.
the class MessageThrottleStage method throttle.
private List<Message> throttle(Map<String, Integer> throttleMap, ClusterConstraints constraint, List<Message> messages, final boolean needThrottle) {
List<Message> throttleOutputMsgs = new ArrayList<Message>();
for (Message message : messages) {
Map<ConstraintAttribute, String> msgAttr = ClusterConstraints.toConstraintAttributes(message);
Set<ConstraintItem> matches = constraint.match(msgAttr);
matches = selectConstraints(matches, msgAttr);
boolean msgThrottled = false;
for (ConstraintItem item : matches) {
String key = item.filter(msgAttr).toString();
if (!throttleMap.containsKey(key)) {
throttleMap.put(key, valueOf(item.getConstraintValue()));
}
int value = throttleMap.get(key);
throttleMap.put(key, --value);
if (needThrottle && value < 0) {
msgThrottled = true;
if (LOG.isDebugEnabled()) {
// TODO: printout constraint item that throttles the message
LOG.debug("message: " + message + " is throttled by constraint: " + item);
}
}
}
if (!msgThrottled) {
throttleOutputMsgs.add(message);
}
}
return throttleOutputMsgs;
}
use of org.apache.helix.model.ClusterConstraints.ConstraintAttribute in project helix by apache.
the class TestConstraint method testMsgConstraint.
@Test
public void testMsgConstraint() {
String className = getShortClassName();
System.out.println("START testMsgConstraint() at " + new Date(System.currentTimeMillis()));
String clusterName = "CLUSTER_" + className + "_msg";
TestHelper.setupEmptyCluster(_gZkClient, clusterName);
ZNRecord record = new ZNRecord("testMsgConstraint");
// constraint0:
// "MESSAGE_TYPE=STATE_TRANSITION,CONSTRAINT_VALUE=ANY"
record.setMapField("constraint0", new TreeMap<String, String>());
record.getMapField("constraint0").put("MESSAGE_TYPE", "STATE_TRANSITION");
record.getMapField("constraint0").put("CONSTRAINT_VALUE", "ANY");
ConstraintItem constraint0 = new ConstraintItem(record.getMapField("constraint0"));
// constraint1:
// "MESSAGE_TYPE=STATE_TRANSITION,TRANSITION=OFFLINE-SLAVE,CONSTRAINT_VALUE=ANY"
record.setMapField("constraint1", new TreeMap<String, String>());
record.getMapField("constraint1").put("MESSAGE_TYPE", "STATE_TRANSITION");
record.getMapField("constraint1").put("TRANSITION", "OFFLINE-SLAVE");
record.getMapField("constraint1").put("CONSTRAINT_VALUE", "50");
ConstraintItem constraint1 = new ConstraintItem(record.getMapField("constraint1"));
// constraint2:
// "MESSAGE_TYPE=STATE_TRANSITION,TRANSITION=OFFLINE-SLAVE,INSTANCE=.*,RESOURCE=TestDB,CONSTRAINT_VALUE=2";
record.setMapField("constraint2", new TreeMap<String, String>());
record.getMapField("constraint2").put("MESSAGE_TYPE", "STATE_TRANSITION");
record.getMapField("constraint2").put("TRANSITION", "OFFLINE-SLAVE");
record.getMapField("constraint2").put("INSTANCE", ".*");
record.getMapField("constraint2").put("RESOURCE", "TestDB");
record.getMapField("constraint2").put("CONSTRAINT_VALUE", "2");
ConstraintItem constraint2 = new ConstraintItem(record.getMapField("constraint2"));
// constraint3:
// "MESSAGE_TYPE=STATE_TRANSITION,TRANSITION=OFFLINE-SLAVE,INSTANCE=localhost_12918,RESOURCE=.*,CONSTRAINT_VALUE=1";
record.setMapField("constraint3", new TreeMap<String, String>());
record.getMapField("constraint3").put("MESSAGE_TYPE", "STATE_TRANSITION");
record.getMapField("constraint3").put("TRANSITION", "OFFLINE-SLAVE");
record.getMapField("constraint3").put("INSTANCE", "localhost_12919");
record.getMapField("constraint3").put("RESOURCE", ".*");
record.getMapField("constraint3").put("CONSTRAINT_VALUE", "1");
ConstraintItem constraint3 = new ConstraintItem(record.getMapField("constraint3"));
// constraint4:
// "MESSAGE_TYPE=STATE_TRANSITION,TRANSITION=OFFLINE-SLAVE,INSTANCE=.*,RESOURCE=.*,CONSTRAINT_VALUE=10"
record.setMapField("constraint4", new TreeMap<String, String>());
record.getMapField("constraint4").put("MESSAGE_TYPE", "STATE_TRANSITION");
record.getMapField("constraint4").put("TRANSITION", "OFFLINE-SLAVE");
record.getMapField("constraint4").put("INSTANCE", ".*");
record.getMapField("constraint4").put("RESOURCE", ".*");
record.getMapField("constraint4").put("CONSTRAINT_VALUE", "10");
ConstraintItem constraint4 = new ConstraintItem(record.getMapField("constraint4"));
// constraint5:
// "MESSAGE_TYPE=STATE_TRANSITION,TRANSITION=OFFLINE-SLAVE,INSTANCE=localhost_12918,RESOURCE=TestDB,CONSTRAINT_VALUE=5"
record.setMapField("constraint5", new TreeMap<String, String>());
record.getMapField("constraint5").put("MESSAGE_TYPE", "STATE_TRANSITION");
record.getMapField("constraint5").put("TRANSITION", "OFFLINE-SLAVE");
record.getMapField("constraint5").put("INSTANCE", "localhost_12918");
record.getMapField("constraint5").put("RESOURCE", "TestDB");
record.getMapField("constraint5").put("CONSTRAINT_VALUE", "5");
ConstraintItem constraint5 = new ConstraintItem(record.getMapField("constraint5"));
ZKHelixDataAccessor accessor = new ZKHelixDataAccessor(clusterName, new ZkBaseDataAccessor(_gZkClient));
Builder keyBuilder = accessor.keyBuilder();
accessor.setProperty(keyBuilder.constraint(ConstraintType.MESSAGE_CONSTRAINT.toString()), new ClusterConstraints(record));
record = accessor.getProperty(keyBuilder.constraint(ConstraintType.MESSAGE_CONSTRAINT.toString())).getRecord();
ClusterConstraints constraint = new ClusterConstraints(record);
// System.out.println("constraint: " + constraint);
// message1
Message msg1 = createMessage(MessageType.STATE_TRANSITION, "msgId-001", "OFFLINE", "SLAVE", "TestDB", "localhost_12918");
Map<ConstraintAttribute, String> msgAttr = ClusterConstraints.toConstraintAttributes(msg1);
Set<ConstraintItem> matches = constraint.match(msgAttr);
System.out.println(msg1 + " matches(" + matches.size() + "): " + matches);
Assert.assertEquals(matches.size(), 5);
Assert.assertTrue(contains(matches, constraint0));
Assert.assertTrue(contains(matches, constraint1));
Assert.assertTrue(contains(matches, constraint2));
Assert.assertTrue(contains(matches, constraint4));
Assert.assertTrue(contains(matches, constraint5));
// message2
Message msg2 = createMessage(MessageType.STATE_TRANSITION, "msgId-002", "OFFLINE", "SLAVE", "TestDB", "localhost_12919");
msgAttr = ClusterConstraints.toConstraintAttributes(msg2);
matches = constraint.match(msgAttr);
System.out.println(msg2 + " matches(" + matches.size() + "): " + matches);
Assert.assertEquals(matches.size(), 5);
Assert.assertTrue(contains(matches, constraint0));
Assert.assertTrue(contains(matches, constraint1));
Assert.assertTrue(contains(matches, constraint2));
Assert.assertTrue(contains(matches, constraint3));
Assert.assertTrue(contains(matches, constraint4));
System.out.println("END testMsgConstraint() at " + new Date(System.currentTimeMillis()));
}
use of org.apache.helix.model.ClusterConstraints.ConstraintAttribute in project helix by apache.
the class TestConstraint method testStateConstraint.
@Test
public void testStateConstraint() {
String className = getShortClassName();
System.out.println("START testStateConstraint() at " + new Date(System.currentTimeMillis()));
String clusterName = "CLUSTER_" + className + "_state";
TestHelper.setupEmptyCluster(_gZkClient, clusterName);
ZNRecord record = new ZNRecord("testStateConstraint");
// constraint0:
// "STATE=MASTER,CONSTRAINT_VALUE=1"
record.setMapField("constraint0", new TreeMap<String, String>());
record.getMapField("constraint0").put("STATE", "MASTER");
record.getMapField("constraint0").put("CONSTRAINT_VALUE", "1");
ConstraintItem constraint0 = new ConstraintItem(record.getMapField("constraint0"));
// constraint1:
// "STATE=MASTER,RESOURCE=TestDB,CONSTRAINT_VALUE=5"
record.setMapField("constraint1", new TreeMap<String, String>());
record.getMapField("constraint1").put("STATE", "MASTER");
record.getMapField("constraint1").put("RESOURCE", "TestDB");
record.getMapField("constraint1").put("CONSTRAINT_VALUE", "1");
ConstraintItem constraint1 = new ConstraintItem(record.getMapField("constraint1"));
// constraint2:
// "STATE=MASTER,RESOURCE=.*,CONSTRAINT_VALUE=2"
record.setMapField("constraint2", new TreeMap<String, String>());
record.getMapField("constraint2").put("STATE", "MASTER");
record.getMapField("constraint2").put("RESOURCE", ".*");
record.getMapField("constraint2").put("CONSTRAINT_VALUE", "2");
ConstraintItem constraint2 = new ConstraintItem(record.getMapField("constraint2"));
ZKHelixDataAccessor accessor = new ZKHelixDataAccessor(clusterName, new ZkBaseDataAccessor(_gZkClient));
Builder keyBuilder = accessor.keyBuilder();
accessor.setProperty(keyBuilder.constraint(ConstraintType.STATE_CONSTRAINT.toString()), new ClusterConstraints(record));
record = accessor.getProperty(keyBuilder.constraint(ConstraintType.STATE_CONSTRAINT.toString())).getRecord();
ClusterConstraints constraint = new ClusterConstraints(record);
// System.out.println("constraint: " + constraint);
// state1: hit rule2
Map<ConstraintAttribute, String> stateAttr1 = new HashMap<ConstraintAttribute, String>();
stateAttr1.put(ConstraintAttribute.STATE, "MASTER");
stateAttr1.put(ConstraintAttribute.RESOURCE, "TestDB");
Set<ConstraintItem> matches = constraint.match(stateAttr1);
System.out.println(stateAttr1 + " matches(" + matches.size() + "): " + matches);
Assert.assertEquals(matches.size(), 3);
Assert.assertTrue(contains(matches, constraint0));
Assert.assertTrue(contains(matches, constraint1));
Assert.assertTrue(contains(matches, constraint2));
// matches = selectConstraints(matches, stateAttr1);
// System.out.println(stateAttr1 + " matches(" + matches.size() + "): " + matches);
// Assert.assertEquals(matches.size(), 2);
// Assert.assertTrue(contains(matches, constraint0));
// Assert.assertTrue(contains(matches, constraint1));
// state2: not hit any rules
Map<ConstraintAttribute, String> stateAttr2 = new HashMap<ConstraintAttribute, String>();
stateAttr2.put(ConstraintAttribute.STATE, "MASTER");
stateAttr2.put(ConstraintAttribute.RESOURCE, "MyDB");
matches = constraint.match(stateAttr2);
System.out.println(stateAttr2 + " matches(" + matches.size() + "): " + matches);
Assert.assertEquals(matches.size(), 2);
Assert.assertTrue(contains(matches, constraint0));
Assert.assertTrue(contains(matches, constraint2));
// matches = selectConstraints(matches, stateAttr2);
// System.out.println(stateAttr2 + " matches(" + matches.size() + "): " + matches);
// Assert.assertEquals(matches.size(), 2);
// Assert.assertTrue(contains(matches, constraint0));
// Assert.assertTrue(contains(matches, constraint2));
System.out.println("END testStateConstraint() at " + new Date(System.currentTimeMillis()));
}
Aggregations