use of com.datatorrent.api.AffinityRule in project apex-core by apache.
the class AffinityRulesTest method testOperatorPartitionsAntiAffinity.
@Test
public void testOperatorPartitionsAntiAffinity() {
LogicalPlan dag = new LogicalPlan();
TestGeneratorInputOperator o1 = dag.addOperator("O1", new TestGeneratorInputOperator());
GenericTestOperator o2 = dag.addOperator("O2", new GenericTestOperator());
GenericTestOperator o3 = dag.addOperator("O3", new GenericTestOperator());
dag.addStream("stream1", o1.outport, o2.inport1);
dag.addStream("stream2", o2.outport1, o3.inport1);
dag.setOperatorAttribute(o2, OperatorContext.PARTITIONER, new StatelessPartitioner<GenericTestOperator>(5));
AffinityRulesSet ruleSet = new AffinityRulesSet();
// Valid case:
List<AffinityRule> rules = new ArrayList<>();
ruleSet.setAffinityRules(rules);
AffinityRule rule1 = new AffinityRule(Type.ANTI_AFFINITY, Locality.NODE_LOCAL, false, "O2", "O2");
rules.add(rule1);
dag.setAttribute(DAGContext.AFFINITY_RULES_SET, ruleSet);
dag.validate();
dag.getAttributes().put(com.datatorrent.api.Context.DAGContext.APPLICATION_PATH, testMeta.getAbsolutePath());
dag.setAttribute(OperatorContext.STORAGE_AGENT, new MemoryStorageAgent());
StreamingContainerManager scm = new StreamingContainerManager(dag);
for (ContainerStartRequest csr : scm.containerStartRequests) {
PTContainer container = csr.container;
if (container.getOperators().get(0).getName().equals("O2")) {
Assert.assertEquals("Anti-affinity containers set should have 4 containers for other partitions ", 4, container.getStrictAntiPrefs().size());
for (PTContainer c : container.getStrictAntiPrefs()) {
for (PTOperator operator : c.getOperators()) {
Assert.assertEquals("Partion for O2 should be Anti Prefs", "O2", operator.getName());
}
}
}
}
// Check resource handler assigns different hosts for each partition
ResourceRequestHandler rr = new ResourceRequestHandler();
int containerMem = 1000;
Map<String, NodeReport> nodeReports = Maps.newHashMap();
for (int i = 0; i < 10; i++) {
String hostName = "host" + i;
NodeReport nr = BuilderUtils.newNodeReport(BuilderUtils.newNodeId(hostName, 0), NodeState.RUNNING, "httpAddress", "rackName", BuilderUtils.newResource(0, 0), BuilderUtils.newResource(containerMem * 2, 2), 0, null, 0);
nodeReports.put(nr.getNodeId().getHost(), nr);
}
// set resources
rr.updateNodeReports(Lists.newArrayList(nodeReports.values()));
Set<String> partitionHostNames = new HashSet<>();
for (ContainerStartRequest csr : scm.containerStartRequests) {
String host = rr.getHost(csr, true);
csr.container.host = host;
if (csr.container.getOperators().get(0).getName().equals("O2")) {
Assert.assertNotNull("Host name should not be null", host);
LOG.info("Partition {} for operator O2 has host = {} ", csr.container.getId(), host);
Assert.assertTrue("Each Partition should have a different host", !partitionHostNames.contains(host));
partitionHostNames.add(host);
}
}
}
use of com.datatorrent.api.AffinityRule in project apex-core by apache.
the class PhysicalPlan method setAntiAffinityForContainers.
public void setAntiAffinityForContainers(LogicalPlan dag, Collection<AffinityRule> affinityRules, Map<PTOperator, PTContainer> operatorContainerMap) {
for (AffinityRule rule : affinityRules) {
if (rule.getOperatorsList() != null && rule.getType() == Type.ANTI_AFFINITY) {
for (int i = 0; i < rule.getOperatorsList().size() - 1; i++) {
for (int j = i + 1; j < rule.getOperatorsList().size(); j++) {
OperatorPair operators = new OperatorPair(rule.getOperatorsList().get(i), rule.getOperatorsList().get(j));
PMapping firstPMapping = logicalToPTOperator.get(dag.getOperatorMeta(operators.first));
OperatorMeta opMeta = dag.getOperatorMeta(operators.second);
PMapping secondMapping = logicalToPTOperator.get(opMeta);
for (PTOperator firstPtOperator : firstPMapping.partitions) {
PTContainer firstContainer = operatorContainerMap.get(firstPtOperator);
for (PTOperator secondPtOperator : secondMapping.partitions) {
PTContainer secondContainer = operatorContainerMap.get(secondPtOperator);
if (firstContainer == secondContainer || firstContainer.getStrictAntiPrefs().contains(secondContainer)) {
continue;
}
if (rule.isRelaxLocality()) {
firstContainer.getPreferredAntiPrefs().add(secondContainer);
secondContainer.getPreferredAntiPrefs().add(firstContainer);
} else {
firstContainer.getStrictAntiPrefs().add(secondContainer);
secondContainer.getStrictAntiPrefs().add(firstContainer);
}
}
}
}
}
}
}
}
use of com.datatorrent.api.AffinityRule in project apex-core by apache.
the class LogicalPlanTest method testAffinityRulesDagValidation.
@Test
public void testAffinityRulesDagValidation() {
TestGeneratorInputOperator o1 = dag.addOperator("O1", new TestGeneratorInputOperator());
GenericTestOperator o2 = dag.addOperator("O2", new GenericTestOperator());
GenericTestOperator o3 = dag.addOperator("O3", new GenericTestOperator());
dag.addStream("stream1", o1.outport, o2.inport1).setLocality(Locality.THREAD_LOCAL);
StreamMeta stream2 = dag.addStream("stream2", o2.outport1, o3.inport1).setLocality(Locality.CONTAINER_LOCAL);
AffinityRulesSet ruleSet = new AffinityRulesSet();
// Valid case:
List<AffinityRule> rules = new ArrayList<>();
ruleSet.setAffinityRules(rules);
AffinityRule rule1 = new AffinityRule(Type.AFFINITY, Locality.CONTAINER_LOCAL, false, "O1", "O3");
rules.add(rule1);
dag.setAttribute(DAGContext.AFFINITY_RULES_SET, ruleSet);
dag.validate();
// Locality conflicts with affinity rules case:
AffinityRule rule2 = new AffinityRule(Type.ANTI_AFFINITY, Locality.NODE_LOCAL, false, "O2", "O3");
rules.add(rule2);
try {
dag.validate();
Assert.fail("DAG validation should fail due to conflicting rules");
} catch (ValidationException e) {
Assert.assertEquals("Anti Affinity rule for operators O2 & O3 conflicts with affinity rules or Stream locality", e.getMessage());
}
// Change Stream2 locality to Node to check if validation passes
stream2.setLocality(Locality.RACK_LOCAL);
dag.validate();
// Add anti-affinity rule conflicting with rule1
AffinityRule rule3 = new AffinityRule(Type.ANTI_AFFINITY, Locality.NODE_LOCAL, false, "O1", "O3");
rules.add(rule3);
try {
dag.validate();
Assert.fail("DAG validation should fail due to conflicting rules");
} catch (ValidationException e) {
Assert.assertEquals("Anti Affinity rule for operators O1 & O3 conflicts with affinity rules or Stream locality", e.getMessage());
}
// Change rule1 to Rack local to see if dag validation passes
rules.clear();
rule1.setLocality(Locality.RACK_LOCAL);
rules.add(rule1);
rules.add(rule2);
rules.add(rule3);
dag.validate();
// Add conflicting rules and set relaxLocality for one rule
AffinityRule rule4 = new AffinityRule(Type.ANTI_AFFINITY, Locality.NODE_LOCAL, true, "O1", "O2");
rules.add(rule4);
dag.validate();
// Set conflicting host locality and check if it fails validation
rules.clear();
AffinityRule rule = new AffinityRule(Type.ANTI_AFFINITY, Locality.NODE_LOCAL, false, "O2", "O3");
rules.add(rule);
dag.getMeta(o2).getAttributes().put(OperatorContext.LOCALITY_HOST, "host1");
dag.getMeta(o3).getAttributes().put(OperatorContext.LOCALITY_HOST, "host1");
try {
dag.validate();
Assert.fail("DAG validation should fail due to conflicting host locality");
} catch (ValidationException e) {
Assert.assertEquals("Host Locality for operators: O2(host: host1) & O3(host: host1) conflict with anti-affinity rules", e.getMessage());
}
// Set conflicting affinity and different host locality for node-local
// operators
rules.clear();
rule = new AffinityRule(Type.AFFINITY, Locality.NODE_LOCAL, false, "O2", "O3");
rules.add(rule);
dag.getMeta(o2).getAttributes().put(OperatorContext.LOCALITY_HOST, "host1");
dag.getMeta(o3).getAttributes().put(OperatorContext.LOCALITY_HOST, "host2");
try {
dag.validate();
Assert.fail("DAG validation should fail due to conflicting host locality");
} catch (ValidationException e) {
Assert.assertEquals("Host Locality for operators: O2(host: host1) & O3(host: host2) conflicts with affinity rules", e.getMessage());
}
// Check affinity Thread local validation for non-connected operators
dag.getAttributes().get(DAGContext.AFFINITY_RULES_SET).getAffinityRules().clear();
rule = new AffinityRule(Type.AFFINITY, Locality.THREAD_LOCAL, false, "O1", "O3");
rules.add(rule);
try {
dag.validate();
Assert.fail("DAG validation should fail due to conflicting host locality");
} catch (ValidationException e) {
Assert.assertEquals("Affinity rule specified THREAD_LOCAL affinity for operators O1 & O3 which are not connected by stream", e.getMessage());
}
// Check indirect conflict
dag = new LogicalPlan();
o1 = dag.addOperator("O1", new TestGeneratorInputOperator());
o2 = dag.addOperator("O2", new GenericTestOperator());
o3 = dag.addOperator("O3", new GenericTestOperator());
GenericTestOperator o4 = dag.addOperator("O4", new GenericTestOperator());
GenericTestOperator o5 = dag.addOperator("O5", new GenericTestOperator());
dag.addStream("stream1", o1.outport, o2.inport1, o3.inport1).setLocality(Locality.NODE_LOCAL);
dag.addStream("stream2", o3.outport1, o4.inport1);
dag.addStream("stream3", o2.outport1, o5.inport1);
rules.clear();
// O3 and O5 cannot have NODE_LOCAL anti-affinity now, since they already have NODE_LOCAL affinity
rules.add(new AffinityRule(Type.AFFINITY, Locality.CONTAINER_LOCAL, false, "O1", "O5"));
rules.add(new AffinityRule(Type.ANTI_AFFINITY, Locality.NODE_LOCAL, false, "O3", "O5"));
ruleSet = new AffinityRulesSet();
ruleSet.setAffinityRules(rules);
dag.setAttribute(DAGContext.AFFINITY_RULES_SET, ruleSet);
try {
dag.validate();
Assert.fail("dag validation should fail due to conflicting affinity rules");
} catch (ValidationException e) {
Assert.assertEquals("Anti Affinity rule for operators O3 & O5 conflicts with affinity rules or Stream locality", e.getMessage());
}
}
use of com.datatorrent.api.AffinityRule in project apex-core by apache.
the class LogicalPlanModificationTest method testAddOperatorWithAffinityRules.
@Test
public void testAddOperatorWithAffinityRules() {
GenericTestOperator o1 = dag.addOperator("o1", GenericTestOperator.class);
GenericTestOperator o2 = dag.addOperator("o2", GenericTestOperator.class);
GenericTestOperator o3 = dag.addOperator("o3", GenericTestOperator.class);
dag.addStream("o1.outport1", o1.outport1, o2.inport1);
dag.addStream("o2.outport1", o2.outport1, o3.inport1);
TestPlanContext ctx = new TestPlanContext();
dag.setAttribute(OperatorContext.STORAGE_AGENT, ctx);
PhysicalPlan plan = new PhysicalPlan(dag, ctx);
ctx.deploy.clear();
ctx.undeploy.clear();
Assert.assertEquals("containers", 3, plan.getContainers().size());
AffinityRulesSet ruleSet = new AffinityRulesSet();
List<AffinityRule> rules = new ArrayList<>();
ruleSet.setAffinityRules(rules);
rules.add(new AffinityRule(Type.AFFINITY, Locality.CONTAINER_LOCAL, false, "o1", "added1"));
rules.add(new AffinityRule(Type.ANTI_AFFINITY, Locality.NODE_LOCAL, false, "o3", "added1"));
dag.setAttribute(DAGContext.AFFINITY_RULES_SET, ruleSet);
PlanModifier pm = new PlanModifier(plan);
GenericTestOperator added1 = new GenericTestOperator();
pm.addOperator("added1", added1);
pm.addStream("added1.outport1", added1.outport1, o3.inport2);
Assert.assertEquals("undeploy " + ctx.undeploy, 0, ctx.undeploy.size());
Assert.assertEquals("deploy " + ctx.deploy, 0, ctx.deploy.size());
pm.applyChanges(ctx);
Assert.assertEquals("containers post change", 4, plan.getContainers().size());
Assert.assertEquals("undeploy " + ctx.undeploy, 1, ctx.undeploy.size());
Assert.assertEquals("deploy " + ctx.deploy, 2, ctx.deploy.size());
// Validate affinity rules are applied
for (PTContainer c : plan.getContainers()) {
if (c.getOperators().contains("added1")) {
Assert.assertEquals("Operators O1 and added1 should be in the same container as per affinity rule", 2, c.getOperators().size());
Assert.assertEquals("Operators O1 and added1 should be in the same container as per affinity rule", "o1", c.getOperators().get(0).getOperatorMeta().getName());
Assert.assertEquals("Operators O1 and added1 should be in the same container as per affinity rule", "added1", c.getOperators().get(1).getOperatorMeta().getName());
Set<PTContainer> antiAffinityList = c.getStrictAntiPrefs();
Assert.assertEquals("There should be one container in antiaffinity list", 1, antiAffinityList.size());
List<PTOperator> antiAffinityOperators = antiAffinityList.iterator().next().getOperators();
Assert.assertEquals("AntiAffinity operators should containn operator O3", antiAffinityOperators.iterator().next().getOperatorMeta().getName(), "o3");
}
}
}
use of com.datatorrent.api.AffinityRule in project apex-core by apache.
the class AffinityRulesTest method testAntiAffinityInOperators.
@Test
public void testAntiAffinityInOperators() {
LogicalPlan dag = new LogicalPlan();
dag.getAttributes().put(com.datatorrent.api.Context.DAGContext.APPLICATION_PATH, testMeta.getAbsolutePath());
dag.setAttribute(OperatorContext.STORAGE_AGENT, new MemoryStorageAgent());
GenericTestOperator o1 = dag.addOperator("O1", GenericTestOperator.class);
dag.setOperatorAttribute(o1, OperatorContext.MEMORY_MB, 256);
GenericTestOperator o2 = dag.addOperator("O2", GenericTestOperator.class);
dag.setOperatorAttribute(o2, OperatorContext.MEMORY_MB, 256);
dag.getMeta(o1).getAttributes().put(OperatorContext.LOCALITY_HOST, "host1");
AffinityRulesSet ruleSet = new AffinityRulesSet();
List<AffinityRule> rules = new ArrayList<>();
ruleSet.setAffinityRules(rules);
AffinityRule rule1 = new AffinityRule(Type.ANTI_AFFINITY, Locality.NODE_LOCAL, false, "O1", "O2");
rules.add(rule1);
dag.setAttribute(DAGContext.AFFINITY_RULES_SET, ruleSet);
// .setLocality(Locality.NODE_LOCAL);
dag.addStream("o1_outport1", o1.outport1, o2.inport1);
StreamingContainerManager scm = new StreamingContainerManager(dag);
ResourceRequestHandler rr = new ResourceRequestHandler();
int containerMem = 1000;
Map<String, NodeReport> nodeReports = Maps.newHashMap();
NodeReport nr = BuilderUtils.newNodeReport(BuilderUtils.newNodeId("host1", 0), NodeState.RUNNING, "httpAddress", "rackName", BuilderUtils.newResource(0, 0), BuilderUtils.newResource(containerMem * 2, 2), 0, null, 0);
nodeReports.put(nr.getNodeId().getHost(), nr);
nr = BuilderUtils.newNodeReport(BuilderUtils.newNodeId("host2", 0), NodeState.RUNNING, "httpAddress", "rackName", BuilderUtils.newResource(0, 0), BuilderUtils.newResource(containerMem * 2, 2), 0, null, 0);
nodeReports.put(nr.getNodeId().getHost(), nr);
// set resources
rr.updateNodeReports(Lists.newArrayList(nodeReports.values()));
for (ContainerStartRequest csr : scm.containerStartRequests) {
String host = rr.getHost(csr, true);
csr.container.host = host;
if (csr.container.getOperators().get(0).getName().equals("O1")) {
Assert.assertEquals("Hosts set to host1 for Operator O1", "host1", host);
}
if (csr.container.getOperators().get(0).getName().equals("O2")) {
Assert.assertEquals("Hosts set to host2 for Operator O2", "host2", host);
}
}
}
Aggregations