Examples with AffinityRulesSet com.datatorrent.api.AffinityRulesSet used on opensource projects

Example 1 with AffinityRulesSet

use of com.datatorrent.api.AffinityRulesSet in project apex-core by apache.

the class AffinityRulesTest method testOperatorPartitionsAntiAffinity.

@Test
public void testOperatorPartitionsAntiAffinity() {
    LogicalPlan dag = new LogicalPlan();
    TestGeneratorInputOperator o1 = dag.addOperator("O1", new TestGeneratorInputOperator());
    GenericTestOperator o2 = dag.addOperator("O2", new GenericTestOperator());
    GenericTestOperator o3 = dag.addOperator("O3", new GenericTestOperator());
    dag.addStream("stream1", o1.outport, o2.inport1);
    dag.addStream("stream2", o2.outport1, o3.inport1);
    dag.setOperatorAttribute(o2, OperatorContext.PARTITIONER, new StatelessPartitioner<GenericTestOperator>(5));
    AffinityRulesSet ruleSet = new AffinityRulesSet();
    // Valid case:
    List<AffinityRule> rules = new ArrayList<>();
    ruleSet.setAffinityRules(rules);
    AffinityRule rule1 = new AffinityRule(Type.ANTI_AFFINITY, Locality.NODE_LOCAL, false, "O2", "O2");
    rules.add(rule1);
    dag.setAttribute(DAGContext.AFFINITY_RULES_SET, ruleSet);
    dag.validate();
    dag.getAttributes().put(com.datatorrent.api.Context.DAGContext.APPLICATION_PATH, testMeta.getAbsolutePath());
    dag.setAttribute(OperatorContext.STORAGE_AGENT, new MemoryStorageAgent());
    StreamingContainerManager scm = new StreamingContainerManager(dag);
    for (ContainerStartRequest csr : scm.containerStartRequests) {
        PTContainer container = csr.container;
        if (container.getOperators().get(0).getName().equals("O2")) {
            Assert.assertEquals("Anti-affinity containers set should have 4 containers for other partitions ", 4, container.getStrictAntiPrefs().size());
            for (PTContainer c : container.getStrictAntiPrefs()) {
                for (PTOperator operator : c.getOperators()) {
                    Assert.assertEquals("Partion for O2 should be Anti Prefs", "O2", operator.getName());
                }
            }
        }
    }
    // Check resource handler assigns different hosts for each partition
    ResourceRequestHandler rr = new ResourceRequestHandler();
    int containerMem = 1000;
    Map<String, NodeReport> nodeReports = Maps.newHashMap();
    for (int i = 0; i < 10; i++) {
        String hostName = "host" + i;
        NodeReport nr = BuilderUtils.newNodeReport(BuilderUtils.newNodeId(hostName, 0), NodeState.RUNNING, "httpAddress", "rackName", BuilderUtils.newResource(0, 0), BuilderUtils.newResource(containerMem * 2, 2), 0, null, 0);
        nodeReports.put(nr.getNodeId().getHost(), nr);
    }
    // set resources
    rr.updateNodeReports(Lists.newArrayList(nodeReports.values()));
    Set<String> partitionHostNames = new HashSet<>();
    for (ContainerStartRequest csr : scm.containerStartRequests) {
        String host = rr.getHost(csr, true);
        csr.container.host = host;
        if (csr.container.getOperators().get(0).getName().equals("O2")) {
            Assert.assertNotNull("Host name should not be null", host);
            LOG.info("Partition {} for operator O2 has host = {} ", csr.container.getId(), host);
            Assert.assertTrue("Each Partition should have a different host", !partitionHostNames.contains(host));
            partitionHostNames.add(host);
        }
    }
}

Example 2 with AffinityRulesSet

use of com.datatorrent.api.AffinityRulesSet in project apex-core by apache.

the class PhysicalPlan method assignContainers.

private void assignContainers(Set<PTContainer> newContainers, Set<PTContainer> releaseContainers) {
    Set<PTOperator> mxnUnifiers = Sets.newHashSet();
    for (PTOperator o : this.newOpers.keySet()) {
        mxnUnifiers.addAll(o.upstreamMerge.values());
    }
    Set<PTContainer> updatedContainers = Sets.newHashSet();
    HashMap<PTOperator, PTContainer> operatorContainerMap = Maps.newHashMap();
    for (Map.Entry<PTOperator, Operator> operEntry : this.newOpers.entrySet()) {
        PTOperator oper = operEntry.getKey();
        Checkpoint checkpoint = getActivationCheckpoint(operEntry.getKey());
        initCheckpoint(oper, operEntry.getValue(), checkpoint);
        if (mxnUnifiers.contains(operEntry.getKey())) {
            // MxN unifiers are assigned with the downstream operator
            continue;
        }
        PTContainer newContainer = null;
        int memoryMB = 0;
        // handle container locality
        for (PTOperator inlineOper : oper.getGrouping(Locality.CONTAINER_LOCAL).getOperatorSet()) {
            if (inlineOper.container != null) {
                newContainer = inlineOper.container;
                break;
            }
            memoryMB += inlineOper.operatorMeta.getValue(OperatorContext.MEMORY_MB);
            memoryMB += inlineOper.getBufferServerMemory();
        }
        if (newContainer == null) {
            int vCores = getVCores(oper.getGrouping(Locality.CONTAINER_LOCAL).getOperatorSet());
            // attempt to find empty container with required size
            for (PTContainer c : this.containers) {
                if (c.operators.isEmpty() && c.getState() == PTContainer.State.ACTIVE && c.getAllocatedMemoryMB() == memoryMB && c.getAllocatedVCores() == vCores) {
                    LOG.debug("Reusing existing container {} for {}", c, oper);
                    c.setRequiredMemoryMB(0);
                    c.setRequiredVCores(0);
                    newContainer = c;
                    break;
                }
            }
            if (newContainer == null) {
                // get new container
                LOG.debug("New container for: " + oper);
                newContainer = new PTContainer(this);
                newContainers.add(newContainer);
                containers.add(newContainer);
            }
            updatedContainers.add(newContainer);
        }
        setContainer(oper, newContainer);
    }
    // release containers that are no longer used and update operator to container map for applying anti-affinity
    for (PTContainer c : this.containers) {
        if (c.operators.isEmpty()) {
            LOG.debug("Container {} to be released", c);
            releaseContainers.add(c);
            containers.remove(c);
        } else {
            for (PTOperator oper : c.operators) {
                operatorContainerMap.put(oper, c);
            }
            c.getStrictAntiPrefs().clear();
            c.getPreferredAntiPrefs().clear();
        }
    }
    for (PTContainer c : updatedContainers) {
        updateContainerMemoryWithBufferServer(c);
        c.setRequiredVCores(getVCores(c.getOperators()));
    }
    AffinityRulesSet affinityRuleSet = dag.getAttributes().get(DAGContext.AFFINITY_RULES_SET);
    // Add anti-affinity restrictions in Containers
    if (affinityRuleSet != null && affinityRuleSet.getAffinityRules() != null) {
        setAntiAffinityForContainers(dag, affinityRuleSet.getAffinityRules(), operatorContainerMap);
    }
}

Example 3 with AffinityRulesSet

use of com.datatorrent.api.AffinityRulesSet in project apex-core by apache.

the class LogicalPlanTest method testAffinityRulesDagValidation.

@Test
public void testAffinityRulesDagValidation() {
    TestGeneratorInputOperator o1 = dag.addOperator("O1", new TestGeneratorInputOperator());
    GenericTestOperator o2 = dag.addOperator("O2", new GenericTestOperator());
    GenericTestOperator o3 = dag.addOperator("O3", new GenericTestOperator());
    dag.addStream("stream1", o1.outport, o2.inport1).setLocality(Locality.THREAD_LOCAL);
    StreamMeta stream2 = dag.addStream("stream2", o2.outport1, o3.inport1).setLocality(Locality.CONTAINER_LOCAL);
    AffinityRulesSet ruleSet = new AffinityRulesSet();
    // Valid case:
    List<AffinityRule> rules = new ArrayList<>();
    ruleSet.setAffinityRules(rules);
    AffinityRule rule1 = new AffinityRule(Type.AFFINITY, Locality.CONTAINER_LOCAL, false, "O1", "O3");
    rules.add(rule1);
    dag.setAttribute(DAGContext.AFFINITY_RULES_SET, ruleSet);
    dag.validate();
    // Locality conflicts with affinity rules case:
    AffinityRule rule2 = new AffinityRule(Type.ANTI_AFFINITY, Locality.NODE_LOCAL, false, "O2", "O3");
    rules.add(rule2);
    try {
        dag.validate();
        Assert.fail("DAG validation should fail due to conflicting rules");
    } catch (ValidationException e) {
        Assert.assertEquals("Anti Affinity rule for operators O2 & O3 conflicts with affinity rules or Stream locality", e.getMessage());
    }
    // Change Stream2 locality to Node to check if validation passes
    stream2.setLocality(Locality.RACK_LOCAL);
    dag.validate();
    // Add anti-affinity rule conflicting with rule1
    AffinityRule rule3 = new AffinityRule(Type.ANTI_AFFINITY, Locality.NODE_LOCAL, false, "O1", "O3");
    rules.add(rule3);
    try {
        dag.validate();
        Assert.fail("DAG validation should fail due to conflicting rules");
    } catch (ValidationException e) {
        Assert.assertEquals("Anti Affinity rule for operators O1 & O3 conflicts with affinity rules or Stream locality", e.getMessage());
    }
    // Change rule1 to Rack local to see if dag validation passes
    rules.clear();
    rule1.setLocality(Locality.RACK_LOCAL);
    rules.add(rule1);
    rules.add(rule2);
    rules.add(rule3);
    dag.validate();
    // Add conflicting rules and set relaxLocality for one rule
    AffinityRule rule4 = new AffinityRule(Type.ANTI_AFFINITY, Locality.NODE_LOCAL, true, "O1", "O2");
    rules.add(rule4);
    dag.validate();
    // Set conflicting host locality and check if it fails validation
    rules.clear();
    AffinityRule rule = new AffinityRule(Type.ANTI_AFFINITY, Locality.NODE_LOCAL, false, "O2", "O3");
    rules.add(rule);
    dag.getMeta(o2).getAttributes().put(OperatorContext.LOCALITY_HOST, "host1");
    dag.getMeta(o3).getAttributes().put(OperatorContext.LOCALITY_HOST, "host1");
    try {
        dag.validate();
        Assert.fail("DAG validation should fail due to conflicting host locality");
    } catch (ValidationException e) {
        Assert.assertEquals("Host Locality for operators: O2(host: host1) & O3(host: host1) conflict with anti-affinity rules", e.getMessage());
    }
    // Set conflicting affinity and different host locality for node-local
    // operators
    rules.clear();
    rule = new AffinityRule(Type.AFFINITY, Locality.NODE_LOCAL, false, "O2", "O3");
    rules.add(rule);
    dag.getMeta(o2).getAttributes().put(OperatorContext.LOCALITY_HOST, "host1");
    dag.getMeta(o3).getAttributes().put(OperatorContext.LOCALITY_HOST, "host2");
    try {
        dag.validate();
        Assert.fail("DAG validation should fail due to conflicting host locality");
    } catch (ValidationException e) {
        Assert.assertEquals("Host Locality for operators: O2(host: host1) & O3(host: host2) conflicts with affinity rules", e.getMessage());
    }
    // Check affinity Thread local validation for non-connected operators
    dag.getAttributes().get(DAGContext.AFFINITY_RULES_SET).getAffinityRules().clear();
    rule = new AffinityRule(Type.AFFINITY, Locality.THREAD_LOCAL, false, "O1", "O3");
    rules.add(rule);
    try {
        dag.validate();
        Assert.fail("DAG validation should fail due to conflicting host locality");
    } catch (ValidationException e) {
        Assert.assertEquals("Affinity rule specified THREAD_LOCAL affinity for operators O1 & O3 which are not connected by stream", e.getMessage());
    }
    // Check indirect conflict
    dag = new LogicalPlan();
    o1 = dag.addOperator("O1", new TestGeneratorInputOperator());
    o2 = dag.addOperator("O2", new GenericTestOperator());
    o3 = dag.addOperator("O3", new GenericTestOperator());
    GenericTestOperator o4 = dag.addOperator("O4", new GenericTestOperator());
    GenericTestOperator o5 = dag.addOperator("O5", new GenericTestOperator());
    dag.addStream("stream1", o1.outport, o2.inport1, o3.inport1).setLocality(Locality.NODE_LOCAL);
    dag.addStream("stream2", o3.outport1, o4.inport1);
    dag.addStream("stream3", o2.outport1, o5.inport1);
    rules.clear();
    // O3 and O5 cannot have NODE_LOCAL anti-affinity now, since they already have NODE_LOCAL affinity
    rules.add(new AffinityRule(Type.AFFINITY, Locality.CONTAINER_LOCAL, false, "O1", "O5"));
    rules.add(new AffinityRule(Type.ANTI_AFFINITY, Locality.NODE_LOCAL, false, "O3", "O5"));
    ruleSet = new AffinityRulesSet();
    ruleSet.setAffinityRules(rules);
    dag.setAttribute(DAGContext.AFFINITY_RULES_SET, ruleSet);
    try {
        dag.validate();
        Assert.fail("dag validation should fail due to conflicting affinity rules");
    } catch (ValidationException e) {
        Assert.assertEquals("Anti Affinity rule for operators O3 & O5 conflicts with affinity rules or Stream locality", e.getMessage());
    }
}

Example 4 with AffinityRulesSet

use of com.datatorrent.api.AffinityRulesSet in project apex-core by apache.

the class LogicalPlanModificationTest method testAddOperatorWithAffinityRules.

@Test
public void testAddOperatorWithAffinityRules() {
    GenericTestOperator o1 = dag.addOperator("o1", GenericTestOperator.class);
    GenericTestOperator o2 = dag.addOperator("o2", GenericTestOperator.class);
    GenericTestOperator o3 = dag.addOperator("o3", GenericTestOperator.class);
    dag.addStream("o1.outport1", o1.outport1, o2.inport1);
    dag.addStream("o2.outport1", o2.outport1, o3.inport1);
    TestPlanContext ctx = new TestPlanContext();
    dag.setAttribute(OperatorContext.STORAGE_AGENT, ctx);
    PhysicalPlan plan = new PhysicalPlan(dag, ctx);
    ctx.deploy.clear();
    ctx.undeploy.clear();
    Assert.assertEquals("containers", 3, plan.getContainers().size());
    AffinityRulesSet ruleSet = new AffinityRulesSet();
    List<AffinityRule> rules = new ArrayList<>();
    ruleSet.setAffinityRules(rules);
    rules.add(new AffinityRule(Type.AFFINITY, Locality.CONTAINER_LOCAL, false, "o1", "added1"));
    rules.add(new AffinityRule(Type.ANTI_AFFINITY, Locality.NODE_LOCAL, false, "o3", "added1"));
    dag.setAttribute(DAGContext.AFFINITY_RULES_SET, ruleSet);
    PlanModifier pm = new PlanModifier(plan);
    GenericTestOperator added1 = new GenericTestOperator();
    pm.addOperator("added1", added1);
    pm.addStream("added1.outport1", added1.outport1, o3.inport2);
    Assert.assertEquals("undeploy " + ctx.undeploy, 0, ctx.undeploy.size());
    Assert.assertEquals("deploy " + ctx.deploy, 0, ctx.deploy.size());
    pm.applyChanges(ctx);
    Assert.assertEquals("containers post change", 4, plan.getContainers().size());
    Assert.assertEquals("undeploy " + ctx.undeploy, 1, ctx.undeploy.size());
    Assert.assertEquals("deploy " + ctx.deploy, 2, ctx.deploy.size());
    // Validate affinity rules are applied
    for (PTContainer c : plan.getContainers()) {
        if (c.getOperators().contains("added1")) {
            Assert.assertEquals("Operators O1 and added1 should be in the same container as per affinity rule", 2, c.getOperators().size());
            Assert.assertEquals("Operators O1 and added1 should be in the same container as per affinity rule", "o1", c.getOperators().get(0).getOperatorMeta().getName());
            Assert.assertEquals("Operators O1 and added1 should be in the same container as per affinity rule", "added1", c.getOperators().get(1).getOperatorMeta().getName());
            Set<PTContainer> antiAffinityList = c.getStrictAntiPrefs();
            Assert.assertEquals("There should be one container in antiaffinity list", 1, antiAffinityList.size());
            List<PTOperator> antiAffinityOperators = antiAffinityList.iterator().next().getOperators();
            Assert.assertEquals("AntiAffinity operators should containn operator O3", antiAffinityOperators.iterator().next().getOperatorMeta().getName(), "o3");
        }
    }
}

Example 5 with AffinityRulesSet

use of com.datatorrent.api.AffinityRulesSet in project apex-core by apache.

the class AffinityRulesTest method testAntiAffinityInOperators.

@Test
public void testAntiAffinityInOperators() {
    LogicalPlan dag = new LogicalPlan();
    dag.getAttributes().put(com.datatorrent.api.Context.DAGContext.APPLICATION_PATH, testMeta.getAbsolutePath());
    dag.setAttribute(OperatorContext.STORAGE_AGENT, new MemoryStorageAgent());
    GenericTestOperator o1 = dag.addOperator("O1", GenericTestOperator.class);
    dag.setOperatorAttribute(o1, OperatorContext.MEMORY_MB, 256);
    GenericTestOperator o2 = dag.addOperator("O2", GenericTestOperator.class);
    dag.setOperatorAttribute(o2, OperatorContext.MEMORY_MB, 256);
    dag.getMeta(o1).getAttributes().put(OperatorContext.LOCALITY_HOST, "host1");
    AffinityRulesSet ruleSet = new AffinityRulesSet();
    List<AffinityRule> rules = new ArrayList<>();
    ruleSet.setAffinityRules(rules);
    AffinityRule rule1 = new AffinityRule(Type.ANTI_AFFINITY, Locality.NODE_LOCAL, false, "O1", "O2");
    rules.add(rule1);
    dag.setAttribute(DAGContext.AFFINITY_RULES_SET, ruleSet);
    // .setLocality(Locality.NODE_LOCAL);
    dag.addStream("o1_outport1", o1.outport1, o2.inport1);
    StreamingContainerManager scm = new StreamingContainerManager(dag);
    ResourceRequestHandler rr = new ResourceRequestHandler();
    int containerMem = 1000;
    Map<String, NodeReport> nodeReports = Maps.newHashMap();
    NodeReport nr = BuilderUtils.newNodeReport(BuilderUtils.newNodeId("host1", 0), NodeState.RUNNING, "httpAddress", "rackName", BuilderUtils.newResource(0, 0), BuilderUtils.newResource(containerMem * 2, 2), 0, null, 0);
    nodeReports.put(nr.getNodeId().getHost(), nr);
    nr = BuilderUtils.newNodeReport(BuilderUtils.newNodeId("host2", 0), NodeState.RUNNING, "httpAddress", "rackName", BuilderUtils.newResource(0, 0), BuilderUtils.newResource(containerMem * 2, 2), 0, null, 0);
    nodeReports.put(nr.getNodeId().getHost(), nr);
    // set resources
    rr.updateNodeReports(Lists.newArrayList(nodeReports.values()));
    for (ContainerStartRequest csr : scm.containerStartRequests) {
        String host = rr.getHost(csr, true);
        csr.container.host = host;
        if (csr.container.getOperators().get(0).getName().equals("O1")) {
            Assert.assertEquals("Hosts set to host1 for Operator O1", "host1", host);
        }
        if (csr.container.getOperators().get(0).getName().equals("O2")) {
            Assert.assertEquals("Hosts set to host2 for Operator O2", "host2", host);
        }
    }
}