Examples with ContainerStartRequest com.datatorrent.stram.StreamingContainerAgent.ContainerStartRequest used on opensource projects

Example 11 with ContainerStartRequest

use of com.datatorrent.stram.StreamingContainerAgent.ContainerStartRequest in project apex-core by apache.

the class AffinityRulesTest method testAntiAffinityInOperators.

@Test
public void testAntiAffinityInOperators() {
    LogicalPlan dag = new LogicalPlan();
    dag.getAttributes().put(com.datatorrent.api.Context.DAGContext.APPLICATION_PATH, testMeta.getAbsolutePath());
    dag.setAttribute(OperatorContext.STORAGE_AGENT, new MemoryStorageAgent());
    GenericTestOperator o1 = dag.addOperator("O1", GenericTestOperator.class);
    dag.setOperatorAttribute(o1, OperatorContext.MEMORY_MB, 256);
    GenericTestOperator o2 = dag.addOperator("O2", GenericTestOperator.class);
    dag.setOperatorAttribute(o2, OperatorContext.MEMORY_MB, 256);
    dag.getMeta(o1).getAttributes().put(OperatorContext.LOCALITY_HOST, "host1");
    AffinityRulesSet ruleSet = new AffinityRulesSet();
    List<AffinityRule> rules = new ArrayList<>();
    ruleSet.setAffinityRules(rules);
    AffinityRule rule1 = new AffinityRule(Type.ANTI_AFFINITY, Locality.NODE_LOCAL, false, "O1", "O2");
    rules.add(rule1);
    dag.setAttribute(DAGContext.AFFINITY_RULES_SET, ruleSet);
    // .setLocality(Locality.NODE_LOCAL);
    dag.addStream("o1_outport1", o1.outport1, o2.inport1);
    StreamingContainerManager scm = new StreamingContainerManager(dag);
    ResourceRequestHandler rr = new ResourceRequestHandler();
    int containerMem = 1000;
    Map<String, NodeReport> nodeReports = Maps.newHashMap();
    NodeReport nr = BuilderUtils.newNodeReport(BuilderUtils.newNodeId("host1", 0), NodeState.RUNNING, "httpAddress", "rackName", BuilderUtils.newResource(0, 0), BuilderUtils.newResource(containerMem * 2, 2), 0, null, 0);
    nodeReports.put(nr.getNodeId().getHost(), nr);
    nr = BuilderUtils.newNodeReport(BuilderUtils.newNodeId("host2", 0), NodeState.RUNNING, "httpAddress", "rackName", BuilderUtils.newResource(0, 0), BuilderUtils.newResource(containerMem * 2, 2), 0, null, 0);
    nodeReports.put(nr.getNodeId().getHost(), nr);
    // set resources
    rr.updateNodeReports(Lists.newArrayList(nodeReports.values()));
    for (ContainerStartRequest csr : scm.containerStartRequests) {
        String host = rr.getHost(csr, true);
        csr.container.host = host;
        if (csr.container.getOperators().get(0).getName().equals("O1")) {
            Assert.assertEquals("Hosts set to host1 for Operator O1", "host1", host);
        }
        if (csr.container.getOperators().get(0).getName().equals("O2")) {
            Assert.assertEquals("Hosts set to host2 for Operator O2", "host2", host);
        }
    }
}

Example 12 with ContainerStartRequest

use of com.datatorrent.stram.StreamingContainerAgent.ContainerStartRequest in project apex-core by apache.

the class HostLocalTest method testContainerLocalWithVCores.

@Test
public void testContainerLocalWithVCores() {
    LogicalPlan dag = new LogicalPlan();
    dag.getAttributes().put(com.datatorrent.api.Context.DAGContext.APPLICATION_PATH, new File("target", HostLocalTest.class.getName()).getAbsolutePath());
    dag.setAttribute(OperatorContext.STORAGE_AGENT, new MemoryStorageAgent());
    GenericTestOperator o1 = dag.addOperator("o1", GenericTestOperator.class);
    dag.getMeta(o1).getAttributes().put(OperatorContext.LOCALITY_HOST, "host2");
    GenericTestOperator partitioned = dag.addOperator("partitioned", GenericTestOperator.class);
    dag.addStream("o1_outport1", o1.outport1, partitioned.inport1).setLocality(Locality.CONTAINER_LOCAL);
    dag.setOperatorAttribute(o1, OperatorContext.MEMORY_MB, 256);
    dag.setOperatorAttribute(o1, OperatorContext.VCORES, 1);
    dag.setOperatorAttribute(partitioned, OperatorContext.VCORES, 1);
    StreamingContainerManager scm = new StreamingContainerManager(dag);
    ResourceRequestHandler rr = new ResourceRequestHandler();
    int containerMem = 1000;
    Map<String, NodeReport> nodeReports = Maps.newHashMap();
    NodeReport nr = BuilderUtils.newNodeReport(BuilderUtils.newNodeId("host1", 0), NodeState.RUNNING, "httpAddress", "rackName", BuilderUtils.newResource(0, 0), BuilderUtils.newResource(containerMem * 2, 2), 0, null, 0);
    nodeReports.put(nr.getNodeId().getHost(), nr);
    nr = BuilderUtils.newNodeReport(BuilderUtils.newNodeId("host2", 0), NodeState.RUNNING, "httpAddress", "rackName", BuilderUtils.newResource(0, 0), BuilderUtils.newResource(containerMem * 2, 2), 0, null, 0);
    nodeReports.put(nr.getNodeId().getHost(), nr);
    // set resources
    rr.updateNodeReports(Lists.newArrayList(nodeReports.values()));
    Assert.assertEquals("number of containers is 1", 1, scm.containerStartRequests.size());
    for (ContainerStartRequest csr : scm.containerStartRequests) {
        String host = rr.getHost(csr, true);
        csr.container.host = host;
        Assert.assertEquals("number of vcores", 2, csr.container.getRequiredVCores());
        Assert.assertEquals("Hosts set to host2", "host2", host);
    }
}

Example 13 with ContainerStartRequest

use of com.datatorrent.stram.StreamingContainerAgent.ContainerStartRequest in project apex-core by apache.

the class HostLocalTest method testNodeLocal.

@Test
public void testNodeLocal() {
    LogicalPlan dag = new LogicalPlan();
    dag.getAttributes().put(com.datatorrent.api.Context.DAGContext.APPLICATION_PATH, new File("target", HostLocalTest.class.getName()).getAbsolutePath());
    dag.setAttribute(OperatorContext.STORAGE_AGENT, new MemoryStorageAgent());
    GenericTestOperator o1 = dag.addOperator("o1", GenericTestOperator.class);
    dag.setOperatorAttribute(o1, OperatorContext.MEMORY_MB, 256);
    GenericTestOperator partitioned = dag.addOperator("partitioned", GenericTestOperator.class);
    dag.setOperatorAttribute(partitioned, OperatorContext.MEMORY_MB, 256);
    dag.getMeta(partitioned).getAttributes().put(OperatorContext.LOCALITY_HOST, "host1");
    dag.addStream("o1_outport1", o1.outport1, partitioned.inport1).setLocality(Locality.NODE_LOCAL);
    StreamingContainerManager scm = new StreamingContainerManager(dag);
    ResourceRequestHandler rr = new ResourceRequestHandler();
    int containerMem = 1000;
    Map<String, NodeReport> nodeReports = Maps.newHashMap();
    NodeReport nr = BuilderUtils.newNodeReport(BuilderUtils.newNodeId("host1", 0), NodeState.RUNNING, "httpAddress", "rackName", BuilderUtils.newResource(0, 0), BuilderUtils.newResource(containerMem * 2, 2), 0, null, 0);
    nodeReports.put(nr.getNodeId().getHost(), nr);
    nr = BuilderUtils.newNodeReport(BuilderUtils.newNodeId("host2", 0), NodeState.RUNNING, "httpAddress", "rackName", BuilderUtils.newResource(0, 0), BuilderUtils.newResource(containerMem * 2, 2), 0, null, 0);
    nodeReports.put(nr.getNodeId().getHost(), nr);
    // set resources
    rr.updateNodeReports(Lists.newArrayList(nodeReports.values()));
    for (ContainerStartRequest csr : scm.containerStartRequests) {
        String host = rr.getHost(csr, true);
        csr.container.host = host;
        Assert.assertEquals("Hosts set to host1", "host1", host);
    }
}

Example 14 with ContainerStartRequest

use of com.datatorrent.stram.StreamingContainerAgent.ContainerStartRequest in project apex-core by apache.

the class BlacklistBasedResourceRequestHandler method reissueContainerRequests.

@Override
public void reissueContainerRequests(AMRMClient<ContainerRequest> amRmClient, Map<StreamingContainerAgent.ContainerStartRequest, MutablePair<Integer, ContainerRequest>> requestedResources, int loopCounter, ResourceRequestHandler resourceRequestor, List<ContainerRequest> containerRequests, List<ContainerRequest> removedContainerRequests) {
    if (!requestedResources.isEmpty()) {
        // Check if any requests timed out, create new requests in that case
        recreateContainerRequest(requestedResources, loopCounter, resourceRequestor, removedContainerRequests);
    }
    // Issue all host specific requests first
    if (!hostSpecificRequestsMap.isEmpty()) {
        LOG.info("Issue Host specific requests first");
        // Blacklist all the nodes and issue request for host specific
        Entry<String, List<ContainerRequest>> set = hostSpecificRequestsMap.entrySet().iterator().next();
        List<ContainerRequest> requests = set.getValue();
        List<String> blacklistNodes = resourceRequestor.getNodesExceptHost(requests.get(0).getNodes());
        amRmClient.updateBlacklist(blacklistNodes, requests.get(0).getNodes());
        blacklistedNodesForHostSpecificRequests = blacklistNodes;
        LOG.info("Sending {} request(s) after blacklisting all nodes other than {}", requests.size(), requests.get(0).getNodes());
        for (ContainerRequest cr : requests) {
            ContainerStartRequest csr = hostSpecificRequests.get(cr);
            ContainerRequest newCr = new ContainerRequest(cr.getCapability(), null, null, cr.getPriority());
            MutablePair<Integer, ContainerRequest> pair = new MutablePair<>(loopCounter, newCr);
            requestedResources.put(csr, pair);
            containerRequests.add(newCr);
            hostSpecificRequests.remove(cr);
        }
        hostSpecificRequestsMap.remove(set.getKey());
    } else {
        if (blacklistedNodesForHostSpecificRequests != null) {
            // Remove the blacklisted nodes during host specific requests
            LOG.debug("All requests done.. Removing nodes from blacklist {}", blacklistedNodesForHostSpecificRequests);
            amRmClient.updateBlacklist(null, blacklistedNodesForHostSpecificRequests);
            blacklistedNodesForHostSpecificRequests = null;
        }
        // Proceed with other requests after host specific requests are done
        if (!otherContainerRequests.isEmpty()) {
            for (Entry<ContainerRequest, ContainerStartRequest> entry : otherContainerRequests.entrySet()) {
                ContainerRequest cr = entry.getKey();
                ContainerStartRequest csr = entry.getValue();
                MutablePair<Integer, ContainerRequest> pair = new MutablePair<>(loopCounter, cr);
                requestedResources.put(csr, pair);
                containerRequests.add(cr);
            }
            otherContainerRequests.clear();
        }
    }
}

Example 15 with ContainerStartRequest

use of com.datatorrent.stram.StreamingContainerAgent.ContainerStartRequest in project apex-core by apache.

the class StreamingContainerManager method requestContainer.

private void requestContainer(PTContainer c) {
    ContainerStartRequest dr = new ContainerStartRequest(c);
    containerStartRequests.add(dr);
    pendingAllocation.add(dr.container);
    lastResourceRequest = System.currentTimeMillis();
    for (PTOperator operator : c.getOperators()) {
        operator.setState(PTOperator.State.INACTIVE);
    }
}