Examples with PTContainer com.datatorrent.stram.plan.physical.PTContainer used on opensource projects

Example 26 with PTContainer

use of com.datatorrent.stram.plan.physical.PTContainer in project apex-core by apache.

the class ResourceRequestHandler method getHost.

public String getHost(ContainerStartRequest csr, boolean first) {
    String host = null;
    PTContainer c = csr.container;
    if (first) {
        for (PTOperator oper : c.getOperators()) {
            HostOperatorSet grpObj = oper.getNodeLocalOperators();
            host = nodeLocalMapping.get(grpObj.getOperatorSet());
            if (host != null) {
                antiAffinityMapping.put(c, host);
                return host;
            }
            if (grpObj.getHost() != null) {
                host = grpObj.getHost();
                // using the 1st host value as host for container
                break;
            }
        }
        if (host != null && nodeReportMap.get(host) != null) {
            for (PTOperator oper : c.getOperators()) {
                HostOperatorSet grpObj = oper.getNodeLocalOperators();
                Set<PTOperator> nodeLocalSet = grpObj.getOperatorSet();
                NodeReport report = nodeReportMap.get(host);
                int aggrMemory = c.getRequiredMemoryMB();
                int vCores = c.getRequiredVCores();
                Set<PTContainer> containers = Sets.newHashSet();
                containers.add(c);
                for (PTOperator nodeLocalOper : nodeLocalSet) {
                    if (!containers.contains(nodeLocalOper.getContainer())) {
                        aggrMemory += nodeLocalOper.getContainer().getRequiredMemoryMB();
                        vCores += nodeLocalOper.getContainer().getRequiredVCores();
                        containers.add(nodeLocalOper.getContainer());
                    }
                }
                int memAvailable = report.getCapability().getMemory() - report.getUsed().getMemory();
                int vCoresAvailable = report.getCapability().getVirtualCores() - report.getUsed().getVirtualCores();
                if (memAvailable >= aggrMemory && vCoresAvailable >= vCores) {
                    nodeLocalMapping.put(nodeLocalSet, host);
                    antiAffinityMapping.put(c, host);
                    return host;
                }
            }
        }
    }
    // the host requested didn't have the resources so looking for other hosts
    host = null;
    List<String> antiHosts = new ArrayList<>();
    List<String> antiPreferredHosts = new ArrayList<>();
    if (!c.getStrictAntiPrefs().isEmpty()) {
        // Check if containers are allocated already for the anti-affinity containers
        populateAntiHostList(c, antiHosts);
    }
    if (!c.getPreferredAntiPrefs().isEmpty()) {
        populateAntiHostList(c, antiPreferredHosts);
    }
    LOG.info("Strict anti-affinity = {} for container with operators {}", antiHosts, StringUtils.join(c.getOperators(), ","));
    for (PTOperator oper : c.getOperators()) {
        HostOperatorSet grpObj = oper.getNodeLocalOperators();
        Set<PTOperator> nodeLocalSet = grpObj.getOperatorSet();
        if (nodeLocalSet.size() > 1 || !c.getStrictAntiPrefs().isEmpty() || !c.getPreferredAntiPrefs().isEmpty()) {
            LOG.info("Finding new host for {}", nodeLocalSet);
            int aggrMemory = c.getRequiredMemoryMB();
            int vCores = c.getRequiredVCores();
            Set<PTContainer> containers = Sets.newHashSet();
            containers.add(c);
            // aggregate memory required for all containers
            for (PTOperator nodeLocalOper : nodeLocalSet) {
                if (!containers.contains(nodeLocalOper.getContainer())) {
                    aggrMemory += nodeLocalOper.getContainer().getRequiredMemoryMB();
                    vCores += nodeLocalOper.getContainer().getRequiredVCores();
                    containers.add(nodeLocalOper.getContainer());
                }
            }
            host = assignHost(host, antiHosts, antiPreferredHosts, grpObj, nodeLocalSet, aggrMemory, vCores);
            if (host == null && !antiPreferredHosts.isEmpty() && !antiHosts.isEmpty()) {
                // Drop the preferred constraint and try allocation
                antiPreferredHosts.clear();
                host = assignHost(host, antiHosts, antiPreferredHosts, grpObj, nodeLocalSet, aggrMemory, vCores);
            }
            if (host != null) {
                antiAffinityMapping.put(c, host);
            } else {
                host = INVALID_HOST;
            }
        }
    }
    LOG.info("Found host {}", host);
    return host;
}

Example 27 with PTContainer

use of com.datatorrent.stram.plan.physical.PTContainer in project apex-core by apache.

the class StramRecoveryTest method testContainerManager.

/**
   * Test serialization of the container manager with mock execution layer.
   * @throws Exception
   */
private void testContainerManager(StorageAgent agent) throws Exception {
    dag.setAttribute(OperatorContext.STORAGE_AGENT, agent);
    StatsListeningOperator o1 = dag.addOperator("o1", StatsListeningOperator.class);
    FSRecoveryHandler recoveryHandler = new FSRecoveryHandler(dag.assertAppPath(), new Configuration(false));
    StreamingContainerManager scm = StreamingContainerManager.getInstance(recoveryHandler, dag, false);
    File expFile = new File(recoveryHandler.getDir(), FSRecoveryHandler.FILE_SNAPSHOT);
    Assert.assertTrue("snapshot file " + expFile, expFile.exists());
    PhysicalPlan plan = scm.getPhysicalPlan();
    assertEquals("number required containers", 1, plan.getContainers().size());
    PTOperator o1p1 = plan.getOperators(dag.getMeta(o1)).get(0);
    @SuppressWarnings("UnusedAssignment") MockContainer /* sneaky: the constructor does some changes to the container */
    mc = new MockContainer(scm, o1p1.getContainer());
    PTContainer originalContainer = o1p1.getContainer();
    Assert.assertNotNull(o1p1.getContainer().bufferServerAddress);
    assertEquals(PTContainer.State.ACTIVE, o1p1.getContainer().getState());
    assertEquals("state " + o1p1, PTOperator.State.PENDING_DEPLOY, o1p1.getState());
    // test restore initial snapshot + log
    dag = StramTestSupport.createDAG(testMeta);
    scm = StreamingContainerManager.getInstance(new FSRecoveryHandler(dag.assertAppPath(), new Configuration(false)), dag, false);
    dag = scm.getLogicalPlan();
    plan = scm.getPhysicalPlan();
    o1p1 = plan.getOperators(dag.getOperatorMeta("o1")).get(0);
    assertEquals("post restore state " + o1p1, PTOperator.State.PENDING_DEPLOY, o1p1.getState());
    o1 = (StatsListeningOperator) o1p1.getOperatorMeta().getOperator();
    assertEquals("containerId", originalContainer.getExternalId(), o1p1.getContainer().getExternalId());
    assertEquals("stats listener", 1, o1p1.statsListeners.size());
    // stats are not logged
    assertEquals("number stats calls", 0, o1.processStatsCnt);
    assertEquals("post restore 1", PTContainer.State.ALLOCATED, o1p1.getContainer().getState());
    assertEquals("post restore 1", originalContainer.bufferServerAddress, o1p1.getContainer().bufferServerAddress);
    StreamingContainerAgent sca = scm.getContainerAgent(originalContainer.getExternalId());
    Assert.assertNotNull("allocated container restored " + originalContainer, sca);
    assertEquals("memory usage allocated container", (int) OperatorContext.MEMORY_MB.defaultValue, sca.container.getAllocatedMemoryMB());
    // YARN-1490 - simulate container terminated on AM recovery
    scm.scheduleContainerRestart(originalContainer.getExternalId());
    assertEquals("memory usage of failed container", 0, sca.container.getAllocatedMemoryMB());
    Checkpoint firstCheckpoint = new Checkpoint(3, 0, 0);
    mc = new MockContainer(scm, o1p1.getContainer());
    checkpoint(scm, o1p1, firstCheckpoint);
    mc.stats(o1p1.getId()).deployState(OperatorHeartbeat.DeployState.ACTIVE).currentWindowId(3).checkpointWindowId(3);
    mc.sendHeartbeat();
    assertEquals("state " + o1p1, PTOperator.State.ACTIVE, o1p1.getState());
    // logical plan modification triggers snapshot
    CreateOperatorRequest cor = new CreateOperatorRequest();
    cor.setOperatorFQCN(GenericTestOperator.class.getName());
    cor.setOperatorName("o2");
    CreateStreamRequest csr = new CreateStreamRequest();
    csr.setSourceOperatorName("o1");
    csr.setSourceOperatorPortName("outport");
    csr.setSinkOperatorName("o2");
    csr.setSinkOperatorPortName("inport1");
    FutureTask<?> lpmf = scm.logicalPlanModification(Lists.newArrayList(cor, csr));
    while (!lpmf.isDone()) {
        scm.monitorHeartbeat(false);
    }
    // unmask exception, if any
    Assert.assertNull(lpmf.get());
    Assert.assertSame("dag references", dag, scm.getLogicalPlan());
    assertEquals("number operators after plan modification", 2, dag.getAllOperators().size());
    // set operator state triggers journal write
    o1p1.setState(PTOperator.State.INACTIVE);
    Checkpoint offlineCheckpoint = new Checkpoint(10, 0, 0);
    // write checkpoint while AM is out,
    // it needs to be picked up as part of restore
    checkpoint(scm, o1p1, offlineCheckpoint);
    // test restore
    dag = StramTestSupport.createDAG(testMeta);
    scm = StreamingContainerManager.getInstance(new FSRecoveryHandler(dag.assertAppPath(), new Configuration(false)), dag, false);
    Assert.assertNotSame("dag references", dag, scm.getLogicalPlan());
    assertEquals("number operators after restore", 2, scm.getLogicalPlan().getAllOperators().size());
    dag = scm.getLogicalPlan();
    plan = scm.getPhysicalPlan();
    o1p1 = plan.getOperators(dag.getOperatorMeta("o1")).get(0);
    assertEquals("post restore state " + o1p1, PTOperator.State.INACTIVE, o1p1.getState());
    o1 = (StatsListeningOperator) o1p1.getOperatorMeta().getOperator();
    assertEquals("stats listener", 1, o1p1.statsListeners.size());
    assertEquals("number stats calls post restore", 1, o1.processStatsCnt);
    assertEquals("post restore 1", PTContainer.State.ACTIVE, o1p1.getContainer().getState());
    assertEquals("post restore 1", originalContainer.bufferServerAddress, o1p1.getContainer().bufferServerAddress);
    // offline checkpoint detection
    assertEquals("checkpoints after recovery", Lists.newArrayList(firstCheckpoint, offlineCheckpoint), o1p1.checkpoints);
}

Example 28 with PTContainer

use of com.datatorrent.stram.plan.physical.PTContainer in project apex-core by apache.

the class StreamCodecTest method testCascadingStreamCodec.

@Test
public void testCascadingStreamCodec() {
    GenericTestOperator node1 = dag.addOperator("node1", GenericTestOperator.class);
    GenericTestOperator node2 = dag.addOperator("node2", GenericTestOperator.class);
    GenericTestOperator node3 = dag.addOperator("node3", GenericTestOperator.class);
    dag.setOperatorAttribute(node1, Context.OperatorContext.PARTITIONER, new StatelessPartitioner<GenericTestOperator>(3));
    dag.setOutputPortAttribute(node1.outport1, Context.PortContext.UNIFIER_LIMIT, 2);
    TestStreamCodec serDe = new TestStreamCodec();
    dag.setInputPortAttribute(node2.inport1, Context.PortContext.STREAM_CODEC, serDe);
    TestStreamCodec2 serDe2 = new TestStreamCodec2();
    dag.setInputPortAttribute(node3.inport1, Context.PortContext.STREAM_CODEC, serDe2);
    dag.addStream("n1n2n3", node1.outport1, node2.inport1, node3.inport1);
    dag.setAttribute(LogicalPlan.CONTAINERS_MAX_COUNT, Integer.MAX_VALUE);
    StramTestSupport.MemoryStorageAgent msa = new StramTestSupport.MemoryStorageAgent();
    dag.setAttribute(Context.OperatorContext.STORAGE_AGENT, msa);
    StreamingContainerManager dnm = new StreamingContainerManager(dag);
    PhysicalPlan plan = dnm.getPhysicalPlan();
    List<PTContainer> containers = plan.getContainers();
    Assert.assertEquals("number containers", 7, containers.size());
    for (int i = 0; i < containers.size(); ++i) {
        StreamingContainerManagerTest.assignContainer(dnm, "container" + (i + 1));
    }
    LogicalPlan.OperatorMeta n1meta = dag.getMeta(node1);
    LogicalPlan.OperatorMeta n2meta = dag.getMeta(node2);
    LogicalPlan.OperatorMeta n3meta = dag.getMeta(node3);
    for (PTContainer container : containers) {
        List<PTOperator> operators = container.getOperators();
        for (PTOperator operator : operators) {
            if (!operator.isUnifier()) {
                if (operator.getOperatorMeta() == n1meta) {
                    OperatorDeployInfo odi = getOperatorDeployInfo(operator, n1meta.getName(), dnm);
                    OperatorDeployInfo.OutputDeployInfo otdi = getOutputDeployInfo(odi, n1meta.getMeta(node1.outport1));
                    String id = n1meta.getName() + " " + otdi.portName;
                    Assert.assertEquals("number stream codecs " + id, otdi.streamCodecs.size(), 2);
                    checkPresentStreamCodec(n2meta, node2.inport1, otdi.streamCodecs, id, plan);
                    checkPresentStreamCodec(n3meta, node3.inport1, otdi.streamCodecs, id, plan);
                } else if (operator.getOperatorMeta() == n2meta) {
                    OperatorDeployInfo odi = getOperatorDeployInfo(operator, n2meta.getName(), dnm);
                    OperatorDeployInfo.InputDeployInfo idi = getInputDeployInfo(odi, n2meta.getMeta(node2.inport1));
                    String id = n2meta.getName() + " " + idi.portName;
                    Assert.assertEquals("number stream codecs " + id, idi.streamCodecs.size(), 1);
                    checkPresentStreamCodec(n2meta, node2.inport1, idi.streamCodecs, id, plan);
                } else if (operator.getOperatorMeta() == n3meta) {
                    OperatorDeployInfo odi = getOperatorDeployInfo(operator, n3meta.getName(), dnm);
                    OperatorDeployInfo.InputDeployInfo idi = getInputDeployInfo(odi, n3meta.getMeta(node3.inport1));
                    String id = n3meta.getName() + " " + idi.portName;
                    Assert.assertEquals("number stream codecs " + id, idi.streamCodecs.size(), 1);
                    checkPresentStreamCodec(n3meta, node3.inport1, idi.streamCodecs, id, plan);
                }
            } else {
                OperatorDeployInfo odi = getOperatorDeployInfo(operator, operator.getName(), dnm);
                Assert.assertEquals("unifier outputs " + operator.getName(), 1, operator.getOutputs().size());
                PTOperator.PTOutput out = operator.getOutputs().get(0);
                Assert.assertEquals("unifier sinks " + operator.getName(), 1, out.sinks.size());
                PTOperator.PTInput idInput = out.sinks.get(0);
                LogicalPlan.OperatorMeta idMeta = StreamingContainerAgent.getIdentifyingInputPortMeta(idInput).getOperatorMeta();
                Operator.InputPort<?> idInputPort = null;
                if (idMeta == n2meta) {
                    idInputPort = node2.inport1;
                } else if (idMeta == n3meta) {
                    idInputPort = node3.inport1;
                }
                List<OperatorDeployInfo.InputDeployInfo> idis = odi.inputs;
                for (OperatorDeployInfo.InputDeployInfo idi : idis) {
                    String id = operator.getName() + " " + idi.portName;
                    Assert.assertEquals("number stream codecs " + id, idi.streamCodecs.size(), 1);
                    checkPresentStreamCodec(idMeta, idInputPort, idi.streamCodecs, id, plan);
                }
            }
        }
    }
}

Example 29 with PTContainer

use of com.datatorrent.stram.plan.physical.PTContainer in project apex-core by apache.

the class StreamCodecTest method testMultipleStreamCodecs.

@Test
public void testMultipleStreamCodecs() {
    GenericTestOperator node1 = dag.addOperator("node1", GenericTestOperator.class);
    GenericTestOperator node2 = dag.addOperator("node2", GenericTestOperator.class);
    GenericTestOperator node3 = dag.addOperator("node3", GenericTestOperator.class);
    TestStreamCodec serDe = new TestStreamCodec();
    dag.setInputPortAttribute(node2.inport1, Context.PortContext.STREAM_CODEC, serDe);
    TestStreamCodec2 serDe2 = new TestStreamCodec2();
    dag.setInputPortAttribute(node3.inport1, Context.PortContext.STREAM_CODEC, serDe2);
    dag.addStream("n1n2n3", node1.outport1, node2.inport1, node3.inport1);
    dag.setAttribute(LogicalPlan.CONTAINERS_MAX_COUNT, Integer.MAX_VALUE);
    StramTestSupport.MemoryStorageAgent msa = new StramTestSupport.MemoryStorageAgent();
    dag.setAttribute(Context.OperatorContext.STORAGE_AGENT, msa);
    StreamingContainerManager dnm = new StreamingContainerManager(dag);
    PhysicalPlan plan = dnm.getPhysicalPlan();
    List<PTContainer> containers = plan.getContainers();
    Assert.assertEquals("number containers", 3, containers.size());
    for (int i = 0; i < containers.size(); ++i) {
        StreamingContainerManagerTest.assignContainer(dnm, "container" + (i + 1));
    }
    LogicalPlan.OperatorMeta n1meta = dag.getMeta(node1);
    LogicalPlan.OperatorMeta n2meta = dag.getMeta(node2);
    LogicalPlan.OperatorMeta n3meta = dag.getMeta(node3);
    OperatorDeployInfo n1di = getSingleOperatorDeployInfo(node1, dnm);
    OperatorDeployInfo.OutputDeployInfo n1odi = getOutputDeployInfo(n1di, n1meta.getMeta(node1.outport1));
    String id = n1meta.getName() + " " + n1odi.portName;
    Assert.assertEquals("number stream codecs " + id, n1odi.streamCodecs.size(), 2);
    checkPresentStreamCodec(n2meta, node2.inport1, n1odi.streamCodecs, id, plan);
    checkPresentStreamCodec(n3meta, node3.inport1, n1odi.streamCodecs, id, plan);
    OperatorDeployInfo n2di = getSingleOperatorDeployInfo(node2, dnm);
    OperatorDeployInfo.InputDeployInfo n2idi = getInputDeployInfo(n2di, n2meta.getMeta(node2.inport1));
    id = n2meta.getName() + " " + n2idi.portName;
    Assert.assertEquals("number stream codecs " + id, n2idi.streamCodecs.size(), 1);
    checkPresentStreamCodec(n2meta, node2.inport1, n2idi.streamCodecs, id, plan);
    OperatorDeployInfo n3di = getSingleOperatorDeployInfo(node3, dnm);
    OperatorDeployInfo.InputDeployInfo n3idi = getInputDeployInfo(n3di, n3meta.getMeta(node3.inport1));
    id = n3meta.getName() + " " + n3idi.portName;
    Assert.assertEquals("number stream codecs " + id, n3idi.streamCodecs.size(), 1);
    checkPresentStreamCodec(n3meta, node3.inport1, n3idi.streamCodecs, id, plan);
}

Example 30 with PTContainer

use of com.datatorrent.stram.plan.physical.PTContainer in project apex-core by apache.

the class StreamCodecTest method testMxNMultipleStreamCodecs.

@Test
public void testMxNMultipleStreamCodecs() {
    GenericTestOperator node1 = dag.addOperator("node1", GenericTestOperator.class);
    dag.setOperatorAttribute(node1, Context.OperatorContext.PARTITIONER, new StatelessPartitioner<GenericTestOperator>(2));
    GenericTestOperator node2 = dag.addOperator("node2", GenericTestOperator.class);
    dag.setOperatorAttribute(node2, Context.OperatorContext.PARTITIONER, new StatelessPartitioner<GenericTestOperator>(3));
    TestStreamCodec serDe = new TestStreamCodec();
    dag.setInputPortAttribute(node2.inport1, Context.PortContext.STREAM_CODEC, serDe);
    GenericTestOperator node3 = dag.addOperator("node3", GenericTestOperator.class);
    dag.setOperatorAttribute(node3, Context.OperatorContext.PARTITIONER, new StatelessPartitioner<GenericTestOperator>(3));
    TestStreamCodec serDe2 = new TestStreamCodec();
    dag.setInputPortAttribute(node3.inport1, Context.PortContext.STREAM_CODEC, serDe2);
    dag.addStream("n1n2n3", node1.outport1, node2.inport1, node3.inport1);
    dag.setAttribute(LogicalPlan.CONTAINERS_MAX_COUNT, Integer.MAX_VALUE);
    StramTestSupport.MemoryStorageAgent msa = new StramTestSupport.MemoryStorageAgent();
    dag.setAttribute(Context.OperatorContext.STORAGE_AGENT, msa);
    StreamingContainerManager dnm = new StreamingContainerManager(dag);
    PhysicalPlan plan = dnm.getPhysicalPlan();
    List<PTContainer> containers = plan.getContainers();
    for (int i = 0; i < containers.size(); ++i) {
        StreamingContainerManagerTest.assignContainer(dnm, "container" + (i + 1));
    }
    LogicalPlan.OperatorMeta n1meta = dag.getMeta(node1);
    LogicalPlan.OperatorMeta n2meta = dag.getMeta(node2);
    LogicalPlan.OperatorMeta n3meta = dag.getMeta(node3);
    // Sanity check that physical operators have been allocated for n1meta and n2meta
    Assert.assertEquals("number operators " + n1meta.getName(), 2, plan.getOperators(n1meta).size());
    Assert.assertEquals("number operators " + n2meta.getName(), 3, plan.getOperators(n2meta).size());
    Assert.assertEquals("number operators " + n3meta.getName(), 3, plan.getOperators(n3meta).size());
    checkMxNStreamCodecs(node1, node2, node3, dnm);
}