Examples with PTOperator com.datatorrent.stram.plan.physical.PTOperator used on opensource projects

Example 31 with PTOperator

use of com.datatorrent.stram.plan.physical.PTOperator in project apex-core by apache.

the class StreamCodecTest method testPartitioningMultipleStreamCodecs.

@Test
public void testPartitioningMultipleStreamCodecs() {
    GenericTestOperator node1 = dag.addOperator("node1", GenericTestOperator.class);
    GenericTestOperator node2 = dag.addOperator("node2", GenericTestOperator.class);
    GenericTestOperator node3 = dag.addOperator("node3", GenericTestOperator.class);
    dag.setOperatorAttribute(node1, Context.OperatorContext.PARTITIONER, new StatelessPartitioner<GenericTestOperator>(2));
    TestStreamCodec serDe = new TestStreamCodec();
    dag.setInputPortAttribute(node2.inport1, Context.PortContext.STREAM_CODEC, serDe);
    TestStreamCodec2 serDe2 = new TestStreamCodec2();
    dag.setInputPortAttribute(node3.inport1, Context.PortContext.STREAM_CODEC, serDe2);
    dag.addStream("n1n2n3", node1.outport1, node2.inport1, node3.inport1);
    dag.setAttribute(LogicalPlan.CONTAINERS_MAX_COUNT, Integer.MAX_VALUE);
    StramTestSupport.MemoryStorageAgent msa = new StramTestSupport.MemoryStorageAgent();
    dag.setAttribute(Context.OperatorContext.STORAGE_AGENT, msa);
    StreamingContainerManager dnm = new StreamingContainerManager(dag);
    PhysicalPlan plan = dnm.getPhysicalPlan();
    List<PTContainer> containers = plan.getContainers();
    Assert.assertEquals("number containers", 4, containers.size());
    for (int i = 0; i < containers.size(); ++i) {
        StreamingContainerManagerTest.assignContainer(dnm, "container" + (i + 1));
    }
    LogicalPlan.OperatorMeta n1meta = dag.getMeta(node1);
    LogicalPlan.OperatorMeta n2meta = dag.getMeta(node2);
    LogicalPlan.OperatorMeta n3meta = dag.getMeta(node3);
    for (PTContainer container : containers) {
        List<PTOperator> operators = container.getOperators();
        for (PTOperator operator : operators) {
            if (!operator.isUnifier()) {
                if (operator.getOperatorMeta() == n1meta) {
                    OperatorDeployInfo odi = getOperatorDeployInfo(operator, n1meta.getName(), dnm);
                    OperatorDeployInfo.OutputDeployInfo otdi = getOutputDeployInfo(odi, n1meta.getMeta(node1.outport1));
                    String id = n1meta.getName() + " " + otdi.portName;
                    Assert.assertEquals("number stream codecs " + id, otdi.streamCodecs.size(), 2);
                    checkPresentStreamCodec(n2meta, node2.inport1, otdi.streamCodecs, id, plan);
                    checkPresentStreamCodec(n3meta, node3.inport1, otdi.streamCodecs, id, plan);
                } else if (operator.getOperatorMeta() == n2meta) {
                    OperatorDeployInfo odi = getOperatorDeployInfo(operator, n2meta.getName(), dnm);
                    OperatorDeployInfo.InputDeployInfo idi = getInputDeployInfo(odi, n2meta.getMeta(node2.inport1));
                    String id = n2meta.getName() + " " + idi.portName;
                    Assert.assertEquals("number stream codecs " + id, idi.streamCodecs.size(), 1);
                    checkPresentStreamCodec(n2meta, node2.inport1, idi.streamCodecs, id, plan);
                } else if (operator.getOperatorMeta() == n3meta) {
                    OperatorDeployInfo odi = getOperatorDeployInfo(operator, n3meta.getName(), dnm);
                    OperatorDeployInfo.InputDeployInfo idi = getInputDeployInfo(odi, n3meta.getMeta(node3.inport1));
                    String id = n3meta.getName() + " " + idi.portName;
                    Assert.assertEquals("number stream codecs " + id, idi.streamCodecs.size(), 1);
                    checkPresentStreamCodec(n3meta, node3.inport1, idi.streamCodecs, id, plan);
                }
            } else {
                OperatorDeployInfo odi = getOperatorDeployInfo(operator, operator.getName(), dnm);
                Assert.assertEquals("unifier outputs " + operator.getName(), 1, operator.getOutputs().size());
                PTOperator.PTOutput out = operator.getOutputs().get(0);
                Assert.assertEquals("unifier sinks " + operator.getName(), 1, out.sinks.size());
                PTOperator.PTInput idInput = out.sinks.get(0);
                LogicalPlan.OperatorMeta idMeta = idInput.target.getOperatorMeta();
                Operator.InputPort<?> idInputPort = null;
                if (idMeta == n2meta) {
                    idInputPort = node2.inport1;
                } else if (idMeta == n3meta) {
                    idInputPort = node3.inport1;
                }
                List<OperatorDeployInfo.InputDeployInfo> idis = odi.inputs;
                for (OperatorDeployInfo.InputDeployInfo idi : idis) {
                    String id = operator.getName() + " " + idi.portName;
                    Assert.assertEquals("number stream codecs " + id, idi.streamCodecs.size(), 1);
                    checkPresentStreamCodec(idMeta, idInputPort, idi.streamCodecs, id, plan);
                }
            }
        }
    }
}

Example 32 with PTOperator

use of com.datatorrent.stram.plan.physical.PTOperator in project apex-core by apache.

the class StreamCodecTest method testInlineStreamCodec.

@Test
public void testInlineStreamCodec() {
    GenericTestOperator node1 = dag.addOperator("node1", GenericTestOperator.class);
    GenericTestOperator node2 = dag.addOperator("node2", GenericTestOperator.class);
    GenericTestOperator node3 = dag.addOperator("node3", GenericTestOperator.class);
    TestStreamCodec serDe = new TestStreamCodec();
    dag.setInputPortAttribute(node2.inport1, Context.PortContext.STREAM_CODEC, serDe);
    dag.setInputPortAttribute(node3.inport1, Context.PortContext.STREAM_CODEC, serDe);
    dag.addStream("n1n2n3", node1.outport1, node2.inport1, node3.inport1);
    // Relying on container max count for the manager to layout node1 and node3 in the
    // same container in inline fashion and node2 in a separate container
    dag.setAttribute(LogicalPlan.CONTAINERS_MAX_COUNT, 2);
    StramTestSupport.MemoryStorageAgent msa = new StramTestSupport.MemoryStorageAgent();
    dag.setAttribute(Context.OperatorContext.STORAGE_AGENT, msa);
    StreamingContainerManager dnm = new StreamingContainerManager(dag);
    PhysicalPlan plan = dnm.getPhysicalPlan();
    List<PTContainer> containers = plan.getContainers();
    Assert.assertEquals("number containers", 2, containers.size());
    for (int i = 0; i < containers.size(); ++i) {
        StreamingContainerManagerTest.assignContainer(dnm, "container" + (i + 1));
    }
    LogicalPlan.OperatorMeta n1meta = dag.getMeta(node1);
    LogicalPlan.OperatorMeta nonInlineMeta = null;
    for (int i = 0; i < containers.size(); ++i) {
        PTContainer container = containers.get(i);
        List<PTOperator> operators = container.getOperators();
        if (operators.size() == 1) {
            nonInlineMeta = operators.get(0).getOperatorMeta();
            break;
        }
    }
    Assert.assertNotNull("non inline operator meta is null", nonInlineMeta);
    GenericTestOperator nonInlineOperator = null;
    Operator.InputPort<?> niInputPort = null;
    if (nonInlineMeta.getName().equals("node2")) {
        nonInlineOperator = node2;
        niInputPort = node2.inport1;
    } else if (nonInlineMeta.getName().equals("node3")) {
        nonInlineOperator = node3;
        niInputPort = node3.inport1;
    }
    Assert.assertNotNull("non inline operator is null", nonInlineOperator);
    OperatorDeployInfo n1di = getSingleOperatorDeployInfo(node1, dnm);
    OperatorDeployInfo.OutputDeployInfo n1odi = getOutputDeployInfo(n1di, n1meta.getMeta(node1.outport1));
    String id = n1meta.getName() + " " + n1odi.portName;
    Assert.assertEquals("number stream codecs " + id, n1odi.streamCodecs.size(), 1);
    checkPresentStreamCodec(nonInlineMeta, niInputPort, n1odi.streamCodecs, id, plan);
    OperatorDeployInfo odi = getSingleOperatorDeployInfo(nonInlineOperator, dnm);
    OperatorDeployInfo.InputDeployInfo idi = getInputDeployInfo(odi, nonInlineMeta.getMeta(niInputPort));
    id = nonInlineMeta.getName() + " " + idi.portName;
    Assert.assertEquals("number stream codecs " + id, idi.streamCodecs.size(), 1);
    checkPresentStreamCodec(nonInlineMeta, niInputPort, idi.streamCodecs, id, plan);
/*
    OperatorDeployInfo n3di = getSingleOperatorDeployInfo(node3, node3.getName(), dnm);

    OperatorDeployInfo.InputDeployInfo n3idi = getInputDeployInfo(n3di, n3meta.getMeta(node3.inport1));
    id = n3meta.getName() + " " + n3idi.portName;
    Assert.assertEquals("number stream codecs " + id, n3idi.streamCodecs.size(), 1);
    streamIdentifier.operName = n3meta.getName();
    streamIdentifier.portName = n3meta.getMeta(node3.inport1).getPortName();
    checkStreamCodecInfo(n3idi.streamCodecs, id, streamIdentifier, serDe2);
    */
}

Example 33 with PTOperator

use of com.datatorrent.stram.plan.physical.PTOperator in project apex-core by apache.

the class ResourceRequestHandler method getHost.

public String getHost(ContainerStartRequest csr, boolean first) {
    String host = null;
    PTContainer c = csr.container;
    if (first) {
        for (PTOperator oper : c.getOperators()) {
            HostOperatorSet grpObj = oper.getNodeLocalOperators();
            host = nodeLocalMapping.get(grpObj.getOperatorSet());
            if (host != null) {
                antiAffinityMapping.put(c, host);
                return host;
            }
            if (grpObj.getHost() != null) {
                host = grpObj.getHost();
                // using the 1st host value as host for container
                break;
            }
        }
        if (host != null && nodeReportMap.get(host) != null) {
            for (PTOperator oper : c.getOperators()) {
                HostOperatorSet grpObj = oper.getNodeLocalOperators();
                Set<PTOperator> nodeLocalSet = grpObj.getOperatorSet();
                NodeReport report = nodeReportMap.get(host);
                int aggrMemory = c.getRequiredMemoryMB();
                int vCores = c.getRequiredVCores();
                Set<PTContainer> containers = Sets.newHashSet();
                containers.add(c);
                for (PTOperator nodeLocalOper : nodeLocalSet) {
                    if (!containers.contains(nodeLocalOper.getContainer())) {
                        aggrMemory += nodeLocalOper.getContainer().getRequiredMemoryMB();
                        vCores += nodeLocalOper.getContainer().getRequiredVCores();
                        containers.add(nodeLocalOper.getContainer());
                    }
                }
                int memAvailable = report.getCapability().getMemory() - report.getUsed().getMemory();
                int vCoresAvailable = report.getCapability().getVirtualCores() - report.getUsed().getVirtualCores();
                if (memAvailable >= aggrMemory && vCoresAvailable >= vCores) {
                    nodeLocalMapping.put(nodeLocalSet, host);
                    antiAffinityMapping.put(c, host);
                    return host;
                }
            }
        }
    }
    // the host requested didn't have the resources so looking for other hosts
    host = null;
    List<String> antiHosts = new ArrayList<>();
    List<String> antiPreferredHosts = new ArrayList<>();
    if (!c.getStrictAntiPrefs().isEmpty()) {
        // Check if containers are allocated already for the anti-affinity containers
        populateAntiHostList(c, antiHosts);
    }
    if (!c.getPreferredAntiPrefs().isEmpty()) {
        populateAntiHostList(c, antiPreferredHosts);
    }
    LOG.info("Strict anti-affinity = {} for container with operators {}", antiHosts, StringUtils.join(c.getOperators(), ","));
    for (PTOperator oper : c.getOperators()) {
        HostOperatorSet grpObj = oper.getNodeLocalOperators();
        Set<PTOperator> nodeLocalSet = grpObj.getOperatorSet();
        if (nodeLocalSet.size() > 1 || !c.getStrictAntiPrefs().isEmpty() || !c.getPreferredAntiPrefs().isEmpty()) {
            LOG.info("Finding new host for {}", nodeLocalSet);
            int aggrMemory = c.getRequiredMemoryMB();
            int vCores = c.getRequiredVCores();
            Set<PTContainer> containers = Sets.newHashSet();
            containers.add(c);
            // aggregate memory required for all containers
            for (PTOperator nodeLocalOper : nodeLocalSet) {
                if (!containers.contains(nodeLocalOper.getContainer())) {
                    aggrMemory += nodeLocalOper.getContainer().getRequiredMemoryMB();
                    vCores += nodeLocalOper.getContainer().getRequiredVCores();
                    containers.add(nodeLocalOper.getContainer());
                }
            }
            host = assignHost(host, antiHosts, antiPreferredHosts, grpObj, nodeLocalSet, aggrMemory, vCores);
            if (host == null && !antiPreferredHosts.isEmpty() && !antiHosts.isEmpty()) {
                // Drop the preferred constraint and try allocation
                antiPreferredHosts.clear();
                host = assignHost(host, antiHosts, antiPreferredHosts, grpObj, nodeLocalSet, aggrMemory, vCores);
            }
            if (host != null) {
                antiAffinityMapping.put(c, host);
            } else {
                host = INVALID_HOST;
            }
        }
    }
    LOG.info("Found host {}", host);
    return host;
}

Example 34 with PTOperator

use of com.datatorrent.stram.plan.physical.PTOperator in project apex-core by apache.

the class LaunchContainerRunnable method getChildVMCommand.

/**
   * Build the command to launch the child VM in the container
   *
   * @param jvmID
   * @return
   */
public List<CharSequence> getChildVMCommand(String jvmID) {
    List<CharSequence> vargs = new ArrayList<>(8);
    if (!StringUtils.isBlank(System.getenv(Environment.JAVA_HOME.key()))) {
        // node manager provides JAVA_HOME
        vargs.add(Environment.JAVA_HOME.$() + "/bin/java");
    } else {
        vargs.add("java");
    }
    String jvmOpts = dag.getAttributes().get(LogicalPlan.CONTAINER_JVM_OPTIONS);
    if (jvmOpts == null) {
        if (dag.isDebug()) {
            vargs.add(JAVA_REMOTE_DEBUG_OPTS);
        }
    } else {
        Map<String, String> params = new HashMap<>();
        params.put("applicationId", Integer.toString(container.getId().getApplicationAttemptId().getApplicationId().getId()));
        params.put("containerId", Integer.toString(container.getId().getId()));
        StrSubstitutor sub = new StrSubstitutor(params, "%(", ")");
        vargs.add(sub.replace(jvmOpts));
        if (dag.isDebug() && !jvmOpts.contains("-agentlib:jdwp=")) {
            vargs.add(JAVA_REMOTE_DEBUG_OPTS);
        }
    }
    List<DAG.OperatorMeta> operatorMetaList = Lists.newArrayList();
    int bufferServerMemory = 0;
    for (PTOperator operator : sca.getContainer().getOperators()) {
        bufferServerMemory += operator.getBufferServerMemory();
        operatorMetaList.add(operator.getOperatorMeta());
    }
    Context.ContainerOptConfigurator containerOptConfigurator = dag.getAttributes().get(LogicalPlan.CONTAINER_OPTS_CONFIGURATOR);
    jvmOpts = containerOptConfigurator.getJVMOptions(operatorMetaList);
    jvmOpts = parseJvmOpts(jvmOpts, ((long) bufferServerMemory) * MB_TO_B);
    LOG.info("Jvm opts {} for container {}", jvmOpts, container.getId());
    vargs.add(jvmOpts);
    Path childTmpDir = new Path(Environment.PWD.$(), YarnConfiguration.DEFAULT_CONTAINER_TEMP_DIR);
    vargs.add(String.format("-D%s=%s", StreamingContainer.PROP_APP_PATH, dag.assertAppPath()));
    vargs.add("-Djava.io.tmpdir=" + childTmpDir);
    vargs.add(String.format("-D%scid=%s", StreamingApplication.DT_PREFIX, jvmID));
    vargs.add("-Dhadoop.root.logger=" + (dag.isDebug() ? "DEBUG" : "INFO") + ",RFA");
    vargs.add("-Dhadoop.log.dir=" + ApplicationConstants.LOG_DIR_EXPANSION_VAR);
    StramClientUtils.addAttributeToArgs(LogicalPlan.APPLICATION_NAME, dag, vargs);
    StramClientUtils.addAttributeToArgs(LogicalPlan.LOGGER_APPENDER, dag, vargs);
    String loggersLevel = System.getProperty(StramUtils.DT_LOGGERS_LEVEL);
    if (loggersLevel != null) {
        vargs.add(String.format("-D%s=%s", StramUtils.DT_LOGGERS_LEVEL, loggersLevel));
    }
    // Add main class and its arguments
    // main of Child
    vargs.add(StreamingContainer.class.getName());
    vargs.add("1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stdout");
    vargs.add("2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stderr");
    // Final commmand
    StringBuilder mergedCommand = new StringBuilder(256);
    for (CharSequence str : vargs) {
        mergedCommand.append(str).append(" ");
    }
    List<CharSequence> vargsFinal = new ArrayList<>(1);
    vargsFinal.add(mergedCommand.toString());
    return vargsFinal;
}

Example 35 with PTOperator

use of com.datatorrent.stram.plan.physical.PTOperator in project apex-core by apache.

the class StreamingContainerManagerTest method testProcessHeartbeat.

@Test
public void testProcessHeartbeat() throws Exception {
    TestGeneratorInputOperator o1 = dag.addOperator("o1", TestGeneratorInputOperator.class);
    dag.setOperatorAttribute(o1, OperatorContext.STATS_LISTENERS, Arrays.asList(new StatsListener[] { new PartitioningTest.PartitionLoadWatch() }));
    dag.setAttribute(OperatorContext.STORAGE_AGENT, new MemoryStorageAgent());
    StreamingContainerManager scm = new StreamingContainerManager(dag);
    PhysicalPlan plan = scm.getPhysicalPlan();
    Assert.assertEquals("number required containers", 1, plan.getContainers().size());
    PTOperator o1p1 = plan.getOperators(dag.getMeta(o1)).get(0);
    // assign container
    String containerId = "container1";
    StreamingContainerAgent sca = scm.assignContainer(new ContainerResource(0, containerId, "localhost", 512, 0, null), InetSocketAddress.createUnresolved("localhost", 0));
    Assert.assertNotNull(sca);
    Assert.assertEquals(PTContainer.State.ALLOCATED, o1p1.getContainer().getState());
    Assert.assertEquals(PTOperator.State.PENDING_DEPLOY, o1p1.getState());
    ContainerStats cstats = new ContainerStats(containerId);
    ContainerHeartbeat hb = new ContainerHeartbeat();
    hb.setContainerStats(cstats);
    // get deploy request
    ContainerHeartbeatResponse chr = scm.processHeartbeat(hb);
    Assert.assertNotNull(chr.deployRequest);
    Assert.assertEquals("" + chr.deployRequest, 1, chr.deployRequest.size());
    Assert.assertEquals(PTContainer.State.ACTIVE, o1p1.getContainer().getState());
    Assert.assertEquals("state " + o1p1, PTOperator.State.PENDING_DEPLOY, o1p1.getState());
    // first operator heartbeat
    OperatorHeartbeat ohb = new OperatorHeartbeat();
    ohb.setNodeId(o1p1.getId());
    ohb.setState(OperatorHeartbeat.DeployState.ACTIVE);
    OperatorStats stats = new OperatorStats();
    stats.checkpoint = new Checkpoint(2, 0, 0);
    stats.windowId = 3;
    stats.outputPorts = Lists.newArrayList();
    PortStats ps = new PortStats(TestGeneratorInputOperator.OUTPUT_PORT);
    ps.bufferServerBytes = 101;
    ps.tupleCount = 1;
    stats.outputPorts.add(ps);
    ohb.windowStats = Lists.newArrayList(stats);
    cstats.operators.add(ohb);
    // activate operator
    scm.processHeartbeat(hb);
    Assert.assertEquals(PTContainer.State.ACTIVE, o1p1.getContainer().getState());
    Assert.assertEquals("state " + o1p1, PTOperator.State.ACTIVE, o1p1.getState());
    Assert.assertEquals("tuples " + o1p1, 1, o1p1.stats.totalTuplesEmitted.get());
    Assert.assertEquals("tuples " + o1p1, 0, o1p1.stats.totalTuplesProcessed.get());
    Assert.assertEquals("window " + o1p1, 3, o1p1.stats.currentWindowId.get());
    Assert.assertEquals("port stats", 1, o1p1.stats.outputPortStatusList.size());
    PortStatus o1p1ps = o1p1.stats.outputPortStatusList.get(TestGeneratorInputOperator.OUTPUT_PORT);
    Assert.assertNotNull("port stats", o1p1ps);
    Assert.assertEquals("port stats", 1, o1p1ps.totalTuples);
    // second operator heartbeat
    stats = new OperatorStats();
    stats.checkpoint = new Checkpoint(2, 0, 0);
    stats.windowId = 4;
    stats.outputPorts = Lists.newArrayList();
    ps = new PortStats(TestGeneratorInputOperator.OUTPUT_PORT);
    ps.bufferServerBytes = 1;
    ps.tupleCount = 1;
    stats.outputPorts.add(ps);
    ohb.windowStats = Lists.newArrayList(stats);
    cstats.operators.clear();
    cstats.operators.add(ohb);
    scm.processHeartbeat(hb);
    Assert.assertEquals("tuples " + o1p1, 2, o1p1.stats.totalTuplesEmitted.get());
    Assert.assertEquals("window " + o1p1, 4, o1p1.stats.currentWindowId.get());
    Assert.assertEquals("statsQueue " + o1p1, 2, o1p1.stats.listenerStats.size());
    scm.processEvents();
    Assert.assertEquals("statsQueue " + o1p1, 0, o1p1.stats.listenerStats.size());
    Assert.assertEquals("lastStats " + o1p1, 2, o1p1.stats.lastWindowedStats.size());
}