Examples with PTContainer com.datatorrent.stram.plan.physical.PTContainer used on opensource projects

Example 11 with PTContainer

use of com.datatorrent.stram.plan.physical.PTContainer in project apex-core by apache.

the class StreamingAppMasterService method previouslyAllocatedContainers.

/**
   * Check for containers that were allocated in a previous attempt.
   * If the containers are still alive, wait for them to check in via heartbeat.
   */
private List<ContainerId> previouslyAllocatedContainers(List<Container> containersListByYarn) {
    List<ContainerId> containersToRelease = new ArrayList<>();
    if (containersListByYarn.size() != 0) {
        LOG.debug("Containers list by YARN - {}", containersListByYarn);
        LOG.debug("Containers list by Streaming Container Manger - {}", dnmgr.getPhysicalPlan().getContainers());
        Map<String, Container> fromYarn = new HashMap<>();
        for (Container container : containersListByYarn) {
            fromYarn.put(container.getId().toString(), container);
        }
        for (PTContainer ptContainer : dnmgr.getPhysicalPlan().getContainers()) {
            String containerId = ptContainer.getExternalId();
            // SCM starts the container without external ID.
            if (containerId == null) {
                continue;
            }
            Container container = fromYarn.get(containerId);
            if (container != null) {
                allocatedContainers.put(containerId, new AllocatedContainer(container));
                fromYarn.remove(containerId);
            } else {
                dnmgr.scheduleContainerRestart(containerId);
            }
        }
        for (Container container : fromYarn.values()) {
            containersToRelease.add(container.getId());
        }
        if (fromYarn.size() != 0) {
            LOG.info("Containers list returned by YARN, has the following container(s) which are not present in PhysicalPlan {}", fromYarn);
        }
    } else {
        dnmgr.deployAfterRestart();
    }
    return containersToRelease;
}

Example 12 with PTContainer

use of com.datatorrent.stram.plan.physical.PTContainer in project apex-core by apache.

the class StreamingContainerManager method processOperatorDeployStatus.

private void processOperatorDeployStatus(final PTOperator oper, OperatorHeartbeat ohb, StreamingContainerAgent sca) {
    OperatorHeartbeat.DeployState ds = null;
    if (ohb != null) {
        ds = ohb.getState();
    }
    LOG.debug("heartbeat {} {}/{} {}", oper, oper.getState(), ds, oper.getContainer().getExternalId());
    switch(oper.getState()) {
        case ACTIVE:
            // operator expected active, check remote status
            if (ds == null) {
                sca.deployOpers.add(oper);
            } else {
                switch(ds) {
                    case SHUTDOWN:
                        // schedule operator deactivation against the windowId
                        // will be processed once window is committed and all dependent operators completed processing
                        long windowId = oper.stats.currentWindowId.get();
                        if (ohb.windowStats != null && !ohb.windowStats.isEmpty()) {
                            windowId = ohb.windowStats.get(ohb.windowStats.size() - 1).windowId;
                        }
                        LOG.debug("Operator {} deactivated at window {}", oper, windowId);
                        synchronized (this.shutdownOperators) {
                            Set<PTOperator> deactivatedOpers = this.shutdownOperators.get(windowId);
                            if (deactivatedOpers == null) {
                                this.shutdownOperators.put(windowId, deactivatedOpers = new HashSet<>());
                            }
                            deactivatedOpers.add(oper);
                        }
                        oper.setState(State.INACTIVE);
                        sca.undeployOpers.add(oper.getId());
                        slowestUpstreamOp.remove(oper);
                        // record operator stop event
                        recordEventAsync(new StramEvent.StopOperatorEvent(oper.getName(), oper.getId(), oper.getContainer().getExternalId()));
                        break;
                    case FAILED:
                        processOperatorFailure(oper);
                        sca.undeployOpers.add(oper.getId());
                        slowestUpstreamOp.remove(oper);
                        recordEventAsync(new StramEvent.StopOperatorEvent(oper.getName(), oper.getId(), oper.getContainer().getExternalId()));
                        break;
                    case ACTIVE:
                    default:
                        break;
                }
            }
            break;
        case PENDING_UNDEPLOY:
            if (ds == null) {
                // operator no longer deployed in container
                recordEventAsync(new StramEvent.StopOperatorEvent(oper.getName(), oper.getId(), oper.getContainer().getExternalId()));
                oper.setState(State.PENDING_DEPLOY);
                sca.deployOpers.add(oper);
            } else {
                // operator is currently deployed, request undeploy
                sca.undeployOpers.add(oper.getId());
                slowestUpstreamOp.remove(oper);
            }
            break;
        case PENDING_DEPLOY:
            if (ds == null) {
                // operator to be deployed
                sca.deployOpers.add(oper);
            } else {
                // operator was deployed in container
                PTContainer container = oper.getContainer();
                LOG.debug("{} marking deployed: {} remote status {}", container.getExternalId(), oper, ds);
                oper.setState(PTOperator.State.ACTIVE);
                // reset on redeploy
                oper.stats.lastHeartbeat = null;
                oper.stats.lastWindowIdChangeTms = clock.getTime();
                recordEventAsync(new StramEvent.StartOperatorEvent(oper.getName(), oper.getId(), container.getExternalId()));
            }
            break;
        default:
            //LOG.warn("Unhandled operator state {} {} remote {}", oper, oper.getState(), ds);
            if (ds != null) {
                // operator was removed and needs to be undeployed from container
                sca.undeployOpers.add(oper.getId());
                slowestUpstreamOp.remove(oper);
                recordEventAsync(new StramEvent.StopOperatorEvent(oper.getName(), oper.getId(), oper.getContainer().getExternalId()));
            }
    }
}

Example 13 with PTContainer

use of com.datatorrent.stram.plan.physical.PTContainer in project apex-core by apache.

the class LogicalPlanConfigurationTest method testModuleUnifierLevelAttributes.

@Test
@SuppressWarnings({ "UnnecessaryBoxing", "AssertEqualsBetweenInconvertibleTypes" })
public void testModuleUnifierLevelAttributes() {
    class DummyOperator extends BaseOperator {

        int prop;

        public transient DefaultInputPort<Integer> input = new DefaultInputPort<Integer>() {

            @Override
            public void process(Integer tuple) {
                LOG.debug(tuple.intValue() + " processed");
                output.emit(tuple);
            }
        };

        public transient DefaultOutputPort<Integer> output = new DefaultOutputPort<>();
    }
    class DummyOutputOperator extends BaseOperator {

        int prop;

        public transient DefaultInputPort<Integer> input = new DefaultInputPort<Integer>() {

            @Override
            public void process(Integer tuple) {
                LOG.debug(tuple.intValue() + " processed");
            }
        };
    }
    class TestUnifierAttributeModule implements Module {

        public transient ProxyInputPort<Integer> moduleInput = new ProxyInputPort<>();

        public transient ProxyOutputPort<Integer> moduleOutput = new Module.ProxyOutputPort<>();

        @Override
        public void populateDAG(DAG dag, Configuration conf) {
            DummyOperator dummyOperator = dag.addOperator("DummyOperator", new DummyOperator());
            dag.setOperatorAttribute(dummyOperator, Context.OperatorContext.PARTITIONER, new StatelessPartitioner<DummyOperator>(3));
            dag.setUnifierAttribute(dummyOperator.output, OperatorContext.TIMEOUT_WINDOW_COUNT, 2);
            moduleInput.set(dummyOperator.input);
            moduleOutput.set(dummyOperator.output);
        }
    }
    StreamingApplication app = new StreamingApplication() {

        @Override
        public void populateDAG(DAG dag, Configuration conf) {
            Module m1 = dag.addModule("TestModule", new TestUnifierAttributeModule());
            DummyOutputOperator dummyOutputOperator = dag.addOperator("DummyOutputOperator", new DummyOutputOperator());
            dag.addStream("Module To Operator", ((TestUnifierAttributeModule) m1).moduleOutput, dummyOutputOperator.input);
        }
    };
    String appName = "UnifierApp";
    LogicalPlanConfiguration dagBuilder = new LogicalPlanConfiguration(new Configuration(false));
    LogicalPlan dag = new LogicalPlan();
    dag.setAttribute(Context.OperatorContext.STORAGE_AGENT, new MockStorageAgent());
    dagBuilder.prepareDAG(dag, app, appName);
    LogicalPlan.OperatorMeta ometa = dag.getOperatorMeta("TestModule$DummyOperator");
    LogicalPlan.OperatorMeta om = null;
    for (Map.Entry<LogicalPlan.OutputPortMeta, LogicalPlan.StreamMeta> entry : ometa.getOutputStreams().entrySet()) {
        if (entry.getKey().getPortName().equals("output")) {
            om = entry.getKey().getUnifierMeta();
        }
    }
    /*
     * Verify the attribute value after preparing DAG.
     */
    Assert.assertNotNull(om);
    Assert.assertEquals("", Integer.valueOf(2), om.getValue(Context.OperatorContext.TIMEOUT_WINDOW_COUNT));
    PhysicalPlan plan = new PhysicalPlan(dag, new TestPlanContext());
    List<PTContainer> containers = plan.getContainers();
    LogicalPlan.OperatorMeta operatorMeta = null;
    for (PTContainer container : containers) {
        List<PTOperator> operators = container.getOperators();
        for (PTOperator operator : operators) {
            if (operator.isUnifier()) {
                operatorMeta = operator.getOperatorMeta();
            }
        }
    }
    /*
     * Verify attribute after physical plan creation with partitioned operators.
     */
    Assert.assertEquals("", Integer.valueOf(2), operatorMeta.getValue(OperatorContext.TIMEOUT_WINDOW_COUNT));
}

Example 14 with PTContainer

use of com.datatorrent.stram.plan.physical.PTContainer in project apex-core by apache.

the class StramRecoveryTest method testWriteAheadLog.

@Test
public void testWriteAheadLog() throws Exception {
    final MutableInt flushCount = new MutableInt();
    final MutableBoolean isClosed = new MutableBoolean(false);
    dag.setAttribute(OperatorContext.STORAGE_AGENT, new FSStorageAgent(testMeta.getPath(), null));
    TestGeneratorInputOperator o1 = dag.addOperator("o1", TestGeneratorInputOperator.class);
    StreamingContainerManager scm = new StreamingContainerManager(dag);
    PhysicalPlan plan = scm.getPhysicalPlan();
    Journal j = scm.getJournal();
    ByteArrayOutputStream bos = new ByteArrayOutputStream() {

        @Override
        public void flush() throws IOException {
            super.flush();
            flushCount.increment();
        }

        @Override
        public void close() throws IOException {
            super.close();
            isClosed.setValue(true);
        }
    };
    j.setOutputStream(new DataOutputStream(bos));
    PTOperator o1p1 = plan.getOperators(dag.getMeta(o1)).get(0);
    assertEquals(PTOperator.State.PENDING_DEPLOY, o1p1.getState());
    String externalId = new MockContainer(scm, o1p1.getContainer()).container.getExternalId();
    assertEquals("flush count", 1, flushCount.intValue());
    o1p1.setState(PTOperator.State.ACTIVE);
    assertEquals(PTOperator.State.ACTIVE, o1p1.getState());
    assertEquals("flush count", 2, flushCount.intValue());
    assertEquals("is closed", false, isClosed.booleanValue());
    // this will close the stream. There are 2 calls to flush() during the close() - one in Kryo Output and one
    // in FilterOutputStream
    j.setOutputStream(null);
    assertEquals("flush count", 4, flushCount.intValue());
    assertEquals("is closed", true, isClosed.booleanValue());
    // output stream is closed, so state will be changed without recording it in the journal
    o1p1.setState(PTOperator.State.INACTIVE);
    assertEquals(PTOperator.State.INACTIVE, o1p1.getState());
    assertEquals("flush count", 4, flushCount.intValue());
    ByteArrayInputStream bis = new ByteArrayInputStream(bos.toByteArray());
    j.replay(new DataInputStream(bis));
    assertEquals(PTOperator.State.ACTIVE, o1p1.getState());
    InetSocketAddress addr1 = InetSocketAddress.createUnresolved("host1", 1);
    PTContainer c1 = plan.getContainers().get(0);
    c1.setState(PTContainer.State.ALLOCATED);
    c1.host = "host1";
    c1.bufferServerAddress = addr1;
    c1.setAllocatedMemoryMB(2);
    c1.setRequiredMemoryMB(1);
    c1.setAllocatedVCores(3);
    c1.setRequiredVCores(4);
    j.setOutputStream(new DataOutputStream(bos));
    j.write(c1.getSetContainerState());
    c1.setExternalId(null);
    c1.setState(PTContainer.State.NEW);
    c1.setExternalId(null);
    c1.host = null;
    c1.bufferServerAddress = null;
    bis = new ByteArrayInputStream(bos.toByteArray());
    j.replay(new DataInputStream(bis));
    assertEquals(externalId, c1.getExternalId());
    assertEquals(PTContainer.State.ALLOCATED, c1.getState());
    assertEquals("host1", c1.host);
    assertEquals(addr1, c1.bufferServerAddress);
    assertEquals(1, c1.getRequiredMemoryMB());
    assertEquals(2, c1.getAllocatedMemoryMB());
    assertEquals(3, c1.getAllocatedVCores());
    assertEquals(4, c1.getRequiredVCores());
    j.write(scm.getSetOperatorProperty("o1", "maxTuples", "100"));
    o1.setMaxTuples(10);
    j.setOutputStream(null);
    bis = new ByteArrayInputStream(bos.toByteArray());
    j.replay(new DataInputStream(bis));
    assertEquals(100, o1.getMaxTuples());
    j.setOutputStream(new DataOutputStream(bos));
    scm.setOperatorProperty("o1", "maxTuples", "10");
    assertEquals(10, o1.getMaxTuples());
    o1.setMaxTuples(100);
    assertEquals(100, o1.getMaxTuples());
    j.setOutputStream(null);
    bis = new ByteArrayInputStream(bos.toByteArray());
    j.replay(new DataInputStream(bis));
    assertEquals(10, o1.getMaxTuples());
    j.setOutputStream(new DataOutputStream(bos));
    scm.setPhysicalOperatorProperty(o1p1.getId(), "maxTuples", "50");
}

Example 15 with PTContainer

use of com.datatorrent.stram.plan.physical.PTContainer in project apex-core by apache.

the class StreamCodecTest method testStreamCodecReuse.

@Test
public void testStreamCodecReuse() {
    GenericTestOperator node1 = dag.addOperator("node1", GenericTestOperator.class);
    GenericTestOperator node2 = dag.addOperator("node2", GenericTestOperator.class);
    GenericTestOperator node3 = dag.addOperator("node3", GenericTestOperator.class);
    GenericTestOperator node4 = dag.addOperator("node4", GenericTestOperator.class);
    TestStreamCodec serDe = new TestStreamCodec();
    dag.setInputPortAttribute(node4.inport1, Context.PortContext.STREAM_CODEC, serDe);
    GenericTestOperator node5 = dag.addOperator("node5", GenericTestOperator.class);
    dag.setInputPortAttribute(node5.inport1, Context.PortContext.STREAM_CODEC, serDe);
    GenericTestOperator node6 = dag.addOperator("node6", GenericTestOperator.class);
    serDe = new TestStreamCodec();
    dag.setInputPortAttribute(node6.inport1, Context.PortContext.STREAM_CODEC, serDe);
    dag.addStream("n1n2", node1.outport1, node2.inport1);
    dag.addStream("n2n3", node2.outport1, node3.inport1);
    dag.addStream("n3n4", node3.outport1, node4.inport1);
    dag.addStream("n4n5", node4.outport1, node5.inport1);
    dag.addStream("n5n6", node5.outport1, node6.inport1);
    dag.setAttribute(LogicalPlan.CONTAINERS_MAX_COUNT, Integer.MAX_VALUE);
    StramTestSupport.MemoryStorageAgent msa = new StramTestSupport.MemoryStorageAgent();
    dag.setAttribute(Context.OperatorContext.STORAGE_AGENT, msa);
    StreamingContainerManager dnm = new StreamingContainerManager(dag);
    PhysicalPlan plan = dnm.getPhysicalPlan();
    List<PTContainer> containers = plan.getContainers();
    Assert.assertEquals("number containers", 6, containers.size());
    for (int i = 0; i < containers.size(); ++i) {
        StreamingContainerManagerTest.assignContainer(dnm, "container" + (i + 1));
    }
    getSingleOperatorDeployInfo(node1, dnm);
    getSingleOperatorDeployInfo(node2, dnm);
    getSingleOperatorDeployInfo(node3, dnm);
    getSingleOperatorDeployInfo(node4, dnm);
    getSingleOperatorDeployInfo(node5, dnm);
    getSingleOperatorDeployInfo(node6, dnm);
    Assert.assertEquals("number of stream codec identifiers", 3, plan.getStreamCodecIdentifiers().size());
}