Examples with PTOperator com.datatorrent.stram.plan.physical.PTOperator used on opensource projects

Example 1 with PTOperator

use of com.datatorrent.stram.plan.physical.PTOperator in project apex-core by apache.

the class StreamPersistanceTests method testDynamicPartitioning.

@Test
public void testDynamicPartitioning() throws ClassNotFoundException, IOException {
    AscendingNumbersOperator ascend = dag.addOperator("ascend", new AscendingNumbersOperator());
    final TestReceiverOperator console = dag.addOperator("console", new TestReceiverOperator());
    dag.setOperatorAttribute(console, Context.OperatorContext.PARTITIONER, new StatelessPartitioner<TestReceiverOperator>(2));
    dag.setOperatorAttribute(console, Context.OperatorContext.STATS_LISTENERS, Lists.newArrayList((StatsListener) new PartitioningTest.PartitionLoadWatch()));
    final PartitionedTestPersistanceOperator console1 = new PartitionedTestPersistanceOperator();
    StreamMeta s = dag.addStream("Stream1", ascend.outputPort, console.inport);
    dag.setInputPortAttribute(console.inport, PortContext.STREAM_CODEC, new TestPartitionCodec());
    s.persistUsing("persister", console1, console1.inport);
    dag.setAttribute(LogicalPlan.CONTAINERS_MAX_COUNT, Integer.MAX_VALUE);
    StramTestSupport.MemoryStorageAgent msa = new StramTestSupport.MemoryStorageAgent();
    dag.setAttribute(Context.OperatorContext.STORAGE_AGENT, msa);
    StreamingContainerManager dnm = new StreamingContainerManager(dag);
    PhysicalPlan plan = dnm.getPhysicalPlan();
    List<PTContainer> containers = plan.getContainers();
    Assert.assertEquals("number containers", 4, containers.size());
    for (int i = 0; i < containers.size(); ++i) {
        StreamingContainerManagerTest.assignContainer(dnm, "container" + (i + 1));
    }
    LogicalPlan.OperatorMeta passThruMeta = dag.getMeta(console);
    List<PTOperator> ptos = plan.getOperators(passThruMeta);
    PTOperator persistOperatorContainer = null;
    for (PTContainer container : plan.getContainers()) {
        for (PTOperator operator : container.getOperators()) {
            operator.setState(PTOperator.State.ACTIVE);
            if (operator.getName().equals("persister")) {
                persistOperatorContainer = operator;
            }
        }
    }
    // Check that persist operator is part of dependents redeployed
    Set<PTOperator> operators = plan.getDependents(ptos);
    logger.debug("Operators to be re-deployed = {}", operators);
    // Validate that persist operator is part of dependents
    assertTrue("persist operator should be part of the operators to be redeployed", operators.contains(persistOperatorContainer));
    LogicalPlan.StreamMeta s1 = (LogicalPlan.StreamMeta) s;
    StreamCodec codec = s1.getPersistOperatorInputPort().getStreamCodec();
    assertEquals("Codec should be instance of StreamCodecWrapper", codec instanceof StreamCodecWrapperForPersistance, true);
    StreamCodecWrapperForPersistance wrapperCodec = (StreamCodecWrapperForPersistance) codec;
    Entry<InputPortMeta, Collection<PartitionKeys>> keys = (Entry<InputPortMeta, Collection<PartitionKeys>>) wrapperCodec.inputPortToPartitionMap.entrySet().iterator().next();
    logger.debug(keys.toString());
    assertEquals("Size of partitions should be 2", 2, keys.getValue().size());
    for (PTOperator ptOperator : ptos) {
        PartitioningTest.PartitionLoadWatch.put(ptOperator, -1);
        plan.onStatusUpdate(ptOperator);
    }
    dnm.processEvents();
    assertEquals("Input port map", wrapperCodec.inputPortToPartitionMap.size(), 1);
    keys = (Entry<InputPortMeta, Collection<PartitionKeys>>) wrapperCodec.inputPortToPartitionMap.entrySet().iterator().next();
    assertEquals("Size of partitions should be 1 after repartition", 1, keys.getValue().size());
    logger.debug(keys.toString());
}

Example 2 with PTOperator

use of com.datatorrent.stram.plan.physical.PTOperator in project apex-core by apache.

the class LocalityTest method testNodeLocal.

@Test
public void testNodeLocal() {
    LogicalPlan dag = new LogicalPlan();
    dag.getAttributes().put(com.datatorrent.api.Context.DAGContext.APPLICATION_PATH, new File("target", LocalityTest.class.getName()).getAbsolutePath());
    dag.setAttribute(OperatorContext.STORAGE_AGENT, new MemoryStorageAgent());
    GenericTestOperator o1 = dag.addOperator("o1", GenericTestOperator.class);
    GenericTestOperator partitioned = dag.addOperator("partitioned", GenericTestOperator.class);
    dag.getMeta(partitioned).getAttributes().put(OperatorContext.PARTITIONER, new StatelessPartitioner<GenericTestOperator>(2));
    GenericTestOperator partitionedParallel = dag.addOperator("partitionedParallel", GenericTestOperator.class);
    dag.addStream("o1_outport1", o1.outport1, partitioned.inport1).setLocality(null);
    dag.addStream("partitioned_outport1", partitioned.outport1, partitionedParallel.inport2).setLocality(Locality.NODE_LOCAL);
    dag.setInputPortAttribute(partitionedParallel.inport2, PortContext.PARTITION_PARALLEL, true);
    GenericTestOperator single = dag.addOperator("single", GenericTestOperator.class);
    dag.addStream("partitionedParallel_outport1", partitionedParallel.outport1, single.inport1);
    int maxContainers = 7;
    dag.setAttribute(LogicalPlan.CONTAINERS_MAX_COUNT, maxContainers);
    StreamingContainerManager scm = new StreamingContainerManager(dag);
    Assert.assertEquals("number required containers", 6, scm.containerStartRequests.size());
    ResourceRequestHandler rr = new ResourceRequestHandler();
    int containerMem = 2000;
    Map<String, NodeReport> nodeReports = Maps.newHashMap();
    NodeReport nr = BuilderUtils.newNodeReport(BuilderUtils.newNodeId("host1", 0), NodeState.RUNNING, "httpAddress", "rackName", BuilderUtils.newResource(0, 0), BuilderUtils.newResource(containerMem * 2, 2), 0, null, 0);
    nodeReports.put(nr.getNodeId().getHost(), nr);
    nr = BuilderUtils.newNodeReport(BuilderUtils.newNodeId("host2", 0), NodeState.RUNNING, "httpAddress", "rackName", BuilderUtils.newResource(0, 0), BuilderUtils.newResource(containerMem * 2, 2), 0, null, 0);
    nodeReports.put(nr.getNodeId().getHost(), nr);
    // set resources
    rr.updateNodeReports(Lists.newArrayList(nodeReports.values()));
    Map<PTContainer, String> requestedHosts = Maps.newHashMap();
    for (ContainerStartRequest csr : scm.containerStartRequests) {
        String host = rr.getHost(csr, true);
        csr.container.host = host;
        // update the node report
        if (host != null) {
            requestedHosts.put(csr.container, host);
            nr = nodeReports.get(host);
            nr.getUsed().setMemory(nr.getUsed().getMemory() + containerMem);
        }
    }
    Assert.assertEquals("" + requestedHosts, nodeReports.keySet(), Sets.newHashSet(requestedHosts.values()));
    for (Map.Entry<PTContainer, String> e : requestedHosts.entrySet()) {
        for (PTOperator oper : e.getKey().getOperators()) {
            if (oper.getNodeLocalOperators().getOperatorSet().size() > 1) {
                String expHost = null;
                for (PTOperator nodeLocalOper : oper.getNodeLocalOperators().getOperatorSet()) {
                    Assert.assertNotNull("host null " + nodeLocalOper.getContainer(), nodeLocalOper.getContainer().host);
                    if (expHost == null) {
                        expHost = nodeLocalOper.getContainer().host;
                    } else {
                        Assert.assertEquals("expected same host " + nodeLocalOper, expHost, nodeLocalOper.getContainer().host);
                    }
                }
            }
        }
    }
}

Example 3 with PTOperator

use of com.datatorrent.stram.plan.physical.PTOperator in project apex-core by apache.

the class AffinityRulesTest method testOperatorPartitionsAntiAffinity.

@Test
public void testOperatorPartitionsAntiAffinity() {
    LogicalPlan dag = new LogicalPlan();
    TestGeneratorInputOperator o1 = dag.addOperator("O1", new TestGeneratorInputOperator());
    GenericTestOperator o2 = dag.addOperator("O2", new GenericTestOperator());
    GenericTestOperator o3 = dag.addOperator("O3", new GenericTestOperator());
    dag.addStream("stream1", o1.outport, o2.inport1);
    dag.addStream("stream2", o2.outport1, o3.inport1);
    dag.setOperatorAttribute(o2, OperatorContext.PARTITIONER, new StatelessPartitioner<GenericTestOperator>(5));
    AffinityRulesSet ruleSet = new AffinityRulesSet();
    // Valid case:
    List<AffinityRule> rules = new ArrayList<>();
    ruleSet.setAffinityRules(rules);
    AffinityRule rule1 = new AffinityRule(Type.ANTI_AFFINITY, Locality.NODE_LOCAL, false, "O2", "O2");
    rules.add(rule1);
    dag.setAttribute(DAGContext.AFFINITY_RULES_SET, ruleSet);
    dag.validate();
    dag.getAttributes().put(com.datatorrent.api.Context.DAGContext.APPLICATION_PATH, testMeta.getAbsolutePath());
    dag.setAttribute(OperatorContext.STORAGE_AGENT, new MemoryStorageAgent());
    StreamingContainerManager scm = new StreamingContainerManager(dag);
    for (ContainerStartRequest csr : scm.containerStartRequests) {
        PTContainer container = csr.container;
        if (container.getOperators().get(0).getName().equals("O2")) {
            Assert.assertEquals("Anti-affinity containers set should have 4 containers for other partitions ", 4, container.getStrictAntiPrefs().size());
            for (PTContainer c : container.getStrictAntiPrefs()) {
                for (PTOperator operator : c.getOperators()) {
                    Assert.assertEquals("Partion for O2 should be Anti Prefs", "O2", operator.getName());
                }
            }
        }
    }
    // Check resource handler assigns different hosts for each partition
    ResourceRequestHandler rr = new ResourceRequestHandler();
    int containerMem = 1000;
    Map<String, NodeReport> nodeReports = Maps.newHashMap();
    for (int i = 0; i < 10; i++) {
        String hostName = "host" + i;
        NodeReport nr = BuilderUtils.newNodeReport(BuilderUtils.newNodeId(hostName, 0), NodeState.RUNNING, "httpAddress", "rackName", BuilderUtils.newResource(0, 0), BuilderUtils.newResource(containerMem * 2, 2), 0, null, 0);
        nodeReports.put(nr.getNodeId().getHost(), nr);
    }
    // set resources
    rr.updateNodeReports(Lists.newArrayList(nodeReports.values()));
    Set<String> partitionHostNames = new HashSet<>();
    for (ContainerStartRequest csr : scm.containerStartRequests) {
        String host = rr.getHost(csr, true);
        csr.container.host = host;
        if (csr.container.getOperators().get(0).getName().equals("O2")) {
            Assert.assertNotNull("Host name should not be null", host);
            LOG.info("Partition {} for operator O2 has host = {} ", csr.container.getId(), host);
            Assert.assertTrue("Each Partition should have a different host", !partitionHostNames.contains(host));
            partitionHostNames.add(host);
        }
    }
}

Example 4 with PTOperator

use of com.datatorrent.stram.plan.physical.PTOperator in project apex-core by apache.

the class CheckpointTest method testBlockedOperatorContainerRestart.

@Test
public void testBlockedOperatorContainerRestart() {
    MockClock clock = new MockClock();
    dag.setAttribute(com.datatorrent.api.Context.OperatorContext.STORAGE_AGENT, new MemoryStorageAgent());
    GenericTestOperator o1 = dag.addOperator("o1", GenericTestOperator.class);
    dag.setOperatorAttribute(o1, OperatorContext.TIMEOUT_WINDOW_COUNT, 2);
    GenericTestOperator o2 = dag.addOperator("o2", GenericTestOperator.class);
    dag.addStream("o1.outport1", o1.outport1, o2.inport1);
    StreamingContainerManager scm = new StreamingContainerManager(dag, false, clock);
    PhysicalPlan plan = scm.getPhysicalPlan();
    List<PTContainer> containers = plan.getContainers();
    Assert.assertEquals("Number of containers", 2, containers.size());
    Map<PTContainer, MockContainer> mockContainers = Maps.newHashMap();
    // allocate/assign all containers
    for (PTContainer c : containers) {
        MockContainer mc = new MockContainer(scm, c);
        mockContainers.put(c, mc);
    }
    // deploy all containers
    for (MockContainer mc : mockContainers.values()) {
        mc.deploy();
    }
    PTOperator o1p1 = plan.getOperators(dag.getMeta(o1)).get(0);
    MockContainer mc1 = mockContainers.get(o1p1.getContainer());
    MockOperatorStats o1p1mos = mockContainers.get(o1p1.getContainer()).stats(o1p1.getId());
    o1p1mos.currentWindowId(1).deployState(DeployState.ACTIVE);
    clock.time = 10;
    mc1.sendHeartbeat();
    Assert.assertEquals(clock.time, o1p1.stats.lastWindowIdChangeTms);
    Assert.assertEquals(1, o1p1.stats.currentWindowId.get());
    Assert.assertEquals(PTOperator.State.ACTIVE, o1p1.getState());
    int timeoutMs = dag.getMeta(o1).getValue(OperatorContext.TIMEOUT_WINDOW_COUNT) * dag.getValue(DAG.STREAMING_WINDOW_SIZE_MILLIS);
    Assert.assertEquals("processing timeout", timeoutMs, o1p1.stats.windowProcessingTimeoutMillis);
    clock.time += timeoutMs;
    mc1.sendHeartbeat();
    Assert.assertEquals(PTOperator.State.ACTIVE, o1p1.getState());
    Assert.assertEquals(10, o1p1.stats.lastWindowIdChangeTms);
    scm.monitorHeartbeat(false);
    Assert.assertTrue(scm.containerStopRequests.isEmpty());
    clock.time++;
    mc1.sendHeartbeat();
    Assert.assertEquals(PTOperator.State.ACTIVE, o1p1.getState());
    scm.monitorHeartbeat(false);
    Assert.assertTrue(scm.containerStopRequests.containsKey(o1p1.getContainer().getExternalId()));
}

Example 5 with PTOperator

use of com.datatorrent.stram.plan.physical.PTOperator in project apex-core by apache.

the class CheckpointTest method testBackup.

/**
   * Test saving of operator state at window boundary.
   * @throws Exception
   */
@Test
public void testBackup() throws Exception {
    AsyncFSStorageAgent storageAgent = new AsyncFSStorageAgent(testMeta.getPath(), null);
    storageAgent.setSyncCheckpoint(true);
    dag.setAttribute(OperatorContext.STORAGE_AGENT, storageAgent);
    dag.setAttribute(LogicalPlan.CHECKPOINT_WINDOW_COUNT, 1);
    dag.setAttribute(LogicalPlan.HEARTBEAT_INTERVAL_MILLIS, 50);
    dag.setAttribute(LogicalPlan.CONTAINERS_MAX_COUNT, 1);
    dag.setAttribute(LogicalPlan.STREAMING_WINDOW_SIZE_MILLIS, 50);
    MockInputOperator o1 = dag.addOperator("o1", new MockInputOperator());
    GenericTestOperator o2 = dag.addOperator("o2", GenericTestOperator.class);
    dag.setOperatorAttribute(o2, OperatorContext.STATELESS, true);
    dag.addStream("o1.outport", o1.outport, o2.inport1).setLocality(Locality.CONTAINER_LOCAL);
    StramLocalCluster sc = new StramLocalCluster(dag);
    sc.setHeartbeatMonitoringEnabled(false);
    sc.run();
    StreamingContainerManager dnm = sc.dnmgr;
    PhysicalPlan plan = dnm.getPhysicalPlan();
    Assert.assertEquals("number required containers", 1, dnm.getPhysicalPlan().getContainers().size());
    PTOperator o1p1 = plan.getOperators(dag.getMeta(o1)).get(0);
    Set<Long> checkpoints = Sets.newHashSet();
    for (long windowId : storageAgent.getWindowIds(o1p1.getId())) {
        checkpoints.add(windowId);
    }
    Assert.assertEquals("number checkpoints " + checkpoints, 3, checkpoints.size());
    Assert.assertTrue("contains " + checkpoints + " " + Stateless.WINDOW_ID, checkpoints.contains(Stateless.WINDOW_ID));
    PTOperator o2p1 = plan.getOperators(dag.getMeta(o2)).get(0);
    checkpoints = Sets.newHashSet();
    for (long windowId : storageAgent.getWindowIds(o2p1.getId())) {
        checkpoints.add(windowId);
    }
    Assert.assertEquals("number checkpoints " + checkpoints, 1, checkpoints.size());
    Assert.assertEquals("checkpoints " + o2p1, Sets.newHashSet(Stateless.WINDOW_ID), checkpoints);
    Assert.assertEquals("checkpoints " + o1p1 + " " + o1p1.checkpoints, 2, o1p1.checkpoints.size());
    Assert.assertNotNull("checkpoint not null for statefull operator " + o1p1, o1p1.stats.checkpointStats);
    for (Checkpoint cp : o1p1.checkpoints) {
        Object load = storageAgent.load(o1p1.getId(), cp.windowId);
        Assert.assertEquals("Stored Operator and Saved State", load.getClass(), o1p1.getOperatorMeta().getOperator().getClass());
    }
}