Examples with TestGeneratorInputOperator com.datatorrent.stram.engine.TestGeneratorInputOperator used on opensource projects

Example 6 with TestGeneratorInputOperator

use of com.datatorrent.stram.engine.TestGeneratorInputOperator in project apex-core by apache.

the class StreamingContainerManagerTest method testLatency.

@Test
public void testLatency() throws Exception {
    TestGeneratorInputOperator o1 = dag.addOperator("o1", TestGeneratorInputOperator.class);
    GenericTestOperator o2 = dag.addOperator("o2", GenericTestOperator.class);
    HighLatencyTestOperator o3 = dag.addOperator("o3", HighLatencyTestOperator.class);
    GenericTestOperator o4 = dag.addOperator("o4", GenericTestOperator.class);
    // 5 seconds
    long latency = 5000;
    o3.setLatency(latency);
    dag.addStream("o1.outport", o1.outport, o2.inport1, o3.inport1);
    dag.addStream("o2.outport1", o2.outport1, o4.inport1);
    dag.addStream("o3.outport1", o3.outport1, o4.inport2);
    // 1 second
    dag.setAttribute(Context.DAGContext.STATS_MAX_ALLOWABLE_WINDOWS_LAG, 2);
    StramLocalCluster lc = new StramLocalCluster(dag);
    StreamingContainerManager dnmgr = lc.dnmgr;
    lc.runAsync();
    Thread.sleep(10000);
    LogicalOperatorInfo o1Info = dnmgr.getLogicalOperatorInfo("o1");
    LogicalOperatorInfo o2Info = dnmgr.getLogicalOperatorInfo("o2");
    LogicalOperatorInfo o3Info = dnmgr.getLogicalOperatorInfo("o3");
    LogicalOperatorInfo o4Info = dnmgr.getLogicalOperatorInfo("o4");
    Assert.assertEquals("Input operator latency must be zero", 0, o1Info.latencyMA);
    Assert.assertTrue("Latency must be greater than or equal to zero", o2Info.latencyMA >= 0);
    Assert.assertTrue("Actual latency must be greater than the artificially introduced latency", o3Info.latencyMA > latency);
    Assert.assertTrue("Latency must be greater than or equal to zero", o4Info.latencyMA >= 0);
    StreamingContainerManager.CriticalPathInfo criticalPathInfo = dnmgr.getCriticalPathInfo();
    Assert.assertArrayEquals("Critical Path must be the path in the DAG that includes the HighLatencyTestOperator", new Integer[] { o1Info.partitions.iterator().next(), o3Info.partitions.iterator().next(), o4Info.partitions.iterator().next() }, criticalPathInfo.path.toArray());
    Assert.assertTrue("Whole DAG latency must be greater than the artificially introduced latency", criticalPathInfo.latency > latency);
    lc.shutdown();
}

Example 7 with TestGeneratorInputOperator

use of com.datatorrent.stram.engine.TestGeneratorInputOperator in project apex-core by apache.

the class AffinityRulesTest method testOperatorPartitionsAntiAffinity.

@Test
public void testOperatorPartitionsAntiAffinity() {
    LogicalPlan dag = new LogicalPlan();
    TestGeneratorInputOperator o1 = dag.addOperator("O1", new TestGeneratorInputOperator());
    GenericTestOperator o2 = dag.addOperator("O2", new GenericTestOperator());
    GenericTestOperator o3 = dag.addOperator("O3", new GenericTestOperator());
    dag.addStream("stream1", o1.outport, o2.inport1);
    dag.addStream("stream2", o2.outport1, o3.inport1);
    dag.setOperatorAttribute(o2, OperatorContext.PARTITIONER, new StatelessPartitioner<GenericTestOperator>(5));
    AffinityRulesSet ruleSet = new AffinityRulesSet();
    // Valid case:
    List<AffinityRule> rules = new ArrayList<>();
    ruleSet.setAffinityRules(rules);
    AffinityRule rule1 = new AffinityRule(Type.ANTI_AFFINITY, Locality.NODE_LOCAL, false, "O2", "O2");
    rules.add(rule1);
    dag.setAttribute(DAGContext.AFFINITY_RULES_SET, ruleSet);
    dag.validate();
    dag.getAttributes().put(com.datatorrent.api.Context.DAGContext.APPLICATION_PATH, testMeta.getAbsolutePath());
    dag.setAttribute(OperatorContext.STORAGE_AGENT, new MemoryStorageAgent());
    StreamingContainerManager scm = new StreamingContainerManager(dag);
    for (ContainerStartRequest csr : scm.containerStartRequests) {
        PTContainer container = csr.container;
        if (container.getOperators().get(0).getName().equals("O2")) {
            Assert.assertEquals("Anti-affinity containers set should have 4 containers for other partitions ", 4, container.getStrictAntiPrefs().size());
            for (PTContainer c : container.getStrictAntiPrefs()) {
                for (PTOperator operator : c.getOperators()) {
                    Assert.assertEquals("Partion for O2 should be Anti Prefs", "O2", operator.getName());
                }
            }
        }
    }
    // Check resource handler assigns different hosts for each partition
    ResourceRequestHandler rr = new ResourceRequestHandler();
    int containerMem = 1000;
    Map<String, NodeReport> nodeReports = Maps.newHashMap();
    for (int i = 0; i < 10; i++) {
        String hostName = "host" + i;
        NodeReport nr = BuilderUtils.newNodeReport(BuilderUtils.newNodeId(hostName, 0), NodeState.RUNNING, "httpAddress", "rackName", BuilderUtils.newResource(0, 0), BuilderUtils.newResource(containerMem * 2, 2), 0, null, 0);
        nodeReports.put(nr.getNodeId().getHost(), nr);
    }
    // set resources
    rr.updateNodeReports(Lists.newArrayList(nodeReports.values()));
    Set<String> partitionHostNames = new HashSet<>();
    for (ContainerStartRequest csr : scm.containerStartRequests) {
        String host = rr.getHost(csr, true);
        csr.container.host = host;
        if (csr.container.getOperators().get(0).getName().equals("O2")) {
            Assert.assertNotNull("Host name should not be null", host);
            LOG.info("Partition {} for operator O2 has host = {} ", csr.container.getId(), host);
            Assert.assertTrue("Each Partition should have a different host", !partitionHostNames.contains(host));
            partitionHostNames.add(host);
        }
    }
}

Example 8 with TestGeneratorInputOperator

use of com.datatorrent.stram.engine.TestGeneratorInputOperator in project apex-core by apache.

the class StreamPersistanceTests method testPersistStreamOperatorIsRemovedWhenStreamIsRemoved.

@Test
public void testPersistStreamOperatorIsRemovedWhenStreamIsRemoved() {
    // Remove Stream and check if persist operator is removed
    TestGeneratorInputOperator input1 = dag.addOperator("input1", TestGeneratorInputOperator.class);
    GenericTestOperator x = dag.addOperator("x", new GenericTestOperator());
    TestReceiverOperator persister = new TestReceiverOperator();
    StreamMeta stream = dag.addStream("Stream1", input1.outport, x.inport1);
    stream.persistUsing("Stream1_persister", persister, persister.inport);
    ((LogicalPlan.StreamMeta) stream).remove();
    // Check operator is added to dag
    OperatorMeta persistOperatorMeta = dag.getOperatorMeta("Stream1_persister");
    assertEquals("Persist operator should be removed from dag after stream.remove", null, persistOperatorMeta);
}

Example 9 with TestGeneratorInputOperator

use of com.datatorrent.stram.engine.TestGeneratorInputOperator in project apex-core by apache.

the class StreamPersistanceTests method testPersistStreamOperatorIsAdded.

@Test
public void testPersistStreamOperatorIsAdded() {
    TestGeneratorInputOperator input1 = dag.addOperator("input1", TestGeneratorInputOperator.class);
    GenericTestOperator x = dag.addOperator("x", new GenericTestOperator());
    TestReceiverOperator persister = new TestReceiverOperator();
    StreamMeta stream = dag.addStream("Stream1", input1.outport, x.inport1);
    stream.persistUsing("Stream1_persister", persister, persister.inport);
    // Check operator is added to dag
    OperatorMeta persistOperatorMeta = dag.getOperatorMeta("Stream1_persister");
    assertEquals("Persist operator not added to dag ", persister, persistOperatorMeta.getOperator());
    dag.validate();
}

Example 10 with TestGeneratorInputOperator

use of com.datatorrent.stram.engine.TestGeneratorInputOperator in project apex-core by apache.

the class StreamPersistanceTests method testaddStreamThrowsExceptionOnInvalidLoggerType.

@Test
public void testaddStreamThrowsExceptionOnInvalidLoggerType() {
    // Test Logger with non-optional output ports
    TestGeneratorInputOperator input1 = dag.addOperator("input1", TestGeneratorInputOperator.class);
    GenericTestOperator x = dag.addOperator("x", new GenericTestOperator());
    StreamMeta stream = dag.addStream("Stream1", input1.outport, x.inport1);
    TestOperatorWithOutputPorts persister = new TestOperatorWithOutputPorts();
    try {
        stream.persistUsing("persister", persister, persister.inputPort);
        Assert.fail("should throw Illegal argument exception: Persist operator has non optional output ports");
    } catch (IllegalArgumentException e) {
        logger.debug(e.getMessage());
    }
    // Test already added operator passed
    TestOperatorWithOutputPorts persister1 = new TestOperatorWithOutputPorts();
    try {
        stream.persistUsing("Stream1_persister", persister1, persister1.inputPort);
        Assert.fail("should throw exception that Stream1_persister object was already added");
    } catch (IllegalArgumentException e) {
        logger.debug(e.getMessage());
    }
    // Test persist operator without any input ports
    dag.removeOperator(dag.getOperatorMeta("Stream1_persister").getOperator());
    TestOperatorWithoutInputPorts logger2 = new TestOperatorWithoutInputPorts();
    try {
        stream.persistUsing("Stream1_persister", logger2);
        Assert.fail("should throw Illegal argument exception: persist operator should have input ports");
    } catch (IllegalArgumentException e) {
        logger.debug(e.getMessage());
    }
    // Test persist operator with more than one input port as non-optional
    dag.removeOperator(dag.getOperatorMeta("Stream1_persister").getOperator());
    TestOperatorWithMultipleNonOptionalInputPorts persister3 = new TestOperatorWithMultipleNonOptionalInputPorts();
    try {
        stream.persistUsing("Stream1_persister", persister3);
        Assert.fail("should throw Illegal argument exception: persist operator should have at most 1 non-optional input port");
    } catch (IllegalArgumentException e) {
        logger.debug(e.getMessage());
    }
}