Examples with LogicalPlan com.datatorrent.stram.plan.logical.LogicalPlan used on opensource projects

Example 86 with LogicalPlan

use of com.datatorrent.stram.plan.logical.LogicalPlan in project apex-core by apache.

the class OiOEndWindowTest method validateOiOImplementation.

@Test
public void validateOiOImplementation() throws Exception {
    LogicalPlan lp = new LogicalPlan();
    String workingDir = new File("target/validateOiOImplementation").getAbsolutePath();
    lp.setAttribute(Context.OperatorContext.STORAGE_AGENT, new AsyncFSStorageAgent(workingDir, null));
    TestInputOperator io = lp.addOperator("Input Operator", new TestInputOperator());
    FirstGenericOperator go = lp.addOperator("First Generic Operator", new FirstGenericOperator());
    SecondGenericOperator out = lp.addOperator("Second Generic Operator", new SecondGenericOperator());
    /*
     * This tests make sure that even if the application_window_count is different the endWindow() is called for
     * end_stream
     */
    lp.getOperatorMeta("Second Generic Operator").getAttributes().put(Context.OperatorContext.APPLICATION_WINDOW_COUNT, 2);
    StreamMeta stream = lp.addStream("Stream", io.output, go.input);
    StreamMeta stream1 = lp.addStream("Stream1", go.output, out.input);
    stream1.setLocality(Locality.THREAD_LOCAL);
    lp.validate();
    StramLocalCluster slc = new StramLocalCluster(lp);
    slc.run();
    Assert.assertEquals("End Window Count", FirstGenericOperator.endwindowCount, SecondGenericOperator.endwindowCount);
}

Example 87 with LogicalPlan

use of com.datatorrent.stram.plan.logical.LogicalPlan in project apex-core by apache.

the class OiOStreamTest method validatePositiveOiOiOExtendeddiamond.

@Test
public void validatePositiveOiOiOExtendeddiamond() {
    logger.info("Checking the logic for sanity checking of OiO");
    LogicalPlan plan = new LogicalPlan();
    ThreadIdValidatingInputOperator inputOperator = plan.addOperator("inputOperator", new ThreadIdValidatingInputOperator());
    ThreadIdValidatingGenericIntermediateOperator intermediateOperator1 = plan.addOperator("intermediateOperator1", new ThreadIdValidatingGenericIntermediateOperator());
    ThreadIdValidatingGenericIntermediateOperator intermediateOperator2 = plan.addOperator("intermediateOperator2", new ThreadIdValidatingGenericIntermediateOperator());
    ThreadIdValidatingGenericIntermediateOperator intermediateOperator3 = plan.addOperator("intermediateOperator3", new ThreadIdValidatingGenericIntermediateOperator());
    ThreadIdValidatingGenericIntermediateOperator intermediateOperator4 = plan.addOperator("intermediateOperator4", new ThreadIdValidatingGenericIntermediateOperator());
    ThreadIdValidatingGenericOperatorWithTwoInputPorts outputOperator = plan.addOperator("outputOperator", new ThreadIdValidatingGenericOperatorWithTwoInputPorts());
    plan.addStream("OiOin", inputOperator.output, intermediateOperator1.input, intermediateOperator3.input).setLocality(Locality.THREAD_LOCAL);
    plan.addStream("OiOIntermediate1", intermediateOperator1.output, intermediateOperator2.input).setLocality(Locality.THREAD_LOCAL);
    plan.addStream("OiOIntermediate2", intermediateOperator3.output, intermediateOperator4.input).setLocality(Locality.THREAD_LOCAL);
    plan.addStream("OiOout1", intermediateOperator2.output, outputOperator.input).setLocality(Locality.THREAD_LOCAL);
    plan.addStream("OiOout2", intermediateOperator4.output, outputOperator.input2).setLocality(Locality.THREAD_LOCAL);
    try {
        plan.validate();
        Assert.assertTrue("OiOiO extended diamond validation", true);
    } catch (ConstraintViolationException ex) {
        Assert.fail("OIOIO extended diamond validation");
    }
}

Example 88 with LogicalPlan

use of com.datatorrent.stram.plan.logical.LogicalPlan in project apex-core by apache.

the class OiOStreamTest method validateOiOImplementation.

@Test
public void validateOiOImplementation() throws Exception {
    LogicalPlan lp = new LogicalPlan();
    lp.setAttribute(com.datatorrent.api.Context.OperatorContext.STORAGE_AGENT, new MemoryStorageAgent());
    ThreadIdValidatingInputOperator io = lp.addOperator("Input Operator", new ThreadIdValidatingInputOperator());
    ThreadIdValidatingOutputOperator go = lp.addOperator("Output Operator", new ThreadIdValidatingOutputOperator());
    StreamMeta stream = lp.addStream("Stream", io.output, go.input);
    /* The first test makes sure that when they are not ThreadLocal they use different threads */
    ThreadIdValidatingOutputOperator.threadList.clear();
    lp.validate();
    StramLocalCluster slc = new StramLocalCluster(lp);
    slc.run();
    Assert.assertFalse("Thread Id", ThreadIdValidatingInputOperator.threadId == ThreadIdValidatingOutputOperator.threadId);
    /* This test makes sure that since they are ThreadLocal, they indeed share a thread */
    ThreadIdValidatingOutputOperator.threadList.clear();
    stream.setLocality(Locality.THREAD_LOCAL);
    lp.validate();
    slc = new StramLocalCluster(lp);
    slc.run();
    Assert.assertEquals("Thread Id", ThreadIdValidatingInputOperator.threadId, ThreadIdValidatingOutputOperator.threadId);
}

Example 89 with LogicalPlan

use of com.datatorrent.stram.plan.logical.LogicalPlan in project apex-core by apache.

the class OiOStreamTest method validatePositiveOiOiOdiamondWithCores.

@Test
public void validatePositiveOiOiOdiamondWithCores() {
    logger.info("Checking the logic for sanity checking of OiO");
    LogicalPlan plan = new LogicalPlan();
    ThreadIdValidatingInputOperator inputOperator = plan.addOperator("inputOperator", new ThreadIdValidatingInputOperator());
    ThreadIdValidatingGenericIntermediateOperator intermediateOperator1 = plan.addOperator("intermediateOperator1", new ThreadIdValidatingGenericIntermediateOperator());
    ThreadIdValidatingGenericIntermediateOperator intermediateOperator2 = plan.addOperator("intermediateOperator2", new ThreadIdValidatingGenericIntermediateOperator());
    ThreadIdValidatingGenericIntermediateOperator intermediateOperator3 = plan.addOperator("intermediateOperator3", new ThreadIdValidatingGenericIntermediateOperator());
    ThreadIdValidatingGenericIntermediateOperator intermediateOperator4 = plan.addOperator("intermediateOperator4", new ThreadIdValidatingGenericIntermediateOperator());
    ThreadIdValidatingGenericOperatorWithTwoInputPorts outputOperator = plan.addOperator("outputOperator", new ThreadIdValidatingGenericOperatorWithTwoInputPorts());
    plan.addStream("OiOin", inputOperator.output, intermediateOperator1.input, intermediateOperator3.input).setLocality(Locality.THREAD_LOCAL);
    plan.addStream("OiOIntermediate1", intermediateOperator1.output, intermediateOperator2.input).setLocality(Locality.THREAD_LOCAL);
    plan.addStream("OiOIntermediate2", intermediateOperator3.output, intermediateOperator4.input).setLocality(Locality.THREAD_LOCAL);
    plan.addStream("OiOout1", intermediateOperator2.output, outputOperator.input).setLocality(Locality.THREAD_LOCAL);
    plan.addStream("OiOout2", intermediateOperator4.output, outputOperator.input2).setLocality(Locality.THREAD_LOCAL);
    plan.setOperatorAttribute(inputOperator, OperatorContext.VCORES, 1);
    plan.setOperatorAttribute(intermediateOperator1, OperatorContext.VCORES, 1);
    plan.setOperatorAttribute(intermediateOperator2, OperatorContext.VCORES, 2);
    plan.setOperatorAttribute(intermediateOperator3, OperatorContext.VCORES, 3);
    plan.setOperatorAttribute(intermediateOperator4, OperatorContext.VCORES, 5);
    plan.setAttribute(OperatorContext.STORAGE_AGENT, new StramTestSupport.MemoryStorageAgent());
    try {
        plan.validate();
        Assert.assertTrue("OiOiO extended diamond validation", true);
    } catch (ConstraintViolationException ex) {
        Assert.fail("OIOIO extended diamond validation");
    }
    PhysicalPlan physicalPlan = new PhysicalPlan(plan, new TestPlanContext());
    Assert.assertTrue("number of containers", 1 == physicalPlan.getContainers().size());
    Assert.assertTrue("number of vcores " + physicalPlan.getContainers().get(0).getRequiredVCores(), 5 == physicalPlan.getContainers().get(0).getRequiredVCores());
}

Example 90 with LogicalPlan

use of com.datatorrent.stram.plan.logical.LogicalPlan in project apex-core by apache.

the class OiOStreamTest method validatePositiveOiO.

@Test
public void validatePositiveOiO() {
    logger.info("Checking the logic for sanity checking of OiO");
    LogicalPlan plan = new LogicalPlan();
    RecoverableInputOperator inputOperator = plan.addOperator("IntegerGenerator", new RecoverableInputOperator());
    CollectorOperator outputOperator = plan.addOperator("IntegerCollector", new CollectorOperator());
    plan.addStream("PossibleOiO", inputOperator.output, outputOperator.input).setLocality(Locality.THREAD_LOCAL);
    try {
        plan.validate();
        Assert.assertTrue("OiO validation", true);
    } catch (ConstraintViolationException ex) {
        Assert.fail("OIO Single InputPort");
    }
}