Examples with StramLocalCluster com.datatorrent.stram.StramLocalCluster used on opensource projects

Example 16 with StramLocalCluster

use of com.datatorrent.stram.StramLocalCluster in project apex-core by apache.

the class OiOStreamTest method validateOiOiODiamondImplementation.

@Test
public void validateOiOiODiamondImplementation() throws Exception {
    LogicalPlan lp = new LogicalPlan();
    lp.setAttribute(com.datatorrent.api.Context.OperatorContext.STORAGE_AGENT, new MemoryStorageAgent());
    ThreadIdValidatingInputOperator inputOperator = lp.addOperator("inputOperator", new ThreadIdValidatingInputOperator());
    ThreadIdValidatingGenericIntermediateOperator intermediateOperator1 = lp.addOperator("intermediateOperator1", new ThreadIdValidatingGenericIntermediateOperator());
    ThreadIdValidatingGenericIntermediateOperator intermediateOperator2 = lp.addOperator("intermediateOperator2", new ThreadIdValidatingGenericIntermediateOperator());
    ThreadIdValidatingGenericOperatorWithTwoInputPorts outputOperator = lp.addOperator("outputOperator", new ThreadIdValidatingGenericOperatorWithTwoInputPorts());
    StreamMeta stream1 = lp.addStream("OiOinput", inputOperator.output, intermediateOperator1.input, intermediateOperator2.input);
    StreamMeta stream2 = lp.addStream("OiOintermediateToOutput1", intermediateOperator1.output, outputOperator.input);
    StreamMeta stream3 = lp.addStream("OiOintermediateToOutput2", intermediateOperator2.output, outputOperator.input2);
    StramLocalCluster slc;
    /*
     * The first test makes sure that when they are not ThreadLocal they use different threads
     */
    ThreadIdValidatingGenericIntermediateOperator.threadList.clear();
    ThreadIdValidatingOutputOperator.threadList.clear();
    lp.validate();
    slc = new StramLocalCluster(lp);
    slc.run();
    Assert.assertEquals("nonOIO: Number of threads", 2, ThreadIdValidatingGenericIntermediateOperator.threadList.size());
    Assert.assertFalse("nonOIO: Thread Ids of input operator and intermediate operator1", ThreadIdValidatingInputOperator.threadId == ThreadIdValidatingGenericIntermediateOperator.threadList.get(0));
    Assert.assertFalse("nonOIO: Thread Ids of input operator and intermediate operator2", ThreadIdValidatingInputOperator.threadId == ThreadIdValidatingGenericIntermediateOperator.threadList.get(1));
    Assert.assertNotEquals("nonOIO: Thread Ids of two intermediate operators", ThreadIdValidatingGenericIntermediateOperator.threadList.get(0), ThreadIdValidatingGenericIntermediateOperator.threadList.get(1));
    Assert.assertNotEquals("nonOIO: Thread Ids of input and output operators", ThreadIdValidatingInputOperator.threadId, ThreadIdValidatingGenericOperatorWithTwoInputPorts.threadId);
    /*
     * This test makes sure that since all operators in diamond are ThreadLocal, they indeed share a thread
     */
    ThreadIdValidatingGenericIntermediateOperator.threadList.clear();
    ThreadIdValidatingOutputOperator.threadList.clear();
    stream1.setLocality(Locality.THREAD_LOCAL);
    stream2.setLocality(Locality.THREAD_LOCAL);
    stream3.setLocality(Locality.THREAD_LOCAL);
    lp.validate();
    slc = new StramLocalCluster(lp);
    slc.run();
    Assert.assertEquals("OIO: Number of threads", 2, ThreadIdValidatingGenericIntermediateOperator.threadList.size());
    Assert.assertEquals("OIO: Thread Ids of input operator and intermediate operator1", ThreadIdValidatingInputOperator.threadId, (long) ThreadIdValidatingGenericIntermediateOperator.threadList.get(0));
    Assert.assertEquals("OIO: Thread Ids of input operator and intermediate operator2", ThreadIdValidatingInputOperator.threadId, (long) ThreadIdValidatingGenericIntermediateOperator.threadList.get(1));
    Assert.assertEquals("OIO: Thread Ids of two intermediate operators", ThreadIdValidatingGenericIntermediateOperator.threadList.get(0), ThreadIdValidatingGenericIntermediateOperator.threadList.get(1));
    Assert.assertEquals("OIO: Thread Ids of input and output operators", ThreadIdValidatingInputOperator.threadId, ThreadIdValidatingGenericOperatorWithTwoInputPorts.threadId);
}

Example 17 with StramLocalCluster

use of com.datatorrent.stram.StramLocalCluster in project apex-core by apache.

the class OiOStreamTest method validateOiOTwoPortBetweenOperatorsImplementation.

@Test
public void validateOiOTwoPortBetweenOperatorsImplementation() throws Exception {
    LogicalPlan lp = new LogicalPlan();
    lp.setAttribute(com.datatorrent.api.Context.OperatorContext.STORAGE_AGENT, new MemoryStorageAgent());
    ThreadIdValidatingInputOperator inputOperator = lp.addOperator("inputOperator", new ThreadIdValidatingInputOperator());
    ThreadIdValidatingGenericIntermediateOperatorWithTwoOutputPorts intermediateOperator = lp.addOperator("intermediateOperator1", new ThreadIdValidatingGenericIntermediateOperatorWithTwoOutputPorts());
    ThreadIdValidatingGenericOperatorWithTwoInputPorts outputOperator = lp.addOperator("outputOperator", new ThreadIdValidatingGenericOperatorWithTwoInputPorts());
    StreamMeta stream1 = lp.addStream("OiOinput", inputOperator.output, intermediateOperator.input);
    StreamMeta stream2 = lp.addStream("OiOintermediateOutput1", intermediateOperator.output, outputOperator.input);
    StreamMeta stream3 = lp.addStream("OiOintermediateOutput2", intermediateOperator.output2, outputOperator.input2);
    StramLocalCluster slc;
    /*
     * The first test makes sure that when they are not ThreadLocal they use different threads
     */
    ThreadIdValidatingGenericIntermediateOperatorWithTwoOutputPorts.threadList.clear();
    ThreadIdValidatingOutputOperator.threadList.clear();
    lp.validate();
    slc = new StramLocalCluster(lp);
    slc.run();
    Assert.assertEquals("nonOIO: Number of threads", 1, ThreadIdValidatingGenericIntermediateOperatorWithTwoOutputPorts.threadList.size());
    Assert.assertNotEquals("nonOIO: Thread Ids of input operator and intermediate operator", ThreadIdValidatingInputOperator.threadId, ThreadIdValidatingGenericIntermediateOperatorWithTwoOutputPorts.threadId);
    Assert.assertNotEquals("nonOIO: Thread Ids of intermediate and output operators", ThreadIdValidatingGenericIntermediateOperatorWithTwoOutputPorts.threadId, ThreadIdValidatingGenericOperatorWithTwoInputPorts.threadId);
    Assert.assertNotEquals("nonOIO: Thread Ids of input and output operators", ThreadIdValidatingInputOperator.threadId, ThreadIdValidatingGenericOperatorWithTwoInputPorts.threadId);
    /*
     * This test makes sure that since all streams between two operators are thread local, they indeed share a thread
     */
    ThreadIdValidatingGenericIntermediateOperatorWithTwoOutputPorts.threadList.clear();
    ThreadIdValidatingOutputOperator.threadList.clear();
    stream2.setLocality(Locality.THREAD_LOCAL);
    stream3.setLocality(Locality.THREAD_LOCAL);
    lp.validate();
    slc = new StramLocalCluster(lp);
    slc.run();
    Assert.assertEquals("OIO: Number of threads", 1, ThreadIdValidatingGenericIntermediateOperatorWithTwoOutputPorts.threadList.size());
    Assert.assertNotEquals("OIO: Thread Ids of input operator and intermediate operator", ThreadIdValidatingInputOperator.threadId, ThreadIdValidatingGenericIntermediateOperatorWithTwoOutputPorts.threadId);
    Assert.assertEquals("OIO: Thread Ids of intermediate and output operators", ThreadIdValidatingGenericIntermediateOperatorWithTwoOutputPorts.threadId, ThreadIdValidatingGenericOperatorWithTwoInputPorts.threadId);
    Assert.assertNotEquals("OIO: Thread Ids of input and output operators", ThreadIdValidatingInputOperator.threadId, ThreadIdValidatingGenericOperatorWithTwoInputPorts.threadId);
}

Example 18 with StramLocalCluster

use of com.datatorrent.stram.StramLocalCluster in project apex-core by apache.

the class DelayOperatorTest method testFibonacciRecovery2.

@Test
public void testFibonacciRecovery2() throws Exception {
    LogicalPlan dag = StramTestSupport.createDAG(testMeta);
    TestGeneratorInputOperator dummyInput = dag.addOperator("DUMMY", TestGeneratorInputOperator.class);
    FibonacciOperator fib = dag.addOperator("FIB", FibonacciOperator.class);
    FailableDelayOperator opDelay = dag.addOperator("opDelay", FailableDelayOperator.class);
    opDelay.setSimulateFailureWindows(5, true);
    dag.addStream("dummy_to_operator", dummyInput.outport, fib.dummyInputPort);
    dag.addStream("operator_to_delay", fib.output, opDelay.input);
    dag.addStream("delay_to_operator", opDelay.output, fib.input);
    dag.getAttributes().put(LogicalPlan.CHECKPOINT_WINDOW_COUNT, 2);
    dag.getAttributes().put(LogicalPlan.STREAMING_WINDOW_SIZE_MILLIS, 300);
    dag.getAttributes().put(LogicalPlan.HEARTBEAT_INTERVAL_MILLIS, 50);
    FibonacciOperator.results.clear();
    FailableDelayOperator.failureSimulated = false;
    final StramLocalCluster localCluster = new StramLocalCluster(dag);
    localCluster.setPerContainerBufferServer(true);
    localCluster.setExitCondition(new Callable<Boolean>() {

        @Override
        public Boolean call() throws Exception {
            return FibonacciOperator.results.size() >= 30;
        }
    });
    localCluster.run(60000);
    Assert.assertTrue("failure should be invoked", FailableDelayOperator.failureSimulated);
    Assert.assertArrayEquals(Arrays.copyOfRange(new TreeSet<>(Arrays.asList(FIBONACCI_NUMBERS)).toArray(), 0, 20), Arrays.copyOfRange(new TreeSet<>(FibonacciOperator.results).toArray(), 0, 20));
}

Example 19 with StramLocalCluster

use of com.datatorrent.stram.StramLocalCluster in project apex-core by apache.

the class StramAppLauncher method runLocal.

/**
   * Run application in-process. Returns only once application completes.
   *
   * @param appConfig
   * @throws Exception
   */
public void runLocal(AppFactory appConfig) throws Exception {
    propertiesBuilder.conf.setEnum(StreamingApplication.ENVIRONMENT, StreamingApplication.Environment.LOCAL);
    LogicalPlan lp = appConfig.createApp(propertiesBuilder);
    String libJarsCsv = lp.getAttributes().get(Context.DAGContext.LIBRARY_JARS);
    if (libJarsCsv != null && libJarsCsv.length() != 0) {
        String[] split = libJarsCsv.split(StramClient.LIB_JARS_SEP);
        for (String jarPath : split) {
            File file = new File(jarPath);
            URL url = file.toURI().toURL();
            launchDependencies.add(url);
        }
    }
    // local mode requires custom classes to be resolved through the context class loader
    loadDependencies();
    StramLocalCluster lc = new StramLocalCluster(lp);
    lc.run();
}

Example 20 with StramLocalCluster

use of com.datatorrent.stram.StramLocalCluster in project apex-core by apache.

the class EmbeddedAppLauncherImpl method launchApp.

@Override
public EmbeddedAppHandleImpl launchApp(StreamingApplication application, Configuration configuration, Attribute.AttributeMap launchParameters) throws LauncherException {
    try {
        prepareDAG(application, configuration);
    } catch (Exception e) {
        throw new LauncherException(e);
    }
    StramLocalCluster lc = getController();
    boolean launched = false;
    if (launchParameters != null) {
        if (StramUtils.getValueWithDefault(launchParameters, SERIALIZE_DAG)) {
            // Check if DAG can be serialized
            try {
                cloneDAG();
            } catch (Exception e) {
                throw new LauncherException(e);
            }
        }
        lc.setHeartbeatMonitoringEnabled(StramUtils.getValueWithDefault(launchParameters, HEARTBEAT_MONITORING));
        if (StramUtils.getValueWithDefault(launchParameters, RUN_ASYNC)) {
            lc.runAsync();
            launched = true;
        } else {
            Long runMillis = StramUtils.getValueWithDefault(launchParameters, RUN_MILLIS);
            if (runMillis != null) {
                lc.run(runMillis);
                launched = true;
            }
        }
    }
    if (!launched) {
        lc.run();
    }
    return new EmbeddedAppHandleImpl(lc);
}