Examples with LogicalPlan com.datatorrent.stram.plan.logical.LogicalPlan used on opensource projects

Example 11 with LogicalPlan

use of com.datatorrent.stram.plan.logical.LogicalPlan in project apex-core by apache.

the class StramAppLauncher method launchApp.

/**
   * Submit application to the cluster and return the app id.
   * Sets the context class loader for application dependencies.
   *
   * @param appConfig
   * @return ApplicationId
   * @throws Exception
   */
public ApplicationId launchApp(AppFactory appConfig) throws Exception {
    loadDependencies();
    Configuration conf = propertiesBuilder.conf;
    conf.setEnum(StreamingApplication.ENVIRONMENT, StreamingApplication.Environment.CLUSTER);
    LogicalPlan dag = appConfig.createApp(propertiesBuilder);
    if (UserGroupInformation.isSecurityEnabled()) {
        long hdfsTokenMaxLifeTime = conf.getLong(StramClientUtils.DT_HDFS_TOKEN_MAX_LIFE_TIME, conf.getLong(StramClientUtils.HDFS_TOKEN_MAX_LIFE_TIME, StramClientUtils.DELEGATION_TOKEN_MAX_LIFETIME_DEFAULT));
        dag.setAttribute(LogicalPlan.HDFS_TOKEN_LIFE_TIME, hdfsTokenMaxLifeTime);
        long rmTokenMaxLifeTime = conf.getLong(StramClientUtils.DT_RM_TOKEN_MAX_LIFE_TIME, conf.getLong(YarnConfiguration.DELEGATION_TOKEN_MAX_LIFETIME_KEY, YarnConfiguration.DELEGATION_TOKEN_MAX_LIFETIME_DEFAULT));
        dag.setAttribute(LogicalPlan.RM_TOKEN_LIFE_TIME, rmTokenMaxLifeTime);
        setTokenRefreshCredentials(dag, conf);
    }
    String tokenRefreshFactor = conf.get(StramClientUtils.TOKEN_ANTICIPATORY_REFRESH_FACTOR);
    if (tokenRefreshFactor != null && tokenRefreshFactor.trim().length() > 0) {
        dag.setAttribute(LogicalPlan.TOKEN_REFRESH_ANTICIPATORY_FACTOR, Double.parseDouble(tokenRefreshFactor));
    }
    StramClient client = new StramClient(conf, dag);
    try {
        client.start();
        LinkedHashSet<String> libjars = Sets.newLinkedHashSet();
        String libjarsCsv = conf.get(LIBJARS_CONF_KEY_NAME);
        if (libjarsCsv != null) {
            String[] jars = StringUtils.splitByWholeSeparator(libjarsCsv, StramClient.LIB_JARS_SEP);
            libjars.addAll(Arrays.asList(jars));
        }
        if (deployJars != null) {
            for (File deployJar : deployJars) {
                libjars.add(deployJar.getAbsolutePath());
            }
        }
        client.setResources(libjars);
        client.setFiles(conf.get(FILES_CONF_KEY_NAME));
        client.setArchives(conf.get(ARCHIVES_CONF_KEY_NAME));
        client.setOriginalAppId(conf.get(ORIGINAL_APP_ID));
        client.setQueueName(conf.get(QUEUE_NAME));
        String tags = conf.get(TAGS);
        if (tags != null) {
            for (String tag : tags.split(",")) {
                client.addTag(tag.trim());
            }
        }
        client.startApplication();
        return client.getApplicationReport().getApplicationId();
    } finally {
        client.stop();
    }
}

Example 12 with LogicalPlan

use of com.datatorrent.stram.plan.logical.LogicalPlan in project apex-core by apache.

the class LogicalPlanSerializer method getLogicalModuleDetails.

/**
   * Return information about operators and inner modules of a module.
   *
   * @param dag        top level DAG
   * @param moduleMeta module information. DAG within module is used for constructing response.
   * @return
   */
private static Map<String, Object> getLogicalModuleDetails(LogicalPlan dag, LogicalPlan.ModuleMeta moduleMeta) {
    Map<String, Object> moduleDetailMap = new HashMap<>();
    ArrayList<String> operatorArray = new ArrayList<>();
    moduleDetailMap.put("name", moduleMeta.getName());
    moduleDetailMap.put("className", moduleMeta.getGenericOperator().getClass().getName());
    moduleDetailMap.put("operators", operatorArray);
    for (OperatorMeta operatorMeta : moduleMeta.getDag().getAllOperators()) {
        if (operatorMeta.getModuleName() == null) {
            String fullName = moduleMeta.getFullName() + LogicalPlan.MODULE_NAMESPACE_SEPARATOR + operatorMeta.getName();
            operatorArray.add(fullName);
        }
    }
    ArrayList<Map<String, Object>> modulesArray = new ArrayList<>();
    moduleDetailMap.put("modules", modulesArray);
    for (LogicalPlan.ModuleMeta meta : moduleMeta.getDag().getAllModules()) {
        modulesArray.add(getLogicalModuleDetails(dag, meta));
    }
    return moduleDetailMap;
}

Example 13 with LogicalPlan

use of com.datatorrent.stram.plan.logical.LogicalPlan in project apex-core by apache.

the class AtLeastOnceTest method testInlineOperatorsRecovery.

@Test
public void testInlineOperatorsRecovery() throws Exception {
    RecoverableInputOperator.initGenTuples();
    CollectorOperator.collection.clear();
    int maxTuples = 30;
    LogicalPlan dag = new LogicalPlan();
    String workingDir = new File("target/testOperatorRecovery").getAbsolutePath();
    AsyncFSStorageAgent asyncFSStorageAgent = new AsyncFSStorageAgent(workingDir, null);
    asyncFSStorageAgent.setSyncCheckpoint(true);
    dag.setAttribute(Context.OperatorContext.STORAGE_AGENT, asyncFSStorageAgent);
    //dag.getAttributes().get(DAG.HEARTBEAT_INTERVAL_MILLIS, 400);
    dag.getAttributes().put(LogicalPlan.CHECKPOINT_WINDOW_COUNT, 2);
    dag.getAttributes().put(LogicalPlan.STREAMING_WINDOW_SIZE_MILLIS, 300);
    dag.getAttributes().put(LogicalPlan.CONTAINERS_MAX_COUNT, 1);
    RecoverableInputOperator rip = dag.addOperator("LongGenerator", RecoverableInputOperator.class);
    rip.setMaximumTuples(maxTuples);
    rip.setSimulateFailure(true);
    CollectorOperator cm = dag.addOperator("LongCollector", CollectorOperator.class);
    cm.setSimulateFailure(true);
    dag.addStream("connection", rip.output, cm.input).setLocality(Locality.CONTAINER_LOCAL);
    StramLocalCluster lc = new StramLocalCluster(dag);
    lc.run();
    //    for (Long l: collection) {
    //      logger.debug(Codec.getStringWindowId(l));
    //    }
    Assert.assertEquals("Generated Outputs", maxTuples, CollectorOperator.collection.size());
}

Example 14 with LogicalPlan

use of com.datatorrent.stram.plan.logical.LogicalPlan in project apex-core by apache.

the class InputOperatorTest method testSomeMethod.

@Test
public void testSomeMethod() throws Exception {
    LogicalPlan dag = new LogicalPlan();
    String testWorkDir = new File("target").getAbsolutePath();
    dag.setAttribute(OperatorContext.STORAGE_AGENT, new AsyncFSStorageAgent(testWorkDir, null));
    EvenOddIntegerGeneratorInputOperator generator = dag.addOperator("NumberGenerator", EvenOddIntegerGeneratorInputOperator.class);
    final CollectorModule<Number> collector = dag.addOperator("NumberCollector", new CollectorModule<Number>());
    dag.addStream("EvenIntegers", generator.even, collector.even).setLocality(Locality.CONTAINER_LOCAL);
    dag.addStream("OddIntegers", generator.odd, collector.odd).setLocality(Locality.CONTAINER_LOCAL);
    final StramLocalCluster lc = new StramLocalCluster(dag);
    lc.setHeartbeatMonitoringEnabled(false);
    lc.runAsync();
    WaitCondition c = new WaitCondition() {

        @Override
        public boolean isComplete() {
            return tupleCount.get() > 2;
        }
    };
    StramTestSupport.awaitCompletion(c, 2000);
    lc.shutdown();
    Assert.assertEquals("Collections size", 2, collections.size());
    Assert.assertFalse("Zero tuple count", collections.get(collector.even.id).isEmpty() && collections.get(collector.odd.id).isEmpty());
    Assert.assertTrue("Tuple count", collections.get(collector.even.id).size() - collections.get(collector.odd.id).size() <= 1);
}

Example 15 with LogicalPlan

use of com.datatorrent.stram.plan.logical.LogicalPlan in project apex-core by apache.

the class SliderTest method test.

private void test(int applicationWindowCount, int slideByWindowCount) throws Exception {
    LogicalPlan dag = new LogicalPlan();
    String workingDir = new File("target/sliderTest").getAbsolutePath();
    dag.setAttribute(Context.OperatorContext.STORAGE_AGENT, new AsyncFSStorageAgent(workingDir, null));
    dag.getAttributes().put(LogicalPlan.STREAMING_WINDOW_SIZE_MILLIS, 100);
    Input input = dag.addOperator("Input", new Input());
    Sum sum = dag.addOperator("Sum", new Sum());
    dag.setOperatorAttribute(sum, OperatorContext.APPLICATION_WINDOW_COUNT, applicationWindowCount);
    dag.setOperatorAttribute(sum, OperatorContext.SLIDE_BY_WINDOW_COUNT, slideByWindowCount);
    Validator validate = dag.addOperator("validator", new Validator());
    Validator.numbersValidated = 0;
    validate.numberOfIntegers = applicationWindowCount;
    validate.slideByNumbers = slideByWindowCount;
    dag.addStream("input-sum", input.defaultOutputPort, sum.inputPort);
    dag.addStream("sum-validator", sum.outputPort, validate.validate);
    StramLocalCluster lc = new StramLocalCluster(dag);
    lc.runAsync();
    long startTms = System.currentTimeMillis();
    while (StramTestSupport.DEFAULT_TIMEOUT_MILLIS > System.currentTimeMillis() - startTms) {
        if (validate.numbersValidated > 5) {
            break;
        }
        Thread.sleep(100);
    }
    lc.shutdown();
    Assert.assertTrue("numbers validated more than zero ", validate.numbersValidated > 0);
}