Examples with OperatorMeta com.datatorrent.stram.plan.logical.LogicalPlan.OperatorMeta used on opensource projects

Example 41 with OperatorMeta

use of com.datatorrent.stram.plan.logical.LogicalPlan.OperatorMeta in project apex-core by apache.

the class StramWebServices method getOperatorProperties.

@GET
@Path(PATH_LOGICAL_PLAN_OPERATORS + "/{operatorName}/properties")
@Produces(MediaType.APPLICATION_JSON)
public JSONObject getOperatorProperties(@PathParam("operatorName") String operatorName, @QueryParam("propertyName") String propertyName) throws IOException, JSONException {
    init();
    OperatorMeta logicalOperator = dagManager.getLogicalPlan().getOperatorMeta(operatorName);
    BeanMap operatorProperties = null;
    if (logicalOperator == null) {
        ModuleMeta logicalModule = dagManager.getModuleMeta(operatorName);
        if (logicalModule == null) {
            throw new NotFoundException();
        }
        operatorProperties = LogicalPlanConfiguration.getObjectProperties(logicalModule.getOperator());
    } else {
        operatorProperties = LogicalPlanConfiguration.getObjectProperties(logicalOperator.getOperator());
    }
    Map<String, Object> m = getPropertiesAsMap(propertyName, operatorProperties);
    return new JSONObject(objectMapper.writeValueAsString(m));
}

Example 42 with OperatorMeta

use of com.datatorrent.stram.plan.logical.LogicalPlan.OperatorMeta in project apex-core by apache.

the class StreamingContainerManager method getCheckpointGroups.

protected Map<OperatorMeta, Set<OperatorMeta>> getCheckpointGroups() {
    if (this.checkpointGroups == null) {
        this.checkpointGroups = new HashMap<>();
        LogicalPlan dag = this.plan.getLogicalPlan();
        dag.resetNIndex();
        LogicalPlan.ValidationContext vc = new LogicalPlan.ValidationContext();
        for (OperatorMeta om : dag.getRootOperators()) {
            this.plan.getLogicalPlan().findStronglyConnected(om, vc);
        }
        for (Set<OperatorMeta> checkpointGroup : vc.stronglyConnected) {
            for (OperatorMeta om : checkpointGroup) {
                this.checkpointGroups.put(om, checkpointGroup);
            }
        }
    }
    return checkpointGroups;
}

Example 43 with OperatorMeta

use of com.datatorrent.stram.plan.logical.LogicalPlan.OperatorMeta in project apex-core by apache.

the class StreamingContainerManager method aggregateMetrics.

private void aggregateMetrics(long windowId, Map<Integer, EndWindowStats> endWindowStatsMap) {
    Collection<OperatorMeta> logicalOperators = getLogicalPlan().getAllOperators();
    //for backward compatibility
    for (OperatorMeta operatorMeta : logicalOperators) {
        @SuppressWarnings("deprecation") Context.CountersAggregator aggregator = operatorMeta.getValue(OperatorContext.COUNTERS_AGGREGATOR);
        if (aggregator == null) {
            continue;
        }
        Collection<PTOperator> physicalOperators = plan.getAllOperators(operatorMeta);
        List<Object> counters = Lists.newArrayList();
        for (PTOperator operator : physicalOperators) {
            EndWindowStats stats = endWindowStatsMap.get(operator.getId());
            if (stats != null && stats.counters != null) {
                counters.add(stats.counters);
            }
        }
        if (counters.size() > 0) {
            @SuppressWarnings("deprecation") Object aggregate = aggregator.aggregate(counters);
            latestLogicalCounters.put(operatorMeta.getName(), aggregate);
        }
    }
    for (OperatorMeta operatorMeta : logicalOperators) {
        AutoMetric.Aggregator aggregator = operatorMeta.getMetricAggregatorMeta() != null ? operatorMeta.getMetricAggregatorMeta().getAggregator() : null;
        if (aggregator == null) {
            continue;
        }
        Collection<PTOperator> physicalOperators = plan.getAllOperators(operatorMeta);
        List<AutoMetric.PhysicalMetricsContext> metricPool = Lists.newArrayList();
        for (PTOperator operator : physicalOperators) {
            EndWindowStats stats = endWindowStatsMap.get(operator.getId());
            if (stats != null && stats.metrics != null) {
                PhysicalMetricsContextImpl physicalMetrics = new PhysicalMetricsContextImpl(operator.getId(), stats.metrics);
                metricPool.add(physicalMetrics);
            }
        }
        if (metricPool.isEmpty()) {
            //nothing to aggregate
            continue;
        }
        Map<String, Object> lm = aggregator.aggregate(windowId, metricPool);
        if (lm != null && lm.size() > 0) {
            Queue<Pair<Long, Map<String, Object>>> windowMetrics = logicalMetrics.get(operatorMeta.getName());
            if (windowMetrics == null) {
                windowMetrics = new LinkedBlockingQueue<Pair<Long, Map<String, Object>>>(METRIC_QUEUE_SIZE) {

                    private static final long serialVersionUID = 1L;

                    @Override
                    public boolean add(Pair<Long, Map<String, Object>> longMapPair) {
                        if (remainingCapacity() <= 1) {
                            remove();
                        }
                        return super.add(longMapPair);
                    }
                };
                logicalMetrics.put(operatorMeta.getName(), windowMetrics);
            }
            LOG.debug("Adding to logical metrics for {}", operatorMeta.getName());
            windowMetrics.add(new Pair<>(windowId, lm));
            Map<String, Object> oldValue = latestLogicalMetrics.put(operatorMeta.getName(), lm);
            if (oldValue == null) {
                try {
                    saveMetaInfo();
                } catch (IOException ex) {
                    LOG.error("Cannot save application meta info to DFS. App data sources will not be available.", ex);
                }
            }
        }
    }
}

Example 44 with OperatorMeta

use of com.datatorrent.stram.plan.logical.LogicalPlan.OperatorMeta in project apex-core by apache.

the class StreamingContainerManager method setOperatorProperty.

public void setOperatorProperty(String operatorName, String propertyName, String propertyValue) {
    OperatorMeta logicalOperator = plan.getLogicalPlan().getOperatorMeta(operatorName);
    if (logicalOperator == null) {
        throw new IllegalArgumentException("Unknown operator " + operatorName);
    }
    writeJournal(new SetOperatorProperty(operatorName, propertyName, propertyValue));
    setOperatorProperty(logicalOperator, propertyName, propertyValue);
}

Example 45 with OperatorMeta

use of com.datatorrent.stram.plan.logical.LogicalPlan.OperatorMeta in project apex-core by apache.

the class LogicalPlanSerializer method convertToProperties.

public static PropertiesConfiguration convertToProperties(LogicalPlan dag) {
    PropertiesConfiguration props = new PropertiesConfiguration();
    Collection<OperatorMeta> allOperators = dag.getAllOperators();
    for (OperatorMeta operatorMeta : allOperators) {
        String operatorKey = LogicalPlanConfiguration.OPERATOR_PREFIX + operatorMeta.getName();
        Operator operator = operatorMeta.getOperator();
        props.setProperty(operatorKey + "." + LogicalPlanConfiguration.OPERATOR_CLASSNAME, operator.getClass().getName());
        BeanMap operatorProperties = LogicalPlanConfiguration.getObjectProperties(operator);
        @SuppressWarnings("rawtypes") Iterator entryIterator = operatorProperties.entryIterator();
        while (entryIterator.hasNext()) {
            try {
                @SuppressWarnings("unchecked") Map.Entry<String, Object> entry = (Map.Entry<String, Object>) entryIterator.next();
                if (!entry.getKey().equals("class") && !entry.getKey().equals("name") && entry.getValue() != null) {
                    props.setProperty(operatorKey + "." + entry.getKey(), entry.getValue());
                }
            } catch (Exception ex) {
                LOG.warn("Error trying to get a property of operator {}", operatorMeta.getName(), ex);
            }
        }
    }
    Collection<StreamMeta> allStreams = dag.getAllStreams();
    for (StreamMeta streamMeta : allStreams) {
        String streamKey = LogicalPlanConfiguration.STREAM_PREFIX + streamMeta.getName();
        OutputPortMeta source = streamMeta.getSource();
        Collection<InputPortMeta> sinks = streamMeta.getSinks();
        props.setProperty(streamKey + "." + LogicalPlanConfiguration.STREAM_SOURCE, source.getOperatorMeta().getName() + "." + source.getPortName());
        String sinksValue = "";
        for (InputPortMeta sink : sinks) {
            if (!sinksValue.isEmpty()) {
                sinksValue += ",";
            }
            sinksValue += sink.getOperatorMeta().getName() + "." + sink.getPortName();
        }
        props.setProperty(streamKey + "." + LogicalPlanConfiguration.STREAM_SINKS, sinksValue);
        if (streamMeta.getLocality() != null) {
            props.setProperty(streamKey + "." + LogicalPlanConfiguration.STREAM_LOCALITY, streamMeta.getLocality().name());
        }
    }
    return props;
}