use of com.datatorrent.stram.plan.logical.LogicalPlan.OutputPortMeta in project apex-core by apache.
the class LogicalPlanSerializer method convertToProperties.
public static PropertiesConfiguration convertToProperties(LogicalPlan dag) {
PropertiesConfiguration props = new PropertiesConfiguration();
Collection<OperatorMeta> allOperators = dag.getAllOperators();
for (OperatorMeta operatorMeta : allOperators) {
String operatorKey = LogicalPlanConfiguration.OPERATOR_PREFIX + operatorMeta.getName();
Operator operator = operatorMeta.getOperator();
props.setProperty(operatorKey + "." + LogicalPlanConfiguration.OPERATOR_CLASSNAME, operator.getClass().getName());
BeanMap operatorProperties = LogicalPlanConfiguration.getObjectProperties(operator);
@SuppressWarnings("rawtypes") Iterator entryIterator = operatorProperties.entryIterator();
while (entryIterator.hasNext()) {
try {
@SuppressWarnings("unchecked") Map.Entry<String, Object> entry = (Map.Entry<String, Object>) entryIterator.next();
if (!entry.getKey().equals("class") && !entry.getKey().equals("name") && entry.getValue() != null) {
props.setProperty(operatorKey + "." + entry.getKey(), entry.getValue());
}
} catch (Exception ex) {
LOG.warn("Error trying to get a property of operator {}", operatorMeta.getName(), ex);
}
}
}
Collection<StreamMeta> allStreams = dag.getAllStreams();
for (StreamMeta streamMeta : allStreams) {
String streamKey = LogicalPlanConfiguration.STREAM_PREFIX + streamMeta.getName();
OutputPortMeta source = streamMeta.getSource();
Collection<InputPortMeta> sinks = streamMeta.getSinks();
props.setProperty(streamKey + "." + LogicalPlanConfiguration.STREAM_SOURCE, source.getOperatorMeta().getName() + "." + source.getPortName());
String sinksValue = "";
for (InputPortMeta sink : sinks) {
if (!sinksValue.isEmpty()) {
sinksValue += ",";
}
sinksValue += sink.getOperatorMeta().getName() + "." + sink.getPortName();
}
props.setProperty(streamKey + "." + LogicalPlanConfiguration.STREAM_SINKS, sinksValue);
if (streamMeta.getLocality() != null) {
props.setProperty(streamKey + "." + LogicalPlanConfiguration.STREAM_LOCALITY, streamMeta.getLocality().name());
}
}
return props;
}
use of com.datatorrent.stram.plan.logical.LogicalPlan.OutputPortMeta in project apex-core by apache.
the class LogicalPlanSerializer method convertToMap.
/**
*
* @param dag
* @return
*/
public static Map<String, Object> convertToMap(LogicalPlan dag, boolean includeModules) {
HashMap<String, Object> result = new HashMap<>();
ArrayList<Object> operatorArray = new ArrayList<>();
ArrayList<Object> streamMap = new ArrayList<>();
//result.put("applicationName", appConfig.getName());
result.put("operators", operatorArray);
result.put("streams", streamMap);
//LogicalPlan dag = StramAppLauncher.prepareDAG(appConfig, StreamingApplication.LAUNCHMODE_YARN);
//
// should we put the DAGContext info here?
Map<String, Object> dagAttrs = new HashMap<>();
for (Map.Entry<Attribute<Object>, Object> e : Attribute.AttributeMap.AttributeInitializer.getAllAttributes(dag, Context.DAGContext.class).entrySet()) {
dagAttrs.put(e.getKey().getSimpleName(), e.getValue());
}
result.put("attributes", dagAttrs);
Collection<OperatorMeta> allOperators = dag.getAllOperators();
ObjectMapper propertyObjectMapper = new ObjectMapper();
propertyObjectMapper.configure(JsonGenerator.Feature.WRITE_NUMBERS_AS_STRINGS, true);
propertyObjectMapper.configure(SerializationConfig.Feature.FAIL_ON_EMPTY_BEANS, false);
StdTypeResolverBuilder typer = new PropertyTypeResolverBuilder();
typer.init(JsonTypeInfo.Id.CLASS, null);
typer = typer.inclusion(JsonTypeInfo.As.PROPERTY);
propertyObjectMapper.setDefaultTyping(typer);
for (OperatorMeta operatorMeta : allOperators) {
HashMap<String, Object> operatorDetailMap = new HashMap<>();
ArrayList<Map<String, Object>> portList = new ArrayList<>();
Map<String, Object> attributeMap = new HashMap<>();
String operatorName = operatorMeta.getName();
operatorArray.add(operatorDetailMap);
operatorDetailMap.put("name", operatorName);
operatorDetailMap.put("ports", portList);
operatorDetailMap.put("class", operatorMeta.getOperator().getClass().getName());
operatorDetailMap.put("attributes", attributeMap);
Map<Attribute<Object>, Object> rawAttributes = Attribute.AttributeMap.AttributeInitializer.getAllAttributes(operatorMeta, Context.OperatorContext.class);
for (Map.Entry<Attribute<Object>, Object> entry : rawAttributes.entrySet()) {
attributeMap.put(entry.getKey().getSimpleName(), entry.getValue());
}
ObjectMapperString str;
try {
str = new ObjectMapperString(propertyObjectMapper.writeValueAsString(operatorMeta.getOperator()));
} catch (Throwable ex) {
LOG.error("Got exception when trying to get properties for operator {}", operatorMeta.getName(), ex);
str = null;
}
operatorDetailMap.put("properties", str);
Operators.PortMappingDescriptor pmd = new Operators.PortMappingDescriptor();
Operators.describe(operatorMeta.getOperator(), pmd);
for (Map.Entry<String, PortContextPair<InputPort<?>>> entry : pmd.inputPorts.entrySet()) {
HashMap<String, Object> portDetailMap = new HashMap<>();
HashMap<String, Object> portAttributeMap = new HashMap<>();
InputPortMeta portMeta = operatorMeta.getMeta(entry.getValue().component);
String portName = portMeta.getPortName();
portDetailMap.put("name", portName);
portDetailMap.put("type", "input");
portDetailMap.put("attributes", portAttributeMap);
rawAttributes = Attribute.AttributeMap.AttributeInitializer.getAllAttributes(portMeta, Context.PortContext.class);
for (Map.Entry<Attribute<Object>, Object> attEntry : rawAttributes.entrySet()) {
portAttributeMap.put(attEntry.getKey().getSimpleName(), attEntry.getValue());
}
portList.add(portDetailMap);
}
for (Map.Entry<String, PortContextPair<OutputPort<?>>> entry : pmd.outputPorts.entrySet()) {
HashMap<String, Object> portDetailMap = new HashMap<>();
HashMap<String, Object> portAttributeMap = new HashMap<>();
OutputPortMeta portMeta = operatorMeta.getMeta(entry.getValue().component);
String portName = portMeta.getPortName();
portDetailMap.put("name", portName);
portDetailMap.put("type", "output");
portDetailMap.put("attributes", portAttributeMap);
rawAttributes = Attribute.AttributeMap.AttributeInitializer.getAllAttributes(portMeta, Context.PortContext.class);
for (Map.Entry<Attribute<Object>, Object> attEntry : rawAttributes.entrySet()) {
portAttributeMap.put(attEntry.getKey().getSimpleName(), attEntry.getValue());
}
portList.add(portDetailMap);
}
}
Collection<StreamMeta> allStreams = dag.getAllStreams();
for (StreamMeta streamMeta : allStreams) {
HashMap<String, Object> streamDetailMap = new HashMap<>();
String streamName = streamMeta.getName();
streamMap.add(streamDetailMap);
String sourcePortName = streamMeta.getSource().getPortName();
OperatorMeta operatorMeta = streamMeta.getSource().getOperatorMeta();
HashMap<String, Object> sourcePortDetailMap = new HashMap<>();
sourcePortDetailMap.put("operatorName", operatorMeta.getName());
sourcePortDetailMap.put("portName", sourcePortName);
streamDetailMap.put("name", streamName);
streamDetailMap.put("source", sourcePortDetailMap);
Collection<InputPortMeta> sinks = streamMeta.getSinks();
ArrayList<HashMap<String, Object>> sinkPortList = new ArrayList<>();
for (InputPortMeta sinkPort : sinks) {
HashMap<String, Object> sinkPortDetailMap = new HashMap<>();
sinkPortDetailMap.put("operatorName", sinkPort.getOperatorMeta().getName());
sinkPortDetailMap.put("portName", sinkPort.getPortName());
sinkPortList.add(sinkPortDetailMap);
}
streamDetailMap.put("sinks", sinkPortList);
if (streamMeta.getLocality() != null) {
streamDetailMap.put("locality", streamMeta.getLocality().name());
}
}
if (includeModules) {
ArrayList<Map<String, Object>> modulesArray = new ArrayList<>();
result.put("modules", modulesArray);
for (LogicalPlan.ModuleMeta meta : dag.getAllModules()) {
modulesArray.add(getLogicalModuleDetails(dag, meta));
}
}
return result;
}
use of com.datatorrent.stram.plan.logical.LogicalPlan.OutputPortMeta in project apex-core by apache.
the class PhysicalPlan method redoPartitions.
private void redoPartitions(PMapping currentMapping, String note) {
Partitioner<Operator> partitioner = getPartitioner(currentMapping);
if (partitioner == null) {
LOG.warn("No partitioner for {}", currentMapping.logicalOperator);
return;
}
RepartitionContext mainPC = new RepartitionContext(partitioner, currentMapping, 0);
if (mainPC.newPartitions.isEmpty()) {
LOG.warn("Empty partition list after repartition: {}", currentMapping.logicalOperator);
return;
}
int memoryPerPartition = currentMapping.logicalOperator.getValue(OperatorContext.MEMORY_MB);
for (Map.Entry<OutputPortMeta, StreamMeta> stream : currentMapping.logicalOperator.getOutputStreams().entrySet()) {
if (stream.getValue().getLocality() != Locality.THREAD_LOCAL && stream.getValue().getLocality() != Locality.CONTAINER_LOCAL) {
memoryPerPartition += stream.getKey().getValue(PortContext.BUFFER_MEMORY_MB);
}
}
for (OperatorMeta pp : currentMapping.parallelPartitions) {
for (Map.Entry<OutputPortMeta, StreamMeta> stream : pp.getOutputStreams().entrySet()) {
if (stream.getValue().getLocality() != Locality.THREAD_LOCAL && stream.getValue().getLocality() != Locality.CONTAINER_LOCAL) {
memoryPerPartition += stream.getKey().getValue(PortContext.BUFFER_MEMORY_MB);
}
}
memoryPerPartition += pp.getValue(OperatorContext.MEMORY_MB);
}
int requiredMemoryMB = (mainPC.newPartitions.size() - mainPC.currentPartitions.size()) * memoryPerPartition;
if (requiredMemoryMB > availableMemoryMB) {
LOG.warn("Insufficient headroom for repartitioning: available {}m required {}m", availableMemoryMB, requiredMemoryMB);
return;
}
List<Partition<Operator>> addedPartitions = new ArrayList<>();
// determine modifications of partition set, identify affected operator instance(s)
for (Partition<Operator> newPartition : mainPC.newPartitions) {
PTOperator op = mainPC.currentPartitionMap.remove(newPartition);
if (op == null) {
addedPartitions.add(newPartition);
} else {
// check whether mapping was changed
for (DefaultPartition<Operator> pi : mainPC.currentPartitions) {
if (pi == newPartition && pi.isModified()) {
// existing partition changed (operator or partition keys)
// remove/add to update subscribers and state
mainPC.currentPartitionMap.put(newPartition, op);
addedPartitions.add(newPartition);
}
}
}
}
// remaining entries represent deprecated partitions
this.undeployOpers.addAll(mainPC.currentPartitionMap.values());
// downstream dependencies require redeploy, resolve prior to modifying plan
Set<PTOperator> deps = this.getDependents(mainPC.currentPartitionMap.values());
this.undeployOpers.addAll(deps);
// dependencies need redeploy, except operators excluded in remove
this.deployOpers.addAll(deps);
// process parallel partitions before removing operators from the plan
LinkedHashMap<PMapping, RepartitionContext> partitionContexts = Maps.newLinkedHashMap();
Stack<OperatorMeta> parallelPartitions = new Stack<>();
parallelPartitions.addAll(currentMapping.parallelPartitions);
pendingLoop: while (!parallelPartitions.isEmpty()) {
OperatorMeta ppMeta = parallelPartitions.pop();
for (StreamMeta s : ppMeta.getInputStreams().values()) {
if (currentMapping.parallelPartitions.contains(s.getSource().getOperatorMeta()) && parallelPartitions.contains(s.getSource().getOperatorMeta())) {
parallelPartitions.push(ppMeta);
parallelPartitions.remove(s.getSource().getOperatorMeta());
parallelPartitions.push(s.getSource().getOperatorMeta());
continue pendingLoop;
}
}
LOG.debug("Processing parallel partition {}", ppMeta);
PMapping ppm = this.logicalToPTOperator.get(ppMeta);
Partitioner<Operator> ppp = getPartitioner(ppm);
if (ppp == null) {
partitionContexts.put(ppm, null);
} else {
RepartitionContext pc = new RepartitionContext(ppp, ppm, mainPC.newPartitions.size());
if (pc.newPartitions == null) {
throw new IllegalStateException("Partitioner returns null for parallel partition " + ppm.logicalOperator);
}
partitionContexts.put(ppm, pc);
}
}
// plan updates start here, after all changes were identified
// remove obsolete operators first, any freed resources
// can subsequently be used for new/modified partitions
List<PTOperator> copyPartitions = Lists.newArrayList(currentMapping.partitions);
// remove deprecated partitions from plan
for (PTOperator p : mainPC.currentPartitionMap.values()) {
copyPartitions.remove(p);
removePartition(p, currentMapping);
mainPC.operatorIdToPartition.remove(p.getId());
}
currentMapping.partitions = copyPartitions;
// add new operators
for (Partition<Operator> newPartition : addedPartitions) {
PTOperator p = addPTOperator(currentMapping, newPartition, mainPC.minCheckpoint);
mainPC.operatorIdToPartition.put(p.getId(), newPartition);
}
// process parallel partition changes
for (Map.Entry<PMapping, RepartitionContext> e : partitionContexts.entrySet()) {
if (e.getValue() == null) {
// no partitioner, add required operators
for (int i = 0; i < addedPartitions.size(); i++) {
LOG.debug("Automatically adding to parallel partition {}", e.getKey());
// set activation windowId to confirm to upstream checkpoints
addPTOperator(e.getKey(), null, mainPC.minCheckpoint);
}
} else {
RepartitionContext pc = e.getValue();
// track previous parallel partition mapping
Map<Partition<Operator>, Partition<Operator>> prevMapping = Maps.newHashMap();
for (int i = 0; i < mainPC.currentPartitions.size(); i++) {
prevMapping.put(pc.currentPartitions.get(i), mainPC.currentPartitions.get(i));
}
// determine which new partitions match upstream, remaining to be treated as new operator
Map<Partition<Operator>, Partition<Operator>> newMapping = Maps.newHashMap();
Iterator<Partition<Operator>> itMain = mainPC.newPartitions.iterator();
Iterator<Partition<Operator>> itParallel = pc.newPartitions.iterator();
while (itMain.hasNext() && itParallel.hasNext()) {
newMapping.put(itParallel.next(), itMain.next());
}
for (Partition<Operator> newPartition : pc.newPartitions) {
PTOperator op = pc.currentPartitionMap.remove(newPartition);
if (op == null) {
pc.addedPartitions.add(newPartition);
} else if (prevMapping.get(newPartition) != newMapping.get(newPartition)) {
// upstream partitions don't match, remove/add to replace with new operator
pc.currentPartitionMap.put(newPartition, op);
pc.addedPartitions.add(newPartition);
} else {
// check whether mapping was changed - based on DefaultPartition implementation
for (DefaultPartition<Operator> pi : pc.currentPartitions) {
if (pi == newPartition && pi.isModified()) {
// existing partition changed (operator or partition keys)
// remove/add to update subscribers and state
mainPC.currentPartitionMap.put(newPartition, op);
pc.addedPartitions.add(newPartition);
}
}
}
}
if (!pc.currentPartitionMap.isEmpty()) {
// remove obsolete partitions
List<PTOperator> cowPartitions = Lists.newArrayList(e.getKey().partitions);
for (PTOperator p : pc.currentPartitionMap.values()) {
cowPartitions.remove(p);
removePartition(p, e.getKey());
pc.operatorIdToPartition.remove(p.getId());
}
e.getKey().partitions = cowPartitions;
}
// add new partitions
for (Partition<Operator> newPartition : pc.addedPartitions) {
PTOperator oper = addPTOperator(e.getKey(), newPartition, mainPC.minCheckpoint);
pc.operatorIdToPartition.put(oper.getId(), newPartition);
}
getPartitioner(e.getKey()).partitioned(pc.operatorIdToPartition);
}
}
updateStreamMappings(currentMapping);
for (PMapping pp : partitionContexts.keySet()) {
updateStreamMappings(pp);
}
deployChanges();
if (mainPC.currentPartitions.size() != mainPC.newPartitions.size()) {
StramEvent ev = new StramEvent.PartitionEvent(currentMapping.logicalOperator.getName(), mainPC.currentPartitions.size(), mainPC.newPartitions.size());
ev.setReason(note);
this.ctx.recordEventAsync(ev);
}
partitioner.partitioned(mainPC.operatorIdToPartition);
}
Aggregations