Examples with StreamMeta com.datatorrent.stram.plan.logical.LogicalPlan.StreamMeta used on opensource projects

Example 26 with StreamMeta

use of com.datatorrent.stram.plan.logical.LogicalPlan.StreamMeta in project apex-core by apache.

the class LogicalPlanSerializer method convertToMap.

/**
   *
   * @param dag
   * @return
   */
public static Map<String, Object> convertToMap(LogicalPlan dag, boolean includeModules) {
    HashMap<String, Object> result = new HashMap<>();
    ArrayList<Object> operatorArray = new ArrayList<>();
    ArrayList<Object> streamMap = new ArrayList<>();
    //result.put("applicationName", appConfig.getName());
    result.put("operators", operatorArray);
    result.put("streams", streamMap);
    //LogicalPlan dag = StramAppLauncher.prepareDAG(appConfig, StreamingApplication.LAUNCHMODE_YARN);
    //
    // should we put the DAGContext info here?
    Map<String, Object> dagAttrs = new HashMap<>();
    for (Map.Entry<Attribute<Object>, Object> e : Attribute.AttributeMap.AttributeInitializer.getAllAttributes(dag, Context.DAGContext.class).entrySet()) {
        dagAttrs.put(e.getKey().getSimpleName(), e.getValue());
    }
    result.put("attributes", dagAttrs);
    Collection<OperatorMeta> allOperators = dag.getAllOperators();
    ObjectMapper propertyObjectMapper = new ObjectMapper();
    propertyObjectMapper.configure(JsonGenerator.Feature.WRITE_NUMBERS_AS_STRINGS, true);
    propertyObjectMapper.configure(SerializationConfig.Feature.FAIL_ON_EMPTY_BEANS, false);
    StdTypeResolverBuilder typer = new PropertyTypeResolverBuilder();
    typer.init(JsonTypeInfo.Id.CLASS, null);
    typer = typer.inclusion(JsonTypeInfo.As.PROPERTY);
    propertyObjectMapper.setDefaultTyping(typer);
    for (OperatorMeta operatorMeta : allOperators) {
        HashMap<String, Object> operatorDetailMap = new HashMap<>();
        ArrayList<Map<String, Object>> portList = new ArrayList<>();
        Map<String, Object> attributeMap = new HashMap<>();
        String operatorName = operatorMeta.getName();
        operatorArray.add(operatorDetailMap);
        operatorDetailMap.put("name", operatorName);
        operatorDetailMap.put("ports", portList);
        operatorDetailMap.put("class", operatorMeta.getOperator().getClass().getName());
        operatorDetailMap.put("attributes", attributeMap);
        Map<Attribute<Object>, Object> rawAttributes = Attribute.AttributeMap.AttributeInitializer.getAllAttributes(operatorMeta, Context.OperatorContext.class);
        for (Map.Entry<Attribute<Object>, Object> entry : rawAttributes.entrySet()) {
            attributeMap.put(entry.getKey().getSimpleName(), entry.getValue());
        }
        ObjectMapperString str;
        try {
            str = new ObjectMapperString(propertyObjectMapper.writeValueAsString(operatorMeta.getOperator()));
        } catch (Throwable ex) {
            LOG.error("Got exception when trying to get properties for operator {}", operatorMeta.getName(), ex);
            str = null;
        }
        operatorDetailMap.put("properties", str);
        Operators.PortMappingDescriptor pmd = new Operators.PortMappingDescriptor();
        Operators.describe(operatorMeta.getOperator(), pmd);
        for (Map.Entry<String, PortContextPair<InputPort<?>>> entry : pmd.inputPorts.entrySet()) {
            HashMap<String, Object> portDetailMap = new HashMap<>();
            HashMap<String, Object> portAttributeMap = new HashMap<>();
            InputPortMeta portMeta = operatorMeta.getMeta(entry.getValue().component);
            String portName = portMeta.getPortName();
            portDetailMap.put("name", portName);
            portDetailMap.put("type", "input");
            portDetailMap.put("attributes", portAttributeMap);
            rawAttributes = Attribute.AttributeMap.AttributeInitializer.getAllAttributes(portMeta, Context.PortContext.class);
            for (Map.Entry<Attribute<Object>, Object> attEntry : rawAttributes.entrySet()) {
                portAttributeMap.put(attEntry.getKey().getSimpleName(), attEntry.getValue());
            }
            portList.add(portDetailMap);
        }
        for (Map.Entry<String, PortContextPair<OutputPort<?>>> entry : pmd.outputPorts.entrySet()) {
            HashMap<String, Object> portDetailMap = new HashMap<>();
            HashMap<String, Object> portAttributeMap = new HashMap<>();
            OutputPortMeta portMeta = operatorMeta.getMeta(entry.getValue().component);
            String portName = portMeta.getPortName();
            portDetailMap.put("name", portName);
            portDetailMap.put("type", "output");
            portDetailMap.put("attributes", portAttributeMap);
            rawAttributes = Attribute.AttributeMap.AttributeInitializer.getAllAttributes(portMeta, Context.PortContext.class);
            for (Map.Entry<Attribute<Object>, Object> attEntry : rawAttributes.entrySet()) {
                portAttributeMap.put(attEntry.getKey().getSimpleName(), attEntry.getValue());
            }
            portList.add(portDetailMap);
        }
    }
    Collection<StreamMeta> allStreams = dag.getAllStreams();
    for (StreamMeta streamMeta : allStreams) {
        HashMap<String, Object> streamDetailMap = new HashMap<>();
        String streamName = streamMeta.getName();
        streamMap.add(streamDetailMap);
        String sourcePortName = streamMeta.getSource().getPortName();
        OperatorMeta operatorMeta = streamMeta.getSource().getOperatorMeta();
        HashMap<String, Object> sourcePortDetailMap = new HashMap<>();
        sourcePortDetailMap.put("operatorName", operatorMeta.getName());
        sourcePortDetailMap.put("portName", sourcePortName);
        streamDetailMap.put("name", streamName);
        streamDetailMap.put("source", sourcePortDetailMap);
        Collection<InputPortMeta> sinks = streamMeta.getSinks();
        ArrayList<HashMap<String, Object>> sinkPortList = new ArrayList<>();
        for (InputPortMeta sinkPort : sinks) {
            HashMap<String, Object> sinkPortDetailMap = new HashMap<>();
            sinkPortDetailMap.put("operatorName", sinkPort.getOperatorMeta().getName());
            sinkPortDetailMap.put("portName", sinkPort.getPortName());
            sinkPortList.add(sinkPortDetailMap);
        }
        streamDetailMap.put("sinks", sinkPortList);
        if (streamMeta.getLocality() != null) {
            streamDetailMap.put("locality", streamMeta.getLocality().name());
        }
    }
    if (includeModules) {
        ArrayList<Map<String, Object>> modulesArray = new ArrayList<>();
        result.put("modules", modulesArray);
        for (LogicalPlan.ModuleMeta meta : dag.getAllModules()) {
            modulesArray.add(getLogicalModuleDetails(dag, meta));
        }
    }
    return result;
}

Example 27 with StreamMeta

use of com.datatorrent.stram.plan.logical.LogicalPlan.StreamMeta in project apex-core by apache.

the class OiOEndWindowTest method validateOiOImplementation.

@Test
public void validateOiOImplementation() throws Exception {
    LogicalPlan lp = new LogicalPlan();
    String workingDir = new File("target/validateOiOImplementation").getAbsolutePath();
    lp.setAttribute(Context.OperatorContext.STORAGE_AGENT, new AsyncFSStorageAgent(workingDir, null));
    TestInputOperator io = lp.addOperator("Input Operator", new TestInputOperator());
    FirstGenericOperator go = lp.addOperator("First Generic Operator", new FirstGenericOperator());
    SecondGenericOperator out = lp.addOperator("Second Generic Operator", new SecondGenericOperator());
    /*
     * This tests make sure that even if the application_window_count is different the endWindow() is called for
     * end_stream
     */
    lp.getOperatorMeta("Second Generic Operator").getAttributes().put(Context.OperatorContext.APPLICATION_WINDOW_COUNT, 2);
    StreamMeta stream = lp.addStream("Stream", io.output, go.input);
    StreamMeta stream1 = lp.addStream("Stream1", go.output, out.input);
    stream1.setLocality(Locality.THREAD_LOCAL);
    lp.validate();
    StramLocalCluster slc = new StramLocalCluster(lp);
    slc.run();
    Assert.assertEquals("End Window Count", FirstGenericOperator.endwindowCount, SecondGenericOperator.endwindowCount);
}

Example 28 with StreamMeta

use of com.datatorrent.stram.plan.logical.LogicalPlan.StreamMeta in project apex-core by apache.

the class OiOStreamTest method validateOiOImplementation.

@Test
public void validateOiOImplementation() throws Exception {
    LogicalPlan lp = new LogicalPlan();
    lp.setAttribute(com.datatorrent.api.Context.OperatorContext.STORAGE_AGENT, new MemoryStorageAgent());
    ThreadIdValidatingInputOperator io = lp.addOperator("Input Operator", new ThreadIdValidatingInputOperator());
    ThreadIdValidatingOutputOperator go = lp.addOperator("Output Operator", new ThreadIdValidatingOutputOperator());
    StreamMeta stream = lp.addStream("Stream", io.output, go.input);
    /* The first test makes sure that when they are not ThreadLocal they use different threads */
    ThreadIdValidatingOutputOperator.threadList.clear();
    lp.validate();
    StramLocalCluster slc = new StramLocalCluster(lp);
    slc.run();
    Assert.assertFalse("Thread Id", ThreadIdValidatingInputOperator.threadId == ThreadIdValidatingOutputOperator.threadId);
    /* This test makes sure that since they are ThreadLocal, they indeed share a thread */
    ThreadIdValidatingOutputOperator.threadList.clear();
    stream.setLocality(Locality.THREAD_LOCAL);
    lp.validate();
    slc = new StramLocalCluster(lp);
    slc.run();
    Assert.assertEquals("Thread Id", ThreadIdValidatingInputOperator.threadId, ThreadIdValidatingOutputOperator.threadId);
}

Example 29 with StreamMeta

use of com.datatorrent.stram.plan.logical.LogicalPlan.StreamMeta in project apex-core by apache.

the class StreamingContainerAgent method getDeployInfoList.

/**
   * Create deploy info for StramChild.
   *
   * @param operators
   * @return StreamingContainerContext
   */
public List<OperatorDeployInfo> getDeployInfoList(Collection<PTOperator> operators) {
    if (container.bufferServerAddress == null) {
        throw new AssertionError("No buffer server address assigned");
    }
    Map<OperatorDeployInfo, PTOperator> nodes = new LinkedHashMap<>();
    HashSet<PTOperator.PTOutput> publishers = new HashSet<>();
    PhysicalPlan physicalPlan = dnmgr.getPhysicalPlan();
    for (PTOperator oper : operators) {
        if (oper.getState() != State.PENDING_DEPLOY) {
            LOG.debug("Skipping deploy for operator {} state {}", oper, oper.getState());
            continue;
        }
        OperatorDeployInfo ndi = createOperatorDeployInfo(oper);
        nodes.put(ndi, oper);
        ndi.inputs = new ArrayList<>(oper.getInputs().size());
        ndi.outputs = new ArrayList<>(oper.getOutputs().size());
        for (PTOperator.PTOutput out : oper.getOutputs()) {
            final StreamMeta streamMeta = out.logicalStream;
            // buffer server or inline publisher
            OutputDeployInfo portInfo = new OutputDeployInfo();
            portInfo.declaredStreamId = streamMeta.getName();
            portInfo.portName = out.portName;
            try {
                portInfo.contextAttributes = streamMeta.getSource().getAttributes().clone();
            } catch (CloneNotSupportedException ex) {
                throw new RuntimeException("Cannot clone attributes", ex);
            }
            boolean outputUnified = false;
            for (PTOperator.PTInput input : out.sinks) {
                if (input.target.isUnifier()) {
                    outputUnified = true;
                    break;
                }
            }
            portInfo.contextAttributes.put(PortContext.IS_OUTPUT_UNIFIED, outputUnified);
            if (ndi.type == OperatorDeployInfo.OperatorType.UNIFIER) {
                // input attributes of the downstream operator
                for (InputPortMeta sink : streamMeta.getSinks()) {
                    try {
                        portInfo.contextAttributes = sink.getAttributes().clone();
                    } catch (CloneNotSupportedException e) {
                        throw new RuntimeException("Cannot clone attributes", e);
                    }
                    break;
                }
            }
            if (!out.isDownStreamInline()) {
                portInfo.bufferServerHost = oper.getContainer().bufferServerAddress.getHostName();
                portInfo.bufferServerPort = oper.getContainer().bufferServerAddress.getPort();
                portInfo.bufferServerToken = oper.getContainer().getBufferServerToken();
                // Build the stream codec configuration of all sinks connected to this port
                for (PTOperator.PTInput input : out.sinks) {
                    // Create mappings for all non-inline operators
                    if (input.target.getContainer() != out.source.getContainer()) {
                        final StreamCodec<?> streamCodec = getIdentifyingInputPortMeta(input).getStreamCodec();
                        final Integer id = physicalPlan.getStreamCodecIdentifier(streamCodec);
                        // TODO: replace with inputInfo.streamCodecs.putIfAbsent() after support for JDK 1.7 is dropped.
                        if (!portInfo.streamCodecs.containsKey(id)) {
                            portInfo.streamCodecs.put(id, streamCodec);
                        }
                    }
                }
            }
            ndi.outputs.add(portInfo);
            publishers.add(out);
        }
    }
    for (Map.Entry<OperatorDeployInfo, PTOperator> operEntry : nodes.entrySet()) {
        OperatorDeployInfo ndi = operEntry.getKey();
        PTOperator oper = operEntry.getValue();
        for (PTOperator.PTInput in : oper.getInputs()) {
            final StreamMeta streamMeta = in.logicalStream;
            if (streamMeta.getSource() == null) {
                throw new AssertionError("source is null: " + in);
            }
            PTOperator.PTOutput sourceOutput = in.source;
            InputDeployInfo inputInfo = new InputDeployInfo();
            inputInfo.declaredStreamId = streamMeta.getName();
            inputInfo.portName = in.portName;
            InputPortMeta inputPortMeta = getInputPortMeta(oper.getOperatorMeta(), streamMeta);
            if (inputPortMeta != null) {
                try {
                    inputInfo.contextAttributes = inputPortMeta.getAttributes().clone();
                } catch (CloneNotSupportedException e) {
                    throw new RuntimeException("Cannot clone attributes", e);
                }
            }
            if (inputInfo.contextAttributes == null && ndi.type == OperatorDeployInfo.OperatorType.UNIFIER) {
                try {
                    inputInfo.contextAttributes = in.source.logicalStream.getSource().getAttributes().clone();
                } catch (CloneNotSupportedException e) {
                    throw new RuntimeException("Cannot clone attributes", e);
                }
            }
            inputInfo.sourceNodeId = sourceOutput.source.getId();
            inputInfo.sourcePortName = sourceOutput.portName;
            if (in.partitions != null && in.partitions.mask != 0) {
                inputInfo.partitionMask = in.partitions.mask;
                inputInfo.partitionKeys = in.partitions.partitions;
            }
            if (sourceOutput.source.getContainer() == oper.getContainer()) {
                // both operators in same container
                if (!publishers.contains(sourceOutput)) {
                    throw new AssertionError("Source not deployed for container local stream " + sourceOutput + " " + in);
                }
                if (streamMeta.getLocality() == Locality.THREAD_LOCAL) {
                    inputInfo.locality = Locality.THREAD_LOCAL;
                    ndi.type = OperatorType.OIO;
                } else {
                    inputInfo.locality = Locality.CONTAINER_LOCAL;
                }
            } else {
                // buffer server input
                PTContainer container = sourceOutput.source.getContainer();
                InetSocketAddress addr = container.bufferServerAddress;
                if (addr == null) {
                    throw new AssertionError("upstream address not assigned: " + sourceOutput);
                }
                inputInfo.bufferServerHost = addr.getHostName();
                inputInfo.bufferServerPort = addr.getPort();
                inputInfo.bufferServerToken = container.getBufferServerToken();
            }
            // On the input side there is a unlikely scenario of partitions even for inline stream that is being
            // handled. Always specifying a stream codec configuration in case that scenario happens.
            final StreamCodec<?> streamCodec = getIdentifyingInputPortMeta(in).getStreamCodec();
            final Integer id = physicalPlan.getStreamCodecIdentifier(streamCodec);
            // TODO: replace with inputInfo.streamCodecs.putIfAbsent() after support for JDK 1.7 is dropped.
            if (!inputInfo.streamCodecs.containsKey(id)) {
                inputInfo.streamCodecs.put(id, streamCodec);
            }
            ndi.inputs.add(inputInfo);
        }
    }
    return new ArrayList<>(nodes.keySet());
}

Example 30 with StreamMeta

use of com.datatorrent.stram.plan.logical.LogicalPlan.StreamMeta in project apex-core by apache.

the class StreamingContainerAgent method getInputPortMeta.

public static InputPortMeta getInputPortMeta(LogicalPlan.OperatorMeta operatorMeta, StreamMeta streamMeta) {
    InputPortMeta inputPortMeta = null;
    Map<InputPortMeta, StreamMeta> inputStreams = operatorMeta.getInputStreams();
    for (Map.Entry<InputPortMeta, StreamMeta> entry : inputStreams.entrySet()) {
        if (entry.getValue() == streamMeta) {
            inputPortMeta = entry.getKey();
            break;
        }
    }
    return inputPortMeta;
}