Examples with BaseFlowStep cascading.flow.planner.BaseFlowStep used on opensource projects

Example 1 with BaseFlowStep

use of cascading.flow.planner.BaseFlowStep in project ambrose by twitter.

the class AmbroseCascadingGraphConverter method convert.

/**
   * Converts the flowStep that generated from cascading to a Map of DAGNode and its name to be used
   * to build Ambrose Graph.
   */
public void convert() {
    // returns a set of the nodes contained in this graph
    Set vertices = jobsGraph.vertexSet();
    // create ambrose nodes
    for (Object vertex : vertices) {
        BaseFlowStep step = (BaseFlowStep) vertex;
        CascadingJob job = new CascadingJob();
        job.setFeatures(getNodeFeatures(step));
        String name = step.getName();
        DAGNode<CascadingJob> node = new DAGNode<CascadingJob>(name, job);
        dagNamesMap.put(name, node);
    }
    // loop again to set the successors for each node after nodes are created
    for (Object vertex : vertices) {
        BaseFlowStep step = (BaseFlowStep) vertex;
        String name = step.getName();
        DAGNode<CascadingJob> node = dagNamesMap.get(name);
        node.setSuccessors(getNodeSuccessors(vertex));
    }
}

Example 2 with BaseFlowStep

use of cascading.flow.planner.BaseFlowStep in project ambrose by twitter.

the class AmbroseCascadingNotifier method onStarting.

/**
   * The onStarting event is fired when a Flow instance receives the start() message. A Flow is cut
   * down into executing units called stepFlow. A stepFlow contains a stepFlowJob which represents
   * the mapreduce job to be submitted to Hadoop. The ambrose graph is constructed from the step
   * graph found in flow object.
   *
   * @param flow the flow.
   */
@Override
@SuppressWarnings("unchecked")
public void onStarting(Flow flow) {
    // init flow
    List<FlowStep> steps = flow.getFlowSteps();
    totalNumberOfJobs = steps.size();
    currentFlowId = flow.getID();
    Properties props = new Properties();
    props.putAll(flow.getConfigAsProperties());
    try {
        statsWriteService.initWriteService(props);
    } catch (IOException e) {
        LOG.error("Failed to initialize statsWriteService", e);
    }
    // convert graph from cascading to jgrapht
    FlowStepGraph flowStepGraph = Flows.getStepGraphFrom(flow);
    DirectedGraph graph = new DefaultDirectedGraph<BaseFlowStep, FlowGraphEdge>(new EdgeFactory<BaseFlowStep, FlowGraphEdge>() {

        @Override
        public FlowGraphEdge createEdge(BaseFlowStep src, BaseFlowStep dest) {
            return new FlowGraphEdge(src.getID(), dest.getID());
        }
    });
    for (FlowStep v : flowStepGraph.vertexSet()) {
        graph.addVertex(v);
    }
    for (ProcessEdge e : flowStepGraph.edgeSet()) {
        graph.addEdge(e.getSourceProcessID(), e.getSinkProcessID());
    }
    // convert graph from jgrapht to ambrose
    AmbroseCascadingGraphConverter converter = new AmbroseCascadingGraphConverter(graph, nodesByName);
    converter.convert();
    AmbroseUtils.sendDagNodeNameMap(statsWriteService, currentFlowId, nodesByName);
}

Example 3 with BaseFlowStep

use of cascading.flow.planner.BaseFlowStep in project ambrose by twitter.

the class AmbroseCascadingGraphConverter method convert.

/**
   * Converts the flowStep that generated from cascading to a Map of DAGNode and its name to be used
   * to build Ambrose Graph.
   */
public void convert() {
    // returns a set of the nodes contained in this graph
    Set vertices = jobsGraph.vertexSet();
    // create ambrose nodes
    for (Object vertex : vertices) {
        BaseFlowStep step = (BaseFlowStep) vertex;
        CascadingJob job = new CascadingJob();
        job.setFeatures(getNodeFeatures(step));
        String name = step.getName();
        DAGNode<CascadingJob> node = new DAGNode<CascadingJob>(name, job);
        dagNamesMap.put(name, node);
    }
    // loop again to set the successors for each node after nodes are created
    for (Object vertex : vertices) {
        BaseFlowStep step = (BaseFlowStep) vertex;
        String name = step.getName();
        DAGNode<CascadingJob> node = dagNamesMap.get(name);
        node.setSuccessors(getNodeSuccessors(vertex));
    }
}

Example 4 with BaseFlowStep

use of cascading.flow.planner.BaseFlowStep in project ambrose by twitter.

the class AmbroseCascadingGraphConverter method getNodeSuccessors.

/**
   * Return a Collection of successor nodes of a certain vertex.
   *
   * @param vertex the step or node its successors nodes will be returned.
   * @return collection of successor DAGNodes for each node.
   */
protected Collection<DAGNode<? extends Job>> getNodeSuccessors(Object vertex) {
    Collection<DAGNode<? extends Job>> nodeSuccessors = Sets.newHashSet();
    List successorNodes = Graphs.successorListOf(jobsGraph, vertex);
    for (Object node : successorNodes) {
        BaseFlowStep step = (BaseFlowStep) node;
        String name = step.getName();
        nodeSuccessors.add(dagNamesMap.get(name));
    }
    return nodeSuccessors;
}

Example 5 with BaseFlowStep

use of cascading.flow.planner.BaseFlowStep in project ambrose by twitter.

the class AmbroseCascadingGraphConverter method getNodeSuccessors.

/**
   * Return a Collection of successor nodes of a certain vertex.
   *
   * @param vertex the step or node its successors nodes will be returned.
   * @return collection of successor DAGNodes for each node.
   */
protected Collection<DAGNode<? extends Job>> getNodeSuccessors(Object vertex) {
    Collection<DAGNode<? extends Job>> nodeSuccessors = Sets.newHashSet();
    List successorNodes = Graphs.successorListOf(jobsGraph, vertex);
    for (Object node : successorNodes) {
        BaseFlowStep step = (BaseFlowStep) node;
        String name = step.getName();
        nodeSuccessors.add(dagNamesMap.get(name));
    }
    return nodeSuccessors;
}