Examples with FlowStep cascading.flow.FlowStep used on opensource projects

Example 1 with FlowStep

use of cascading.flow.FlowStep in project ambrose by twitter.

the class AmbroseCascadingNotifier method onStarting.

/**
   * The onStarting event is fired when a Flow instance receives the start() message. A Flow is cut
   * down into executing units called stepFlow. A stepFlow contains a stepFlowJob which represents
   * the mapreduce job to be submitted to Hadoop. The ambrose graph is constructed from the step
   * graph found in flow object.
   *
   * @param flow the flow.
   */
@Override
@SuppressWarnings("unchecked")
public void onStarting(Flow flow) {
    // init flow
    List<FlowStep> steps = flow.getFlowSteps();
    totalNumberOfJobs = steps.size();
    currentFlowId = flow.getID();
    Properties props = new Properties();
    props.putAll(flow.getConfigAsProperties());
    try {
        statsWriteService.initWriteService(props);
    } catch (IOException e) {
        LOG.error("Failed to initialize statsWriteService", e);
    }
    // convert graph from cascading to jgrapht
    FlowStepGraph flowStepGraph = Flows.getStepGraphFrom(flow);
    DirectedGraph graph = new DefaultDirectedGraph<BaseFlowStep, FlowGraphEdge>(new EdgeFactory<BaseFlowStep, FlowGraphEdge>() {

        @Override
        public FlowGraphEdge createEdge(BaseFlowStep src, BaseFlowStep dest) {
            return new FlowGraphEdge(src.getID(), dest.getID());
        }
    });
    for (FlowStep v : flowStepGraph.vertexSet()) {
        graph.addVertex(v);
    }
    for (ProcessEdge e : flowStepGraph.edgeSet()) {
        graph.addEdge(e.getSourceProcessID(), e.getSinkProcessID());
    }
    // convert graph from jgrapht to ambrose
    AmbroseCascadingGraphConverter converter = new AmbroseCascadingGraphConverter(graph, nodesByName);
    converter.convert();
    AmbroseUtils.sendDagNodeNameMap(statsWriteService, currentFlowId, nodesByName);
}

Example 2 with FlowStep

use of cascading.flow.FlowStep in project ambrose by twitter.

the class AmbroseCascadingNotifier method onStarting.

/**
   * The onStarting event is fired when a Flow instance receives the start() message. A Flow is cut
   * down into executing units called stepFlow. A stepFlow contains a stepFlowJob which represents
   * the mapreduce job to be submitted to Hadoop. The ambrose graph is constructed from the step
   * graph found in flow object.
   *
   * @param flow the flow.
   */
@Override
@SuppressWarnings("unchecked")
public void onStarting(Flow flow) {
    // init flow
    List<FlowStep> steps = flow.getFlowSteps();
    totalNumberOfJobs = steps.size();
    currentFlowId = flow.getID();
    Properties props = new Properties();
    props.putAll(flow.getConfigAsProperties());
    try {
        statsWriteService.initWriteService(props);
    } catch (IOException e) {
        LOG.error("Failed to initialize statsWriteService", e);
    }
    // convert graph from cascading to ambrose
    AmbroseCascadingGraphConverter converter = new AmbroseCascadingGraphConverter(Flows.getStepGraphFrom(flow), nodesByName);
    converter.convert();
    AmbroseUtils.sendDagNodeNameMap(statsWriteService, currentFlowId, nodesByName);
}