Examples with FlowEdge org.apache.gobblin.runtime.api.FlowEdge used on opensource projects

Example 6 with FlowEdge

use of org.apache.gobblin.runtime.api.FlowEdge in project incubator-gobblin by apache.

the class MultiHopsFlowToJobSpecCompiler method findPath.

// Basically a dijkstra path finding for connecting source and sink by multiple hops in between.
// If there's any user-specified prioritization, conduct the DFS and see if the user-specified path is available.
// there's no updates on TopologySpec, or user should be aware of the possibility
// that a topologySpec not being reflected in findPath.
private void findPath(Map<Spec, SpecExecutor> specExecutorInstanceMap, Spec spec) {
    inMemoryWeightGraphGenerator();
    FlowSpec flowSpec = (FlowSpec) spec;
    if (optionalUserSpecifiedPath.isPresent()) {
        log.info("Starting to evaluate user's specified path ... ");
        if (userSpecifiedPathVerificator(specExecutorInstanceMap, flowSpec)) {
            log.info("User specified path[ " + optionalUserSpecifiedPath.get() + "] successfully verified.");
            return;
        } else {
            log.error("Will not execute user specified path[ " + optionalUserSpecifiedPath.get() + "]");
            log.info("Start to execute FlowCompiler's algorithm for valid data movement path");
        }
    }
    ServiceNode sourceNode = new BaseServiceNodeImpl(flowSpec.getConfig().getString(ServiceConfigKeys.FLOW_SOURCE_IDENTIFIER_KEY));
    ServiceNode targetNode = new BaseServiceNodeImpl(flowSpec.getConfig().getString(ServiceConfigKeys.FLOW_DESTINATION_IDENTIFIER_KEY));
    List<FlowEdge> resultEdgePath = dijkstraBasedPathFindingHelper(sourceNode, targetNode, this.weightedGraph);
    for (int i = 0; i < resultEdgePath.size(); i++) {
        FlowEdge tmpFlowEdge = resultEdgePath.get(i);
        ServiceNode edgeSrcNode = ((LoadBasedFlowEdgeImpl) tmpFlowEdge).getSourceNode();
        ServiceNode edgeTgtNode = ((LoadBasedFlowEdgeImpl) tmpFlowEdge).getTargetNode();
        specExecutorInstanceMap.put(convertHopToJobSpec(edgeSrcNode, edgeTgtNode, flowSpec), ((LoadBasedFlowEdgeImpl) (resultEdgePath.get(i))).getSpecExecutorInstance());
    }
}

Example 7 with FlowEdge

use of org.apache.gobblin.runtime.api.FlowEdge in project incubator-gobblin by apache.

the class MultiHopsFlowToJobSpecCompiler method convertHopToJobSpec.

/**
 * A naive implementation of resolving templates in each JobSpec among Multi-hop FlowSpec.
 * Handle the case when edge is not specified.
 * Always select the first available template.
 */
private JobSpec convertHopToJobSpec(ServiceNode sourceNode, ServiceNode targetNode, FlowSpec flowSpec) {
    FlowEdge flowEdge = weightedGraph.getAllEdges(sourceNode, targetNode).iterator().next();
    URI templateURI = getTemplateURI(sourceNode, targetNode, flowSpec, flowEdge);
    return buildJobSpec(sourceNode, targetNode, templateURI, flowSpec);
}

Example 8 with FlowEdge

use of org.apache.gobblin.runtime.api.FlowEdge in project incubator-gobblin by apache.

the class MultiHopsFlowToJobSpecCompiler method weightGraphGenerateHelper.

// Helper function for transform TopologySpecMap into a weightedDirectedGraph.
private void weightGraphGenerateHelper(TopologySpec topologySpec) {
    try {
        Map<ServiceNode, ServiceNode> capabilities = topologySpec.getSpecExecutor().getCapabilities().get();
        for (Map.Entry<ServiceNode, ServiceNode> capability : capabilities.entrySet()) {
            BaseServiceNodeImpl sourceNode = new BaseServiceNodeImpl(capability.getKey().getNodeName());
            BaseServiceNodeImpl targetNode = new BaseServiceNodeImpl(capability.getValue().getNodeName());
            if (!weightedGraph.containsVertex(sourceNode)) {
                weightedGraph.addVertex(sourceNode);
            }
            if (!weightedGraph.containsVertex(targetNode)) {
                weightedGraph.addVertex(targetNode);
            }
            FlowEdge flowEdge = new LoadBasedFlowEdgeImpl(sourceNode, targetNode, defaultFlowEdgeProps, topologySpec.getSpecExecutor());
            // In Multi-Graph if flowEdge existed, just skip it.
            if (!weightedGraph.containsEdge(flowEdge)) {
                weightedGraph.addEdge(sourceNode, targetNode, flowEdge);
            }
        }
    } catch (InterruptedException | ExecutionException e) {
        Instrumented.markMeter(this.flowCompilationFailedMeter);
        throw new RuntimeException("Cannot determine topology capabilities", e);
    }
}

Example 9 with FlowEdge

use of org.apache.gobblin.runtime.api.FlowEdge in project incubator-gobblin by apache.

the class StaticServicePolicy method populateBlackListedEdges.

@Override
public void populateBlackListedEdges(DirectedWeightedMultigraph<ServiceNode, FlowEdge> graph) {
    for (ServiceNode node : serviceNodes) {
        if (graph.containsVertex(node)) {
            blacklistedEdges.addAll(graph.incomingEdgesOf(node));
            blacklistedEdges.addAll(graph.outgoingEdgesOf(node));
        } else {
            log.info("The graph " + graph + " doesn't contains node " + node.toString());
        }
    }
    for (FlowEdge flowEdge : flowEdges) {
        if (graph.containsEdge(flowEdge)) {
            blacklistedEdges.add(flowEdge);
        } else {
            log.info("The graph " + graph + "doesn't contains edge " + flowEdge.toString());
        }
    }
}