Examples with BaseServiceNodeImpl org.apache.gobblin.runtime.spec_executorInstance.BaseServiceNodeImpl used on opensource projects

Example 1 with BaseServiceNodeImpl

use of org.apache.gobblin.runtime.spec_executorInstance.BaseServiceNodeImpl in project incubator-gobblin by apache.

the class MultiHopsFlowToJobSpecCompilerTest method setUp.

@BeforeClass
public void setUp() throws Exception {
    // Create dir for template catalog
    FileUtils.forceMkdir(new File(TEST_TEMPLATE_CATALOG_PATH));
    // Create template to use in test
    List<String> templateEntries = new ArrayList<>();
    templateEntries.add("testProperty1 = \"testValue1\"");
    templateEntries.add("testProperty2 = \"test.Value1\"");
    templateEntries.add("testProperty3 = 100");
    FileUtils.writeLines(new File(TEST_TEMPLATE_CATALOG_PATH + "/" + TEST_TEMPLATE_NAME), templateEntries);
    // Initialize complier with template catalog
    Properties compilerWithTemplateCatalogProperties = new Properties();
    compilerWithTemplateCatalogProperties.setProperty(ServiceConfigKeys.TEMPLATE_CATALOGS_FULLY_QUALIFIED_PATH_KEY, TEST_TEMPLATE_CATALOG_URI);
    // Initialize compiler with common useful properties
    String testPath = TEST_SOURCE_NAME + "," + TEST_HOP_NAME_A + "," + TEST_HOP_NAME_B + "," + TEST_SINK_NAME;
    compilerWithTemplateCatalogProperties.setProperty(ServiceConfigKeys.POLICY_BASED_DATA_MOVEMENT_PATH, testPath);
    this.compilerWithTemplateCalague = new MultiHopsFlowToJobSpecCompiler(ConfigUtils.propertiesToConfig(compilerWithTemplateCatalogProperties));
    vertexSource = new BaseServiceNodeImpl(TEST_SOURCE_NAME);
    vertexHopA = new BaseServiceNodeImpl(TEST_HOP_NAME_A);
    vertexHopB = new BaseServiceNodeImpl(TEST_HOP_NAME_B);
    vertexHopC = new BaseServiceNodeImpl(TEST_HOP_NAME_C);
    vertexSink = new BaseServiceNodeImpl(TEST_SINK_NAME);
}

Example 2 with BaseServiceNodeImpl

use of org.apache.gobblin.runtime.spec_executorInstance.BaseServiceNodeImpl in project incubator-gobblin by apache.

the class MultiHopsFlowToJobSpecCompiler method userSpecifiedPathVerificator.

// If path specified not existed, return false;
// else return true.
private boolean userSpecifiedPathVerificator(Map<Spec, SpecExecutor> specExecutorInstanceMap, FlowSpec flowSpec) {
    Map<Spec, SpecExecutor> tmpSpecExecutorInstanceMap = new HashMap<>();
    List<String> userSpecfiedPath = Arrays.asList(optionalUserSpecifiedPath.get().split(","));
    for (int i = 0; i < userSpecfiedPath.size() - 1; i++) {
        ServiceNode sourceNode = new BaseServiceNodeImpl(userSpecfiedPath.get(i));
        ServiceNode targetNode = new BaseServiceNodeImpl(userSpecfiedPath.get(i + 1));
        if (weightedGraph.containsVertex(sourceNode) && weightedGraph.containsVertex(targetNode) && weightedGraph.containsEdge(sourceNode, targetNode)) {
            tmpSpecExecutorInstanceMap.put(convertHopToJobSpec(sourceNode, targetNode, flowSpec), (((LoadBasedFlowEdgeImpl) weightedGraph.getEdge(sourceNode, targetNode)).getSpecExecutorInstance()));
        } else {
            log.error("User Specified Path is invalid");
            return false;
        }
    }
    specExecutorInstanceMap.putAll(tmpSpecExecutorInstanceMap);
    return true;
}

Example 3 with BaseServiceNodeImpl

use of org.apache.gobblin.runtime.spec_executorInstance.BaseServiceNodeImpl in project incubator-gobblin by apache.

the class MultiHopsFlowToJobSpecCompiler method findPath.

// Basically a dijkstra path finding for connecting source and sink by multiple hops in between.
// If there's any user-specified prioritization, conduct the DFS and see if the user-specified path is available.
// there's no updates on TopologySpec, or user should be aware of the possibility
// that a topologySpec not being reflected in findPath.
private void findPath(Map<Spec, SpecExecutor> specExecutorInstanceMap, Spec spec) {
    inMemoryWeightGraphGenerator();
    FlowSpec flowSpec = (FlowSpec) spec;
    if (optionalUserSpecifiedPath.isPresent()) {
        log.info("Starting to evaluate user's specified path ... ");
        if (userSpecifiedPathVerificator(specExecutorInstanceMap, flowSpec)) {
            log.info("User specified path[ " + optionalUserSpecifiedPath.get() + "] successfully verified.");
            return;
        } else {
            log.error("Will not execute user specified path[ " + optionalUserSpecifiedPath.get() + "]");
            log.info("Start to execute FlowCompiler's algorithm for valid data movement path");
        }
    }
    ServiceNode sourceNode = new BaseServiceNodeImpl(flowSpec.getConfig().getString(ServiceConfigKeys.FLOW_SOURCE_IDENTIFIER_KEY));
    ServiceNode targetNode = new BaseServiceNodeImpl(flowSpec.getConfig().getString(ServiceConfigKeys.FLOW_DESTINATION_IDENTIFIER_KEY));
    List<FlowEdge> resultEdgePath = dijkstraBasedPathFindingHelper(sourceNode, targetNode, this.weightedGraph);
    for (int i = 0; i < resultEdgePath.size(); i++) {
        FlowEdge tmpFlowEdge = resultEdgePath.get(i);
        ServiceNode edgeSrcNode = ((LoadBasedFlowEdgeImpl) tmpFlowEdge).getSourceNode();
        ServiceNode edgeTgtNode = ((LoadBasedFlowEdgeImpl) tmpFlowEdge).getTargetNode();
        specExecutorInstanceMap.put(convertHopToJobSpec(edgeSrcNode, edgeTgtNode, flowSpec), ((LoadBasedFlowEdgeImpl) (resultEdgePath.get(i))).getSpecExecutorInstance());
    }
}

Example 4 with BaseServiceNodeImpl

use of org.apache.gobblin.runtime.spec_executorInstance.BaseServiceNodeImpl in project incubator-gobblin by apache.

the class MultiHopsFlowToJobSpecCompiler method weightGraphGenerateHelper.

// Helper function for transform TopologySpecMap into a weightedDirectedGraph.
private void weightGraphGenerateHelper(TopologySpec topologySpec) {
    try {
        Map<ServiceNode, ServiceNode> capabilities = topologySpec.getSpecExecutor().getCapabilities().get();
        for (Map.Entry<ServiceNode, ServiceNode> capability : capabilities.entrySet()) {
            BaseServiceNodeImpl sourceNode = new BaseServiceNodeImpl(capability.getKey().getNodeName());
            BaseServiceNodeImpl targetNode = new BaseServiceNodeImpl(capability.getValue().getNodeName());
            if (!weightedGraph.containsVertex(sourceNode)) {
                weightedGraph.addVertex(sourceNode);
            }
            if (!weightedGraph.containsVertex(targetNode)) {
                weightedGraph.addVertex(targetNode);
            }
            FlowEdge flowEdge = new LoadBasedFlowEdgeImpl(sourceNode, targetNode, defaultFlowEdgeProps, topologySpec.getSpecExecutor());
            // In Multi-Graph if flowEdge existed, just skip it.
            if (!weightedGraph.containsEdge(flowEdge)) {
                weightedGraph.addEdge(sourceNode, targetNode, flowEdge);
            }
        }
    } catch (InterruptedException | ExecutionException e) {
        Instrumented.markMeter(this.flowCompilationFailedMeter);
        throw new RuntimeException("Cannot determine topology capabilities", e);
    }
}