Examples with MultiHopsFlowToJobSpecCompiler org.apache.gobblin.service.modules.flow.MultiHopsFlowToJobSpecCompiler used on opensource projects

Example 1 with MultiHopsFlowToJobSpecCompiler

use of org.apache.gobblin.service.modules.flow.MultiHopsFlowToJobSpecCompiler in project incubator-gobblin by apache.

the class MultiHopsFlowToJobSpecCompilerTest method setUp.

@BeforeClass
public void setUp() throws Exception {
    // Create dir for template catalog
    FileUtils.forceMkdir(new File(TEST_TEMPLATE_CATALOG_PATH));
    // Create template to use in test
    List<String> templateEntries = new ArrayList<>();
    templateEntries.add("testProperty1 = \"testValue1\"");
    templateEntries.add("testProperty2 = \"test.Value1\"");
    templateEntries.add("testProperty3 = 100");
    FileUtils.writeLines(new File(TEST_TEMPLATE_CATALOG_PATH + "/" + TEST_TEMPLATE_NAME), templateEntries);
    // Initialize complier with template catalog
    Properties compilerWithTemplateCatalogProperties = new Properties();
    compilerWithTemplateCatalogProperties.setProperty(ServiceConfigKeys.TEMPLATE_CATALOGS_FULLY_QUALIFIED_PATH_KEY, TEST_TEMPLATE_CATALOG_URI);
    // Initialize compiler with common useful properties
    String testPath = TEST_SOURCE_NAME + "," + TEST_HOP_NAME_A + "," + TEST_HOP_NAME_B + "," + TEST_SINK_NAME;
    compilerWithTemplateCatalogProperties.setProperty(ServiceConfigKeys.POLICY_BASED_DATA_MOVEMENT_PATH, testPath);
    this.compilerWithTemplateCalague = new MultiHopsFlowToJobSpecCompiler(ConfigUtils.propertiesToConfig(compilerWithTemplateCatalogProperties));
    vertexSource = new BaseServiceNodeImpl(TEST_SOURCE_NAME);
    vertexHopA = new BaseServiceNodeImpl(TEST_HOP_NAME_A);
    vertexHopB = new BaseServiceNodeImpl(TEST_HOP_NAME_B);
    vertexHopC = new BaseServiceNodeImpl(TEST_HOP_NAME_C);
    vertexSink = new BaseServiceNodeImpl(TEST_SINK_NAME);
}

Example 2 with MultiHopsFlowToJobSpecCompiler

use of org.apache.gobblin.service.modules.flow.MultiHopsFlowToJobSpecCompiler in project incubator-gobblin by apache.

the class MultiHopsFlowToJobSpecCompilerTest method testServicePolicy.

@Test
public void testServicePolicy() {
    // Initialize compiler with some blacklist properties
    Properties properties = new Properties();
    properties.setProperty(ServiceConfigKeys.TEMPLATE_CATALOGS_FULLY_QUALIFIED_PATH_KEY, TEST_TEMPLATE_CATALOG_URI);
    String testPath = TEST_SOURCE_NAME + "," + TEST_HOP_NAME_A + "," + TEST_HOP_NAME_B + "," + TEST_SINK_NAME;
    properties.setProperty(ServiceConfigKeys.POLICY_BASED_DATA_MOVEMENT_PATH, testPath);
    properties.setProperty(ServiceConfigKeys.POLICY_BASED_BLOCKED_NODES, "testHopA");
    MultiHopsFlowToJobSpecCompiler compiler = new MultiHopsFlowToJobSpecCompiler(ConfigUtils.propertiesToConfig(properties));
    FlowSpec flowSpec = initFlowSpec();
    TopologySpec topologySpec = initTopologySpec(TOPOLOGY_SPEC_STORE_DIR, TEST_SOURCE_NAME, TEST_HOP_NAME_A, TEST_HOP_NAME_B, TEST_SINK_NAME);
    compiler.onAddSpec(topologySpec);
    // invocation of compileFlow trigger the weighedGraph construction
    compiler.compileFlow(flowSpec);
    compiler.servicePolicy.populateBlackListedEdges(compiler.getWeightedGraph());
    Assert.assertEquals(compiler.servicePolicy.getBlacklistedEdges().size(), 2);
    FlowEdge edgeSrc2A = new LoadBasedFlowEdgeImpl(vertexSource, vertexHopA, topologySpec.getSpecExecutor());
    FlowEdge edgeA2B = new LoadBasedFlowEdgeImpl(vertexHopA, vertexHopB, topologySpec.getSpecExecutor());
    Assert.assertTrue(compiler.servicePolicy.getBlacklistedEdges().contains(edgeSrc2A));
    Assert.assertTrue(compiler.servicePolicy.getBlacklistedEdges().contains(edgeA2B));
}