Examples with FlowSpec org.apache.gobblin.runtime.api.FlowSpec used on opensource projects

Example 16 with FlowSpec

use of org.apache.gobblin.runtime.api.FlowSpec in project incubator-gobblin by apache.

the class GobblinServiceManagerTest method testGitCreate.

@Test(dependsOnMethods = "testDelete")
public void testGitCreate() throws Exception {
    // push a new config file
    File testFlowFile = new File(GIT_CLONE_DIR + "/gobblin-config/testGroup/testFlow.pull");
    testFlowFile.getParentFile().mkdirs();
    Files.write("flow.name=testFlow\nflow.group=testGroup\nparam1=value20\n", testFlowFile, Charsets.UTF_8);
    Collection<Spec> specs = this.gobblinServiceManager.flowCatalog.getSpecs();
    Assert.assertTrue(specs.size() == 0);
    // add, commit, push
    this.gitForPush.add().addFilepattern("gobblin-config/testGroup/testFlow.pull").call();
    this.gitForPush.commit().setMessage("second commit").call();
    this.gitForPush.push().setRemote("origin").setRefSpecs(new RefSpec("master")).call();
    // polling is every 5 seconds, so wait twice as long and check
    TimeUnit.SECONDS.sleep(10);
    specs = this.gobblinServiceManager.flowCatalog.getSpecs();
    Assert.assertTrue(specs.size() == 1);
    FlowSpec spec = (FlowSpec) (specs.iterator().next());
    Assert.assertEquals(spec.getUri(), new URI("gobblin-flow:/testGroup/testFlow"));
    Assert.assertEquals(spec.getConfig().getString(ConfigurationKeys.FLOW_NAME_KEY), "testFlow");
    Assert.assertEquals(spec.getConfig().getString(ConfigurationKeys.FLOW_GROUP_KEY), "testGroup");
    Assert.assertEquals(spec.getConfig().getString("param1"), "value20");
}

Example 17 with FlowSpec

use of org.apache.gobblin.runtime.api.FlowSpec in project incubator-gobblin by apache.

the class IdentityFlowToJobSpecCompilerTest method testNoJobSpecCompilation.

@Test
public void testNoJobSpecCompilation() {
    FlowSpec flowSpec = initFlowSpec(TEST_FLOW_GROUP, TEST_FLOW_NAME, "unsupportedSource", "unsupportedSink");
    // Run compiler on flowSpec
    Map<Spec, SpecExecutor> specExecutorMapping = this.compilerWithTemplateCalague.compileFlow(flowSpec);
    // Assert pre-requisites
    Assert.assertNotNull(specExecutorMapping, "Expected non null mapping.");
    Assert.assertTrue(specExecutorMapping.size() == 0, "Exepected 1 executor for FlowSpec.");
}

Example 18 with FlowSpec

use of org.apache.gobblin.runtime.api.FlowSpec in project incubator-gobblin by apache.

the class IdentityFlowToJobSpecCompilerTest method testCompilerWithTemplateCatalog.

@Test
public void testCompilerWithTemplateCatalog() {
    FlowSpec flowSpec = initFlowSpec();
    // Run compiler on flowSpec
    Map<Spec, SpecExecutor> specExecutorMapping = this.compilerWithTemplateCalague.compileFlow(flowSpec);
    // Assert pre-requisites
    Assert.assertNotNull(specExecutorMapping, "Expected non null mapping.");
    Assert.assertTrue(specExecutorMapping.size() == 1, "Exepected 1 executor for FlowSpec.");
    // Assert FlowSpec compilation
    Spec spec = specExecutorMapping.keySet().iterator().next();
    Assert.assertTrue(spec instanceof JobSpec, "Expected JobSpec compiled from FlowSpec.");
    // Assert JobSpec properties
    JobSpec jobSpec = (JobSpec) spec;
    Assert.assertEquals(jobSpec.getConfig().getString("testProperty1"), "testValue1");
    Assert.assertEquals(jobSpec.getConfig().getString("testProperty2"), "test.Value1");
    Assert.assertEquals(jobSpec.getConfig().getString("testProperty3"), "100");
    Assert.assertEquals(jobSpec.getConfig().getString(ServiceConfigKeys.FLOW_SOURCE_IDENTIFIER_KEY), TEST_SOURCE_NAME);
    Assert.assertFalse(jobSpec.getConfig().hasPath(ConfigurationKeys.JOB_SCHEDULE_KEY));
    Assert.assertEquals(jobSpec.getConfig().getString(ConfigurationKeys.JOB_NAME_KEY), TEST_FLOW_NAME);
    Assert.assertEquals(jobSpec.getConfig().getString(ConfigurationKeys.JOB_GROUP_KEY), TEST_FLOW_GROUP);
    Assert.assertEquals(jobSpec.getConfig().getString(ConfigurationKeys.FLOW_NAME_KEY), TEST_FLOW_NAME);
    Assert.assertEquals(jobSpec.getConfig().getString(ConfigurationKeys.FLOW_GROUP_KEY), TEST_FLOW_GROUP);
    Assert.assertTrue(jobSpec.getConfig().hasPath(ConfigurationKeys.FLOW_EXECUTION_ID_KEY));
}

Example 19 with FlowSpec

use of org.apache.gobblin.runtime.api.FlowSpec in project incubator-gobblin by apache.

the class MultiHopsFlowToJobSpecCompilerTest method testWeightedGraphConstruction.

@Test
public void testWeightedGraphConstruction() {
    FlowSpec flowSpec = initFlowSpec();
    TopologySpec topologySpec = initTopologySpec(TOPOLOGY_SPEC_STORE_DIR, TEST_SOURCE_NAME, TEST_HOP_NAME_A, TEST_HOP_NAME_B, TEST_SINK_NAME);
    this.compilerWithTemplateCalague.onAddSpec(topologySpec);
    // invocation of compileFlow trigger the weighedGraph construction
    this.compilerWithTemplateCalague.compileFlow(flowSpec);
    DirectedWeightedMultigraph<ServiceNode, FlowEdge> weightedGraph = compilerWithTemplateCalague.getWeightedGraph();
    Assert.assertTrue(weightedGraph.containsVertex(vertexSource));
    Assert.assertTrue(weightedGraph.containsVertex(vertexHopA));
    Assert.assertTrue(weightedGraph.containsVertex(vertexHopB));
    Assert.assertTrue(weightedGraph.containsVertex(vertexSink));
    FlowEdge edgeSrc2A = new LoadBasedFlowEdgeImpl(vertexSource, vertexHopA, topologySpec.getSpecExecutor());
    FlowEdge edgeA2B = new LoadBasedFlowEdgeImpl(vertexHopA, vertexHopB, topologySpec.getSpecExecutor());
    FlowEdge edgeB2Sink = new LoadBasedFlowEdgeImpl(vertexHopB, vertexSink, topologySpec.getSpecExecutor());
    Assert.assertTrue(weightedGraph.containsEdge(edgeSrc2A));
    Assert.assertTrue(weightedGraph.containsEdge(edgeA2B));
    Assert.assertTrue(weightedGraph.containsEdge(edgeB2Sink));
    Assert.assertTrue(edgeEqual(weightedGraph.getEdge(vertexSource, vertexHopA), edgeSrc2A));
    Assert.assertTrue(edgeEqual(weightedGraph.getEdge(vertexHopA, vertexHopB), edgeA2B));
    Assert.assertTrue(edgeEqual(weightedGraph.getEdge(vertexHopB, vertexSink), edgeB2Sink));
    this.compilerWithTemplateCalague.onDeleteSpec(topologySpec.getUri(), "");
}

Example 20 with FlowSpec

use of org.apache.gobblin.runtime.api.FlowSpec in project incubator-gobblin by apache.

the class MultiHopsFlowToJobSpecCompilerTest method initFlowSpec.

private FlowSpec initFlowSpec(String flowGroup, String flowName, String source, String destination) {
    Properties properties = new Properties();
    properties.put(ConfigurationKeys.JOB_SCHEDULE_KEY, "* * * * *");
    properties.put(ConfigurationKeys.FLOW_GROUP_KEY, flowGroup);
    properties.put(ConfigurationKeys.FLOW_NAME_KEY, flowName);
    properties.put(ServiceConfigKeys.FLOW_SOURCE_IDENTIFIER_KEY, source);
    properties.put(ServiceConfigKeys.FLOW_DESTINATION_IDENTIFIER_KEY, destination);
    Config config = ConfigUtils.propertiesToConfig(properties);
    FlowSpec.Builder flowSpecBuilder = null;
    try {
        flowSpecBuilder = FlowSpec.builder(computeTopologySpecURI(SPEC_STORE_PARENT_DIR, FLOW_SPEC_STORE_DIR)).withConfig(config).withDescription("dummy description").withVersion(SPEC_VERSION).withTemplate(new URI(TEST_TEMPLATE_URI));
    } catch (URISyntaxException e) {
        throw new RuntimeException(e);
    }
    return flowSpecBuilder.build();
}