Examples with FlowSpec org.apache.gobblin.runtime.api.FlowSpec used on opensource projects

Example 1 with FlowSpec

use of org.apache.gobblin.runtime.api.FlowSpec in project incubator-gobblin by apache.

the class GitConfigMonitorTest method testForcedPushConfig.

@Test(dependsOnMethods = "testDeleteConfig")
public void testForcedPushConfig() throws IOException, GitAPIException, URISyntaxException {
    // push a new config file
    this.testGroupDir.mkdirs();
    this.testFlowFile.createNewFile();
    Files.write("flow.name=testFlow\nflow.group=testGroup\nparam1=value1\n", testFlowFile, Charsets.UTF_8);
    this.testFlowFile2.createNewFile();
    Files.write("flow.name=testFlow2\nflow.group=testGroup\nparam1=value2\n", testFlowFile2, Charsets.UTF_8);
    // add, commit, push
    this.gitForPush.add().addFilepattern(formConfigFilePath(this.testGroupDir.getName(), this.testFlowFile.getName())).call();
    this.gitForPush.add().addFilepattern(formConfigFilePath(this.testGroupDir.getName(), this.testFlowFile2.getName())).call();
    this.gitForPush.commit().setMessage("Fifth commit").call();
    this.gitForPush.push().setRemote("origin").setRefSpecs(this.masterRefSpec).call();
    this.gitConfigMonitor.processGitConfigChanges();
    Collection<Spec> specs = this.flowCatalog.getSpecs();
    Assert.assertTrue(specs.size() == 2);
    List<Spec> specList = Lists.newArrayList(specs);
    specList.sort(new Comparator<Spec>() {

        @Override
        public int compare(Spec o1, Spec o2) {
            return o1.getUri().compareTo(o2.getUri());
        }
    });
    FlowSpec spec = (FlowSpec) specList.get(0);
    Assert.assertEquals(spec.getUri(), new URI("gobblin-flow:/testGroup/testFlow"));
    Assert.assertEquals(spec.getConfig().getString(ConfigurationKeys.FLOW_NAME_KEY), "testFlow");
    Assert.assertEquals(spec.getConfig().getString(ConfigurationKeys.FLOW_GROUP_KEY), "testGroup");
    Assert.assertEquals(spec.getConfig().getString("param1"), "value1");
    spec = (FlowSpec) specList.get(1);
    Assert.assertEquals(spec.getUri(), new URI("gobblin-flow:/testGroup/testFlow2"));
    Assert.assertEquals(spec.getConfig().getString(ConfigurationKeys.FLOW_NAME_KEY), "testFlow2");
    Assert.assertEquals(spec.getConfig().getString(ConfigurationKeys.FLOW_GROUP_KEY), "testGroup");
    Assert.assertEquals(spec.getConfig().getString("param1"), "value2");
    // go back in time to cause conflict
    this.gitForPush.reset().setMode(ResetCommand.ResetType.HARD).setRef("HEAD~1").call();
    this.gitForPush.push().setForce(true).setRemote("origin").setRefSpecs(this.masterRefSpec).call();
    // add new files
    this.testGroupDir.mkdirs();
    this.testFlowFile2.createNewFile();
    Files.write("flow.name=testFlow2\nflow.group=testGroup\nparam1=value4\n", testFlowFile2, Charsets.UTF_8);
    this.testFlowFile3.createNewFile();
    Files.write("flow.name=testFlow3\nflow.group=testGroup\nparam1=value5\n", testFlowFile3, Charsets.UTF_8);
    // add, commit, push
    this.gitForPush.add().addFilepattern(formConfigFilePath(this.testGroupDir.getName(), this.testFlowFile2.getName())).call();
    this.gitForPush.add().addFilepattern(formConfigFilePath(this.testGroupDir.getName(), this.testFlowFile3.getName())).call();
    this.gitForPush.commit().setMessage("Sixth commit").call();
    this.gitForPush.push().setRemote("origin").setRefSpecs(this.masterRefSpec).call();
    this.gitConfigMonitor.processGitConfigChanges();
    specs = this.flowCatalog.getSpecs();
    Assert.assertTrue(specs.size() == 2);
    specList = Lists.newArrayList(specs);
    specList.sort(new Comparator<Spec>() {

        @Override
        public int compare(Spec o1, Spec o2) {
            return o1.getUri().compareTo(o2.getUri());
        }
    });
    spec = (FlowSpec) specList.get(0);
    Assert.assertEquals(spec.getUri(), new URI("gobblin-flow:/testGroup/testFlow2"));
    Assert.assertEquals(spec.getConfig().getString(ConfigurationKeys.FLOW_NAME_KEY), "testFlow2");
    Assert.assertEquals(spec.getConfig().getString(ConfigurationKeys.FLOW_GROUP_KEY), "testGroup");
    Assert.assertEquals(spec.getConfig().getString("param1"), "value4");
    spec = (FlowSpec) specList.get(1);
    Assert.assertEquals(spec.getUri(), new URI("gobblin-flow:/testGroup/testFlow3"));
    Assert.assertEquals(spec.getConfig().getString(ConfigurationKeys.FLOW_NAME_KEY), "testFlow3");
    Assert.assertEquals(spec.getConfig().getString(ConfigurationKeys.FLOW_GROUP_KEY), "testGroup");
    Assert.assertEquals(spec.getConfig().getString("param1"), "value5");
    // reset for next test case
    this.gitForPush.reset().setMode(ResetCommand.ResetType.HARD).setRef("HEAD~4").call();
    this.gitForPush.push().setForce(true).setRemote("origin").setRefSpecs(this.masterRefSpec).call();
    this.gitConfigMonitor.processGitConfigChanges();
    specs = this.flowCatalog.getSpecs();
    Assert.assertTrue(specs.size() == 0);
}

Example 2 with FlowSpec

use of org.apache.gobblin.runtime.api.FlowSpec in project incubator-gobblin by apache.

the class GitConfigMonitorTest method testUpdateConfig.

@Test(dependsOnMethods = "testAddConfig")
public void testUpdateConfig() throws IOException, GitAPIException, URISyntaxException {
    // push an updated config file
    Files.write("flow.name=testFlow\nflow.group=testGroup\nparam1=value2\n", testFlowFile, Charsets.UTF_8);
    // add, commit, push
    this.gitForPush.add().addFilepattern(formConfigFilePath(this.testGroupDir.getName(), this.testFlowFile.getName())).call();
    this.gitForPush.commit().setMessage("Third commit").call();
    this.gitForPush.push().setRemote("origin").setRefSpecs(this.masterRefSpec).call();
    this.gitConfigMonitor.processGitConfigChanges();
    Collection<Spec> specs = this.flowCatalog.getSpecs();
    Assert.assertTrue(specs.size() == 1);
    FlowSpec spec = (FlowSpec) (specs.iterator().next());
    Assert.assertEquals(spec.getUri(), new URI("gobblin-flow:/testGroup/testFlow"));
    Assert.assertEquals(spec.getConfig().getString(ConfigurationKeys.FLOW_NAME_KEY), "testFlow");
    Assert.assertEquals(spec.getConfig().getString(ConfigurationKeys.FLOW_GROUP_KEY), "testGroup");
    Assert.assertEquals(spec.getConfig().getString("param1"), "value2");
}

Example 3 with FlowSpec

use of org.apache.gobblin.runtime.api.FlowSpec in project incubator-gobblin by apache.

the class IdentityFlowToJobSpecCompilerTest method testCompilerWithoutTemplateCatalog.

@Test
public void testCompilerWithoutTemplateCatalog() {
    FlowSpec flowSpec = initFlowSpec();
    // Run compiler on flowSpec
    Map<Spec, SpecExecutor> specExecutorMapping = this.compilerWithoutTemplateCalague.compileFlow(flowSpec);
    // Assert pre-requisites
    Assert.assertNotNull(specExecutorMapping, "Expected non null mapping.");
    Assert.assertTrue(specExecutorMapping.size() == 1, "Exepected 1 executor for FlowSpec.");
    // Assert FlowSpec compilation
    Spec spec = specExecutorMapping.keySet().iterator().next();
    Assert.assertTrue(spec instanceof JobSpec, "Expected JobSpec compiled from FlowSpec.");
    // Assert JobSpec properties
    JobSpec jobSpec = (JobSpec) spec;
    Assert.assertTrue(!jobSpec.getConfig().hasPath("testProperty1"));
    Assert.assertTrue(!jobSpec.getConfig().hasPath("testProperty2"));
    Assert.assertTrue(!jobSpec.getConfig().hasPath("testProperty3"));
    Assert.assertEquals(jobSpec.getConfig().getString(ServiceConfigKeys.FLOW_SOURCE_IDENTIFIER_KEY), TEST_SOURCE_NAME);
    Assert.assertFalse(jobSpec.getConfig().hasPath(ConfigurationKeys.JOB_SCHEDULE_KEY));
    Assert.assertEquals(jobSpec.getConfig().getString(ConfigurationKeys.JOB_NAME_KEY), TEST_FLOW_NAME);
    Assert.assertEquals(jobSpec.getConfig().getString(ConfigurationKeys.JOB_GROUP_KEY), TEST_FLOW_GROUP);
    Assert.assertEquals(jobSpec.getConfig().getString(ConfigurationKeys.FLOW_NAME_KEY), TEST_FLOW_NAME);
    Assert.assertEquals(jobSpec.getConfig().getString(ConfigurationKeys.FLOW_GROUP_KEY), TEST_FLOW_GROUP);
    Assert.assertTrue(jobSpec.getConfig().hasPath(ConfigurationKeys.FLOW_EXECUTION_ID_KEY));
}

Example 4 with FlowSpec

use of org.apache.gobblin.runtime.api.FlowSpec in project incubator-gobblin by apache.

the class IdentityFlowToJobSpecCompilerTest method initFlowSpec.

private FlowSpec initFlowSpec(String flowGroup, String flowName, String source, String destination) {
    Properties properties = new Properties();
    properties.put(ConfigurationKeys.JOB_SCHEDULE_KEY, "* * * * *");
    properties.put(ConfigurationKeys.FLOW_GROUP_KEY, flowGroup);
    properties.put(ConfigurationKeys.FLOW_NAME_KEY, flowName);
    properties.put(ServiceConfigKeys.FLOW_SOURCE_IDENTIFIER_KEY, source);
    properties.put(ServiceConfigKeys.FLOW_DESTINATION_IDENTIFIER_KEY, destination);
    Config config = ConfigUtils.propertiesToConfig(properties);
    FlowSpec.Builder flowSpecBuilder = null;
    try {
        flowSpecBuilder = FlowSpec.builder(computeTopologySpecURI(SPEC_STORE_PARENT_DIR, FLOW_SPEC_STORE_DIR)).withConfig(config).withDescription("dummy description").withVersion(SPEC_VERSION).withTemplate(new URI(TEST_TEMPLATE_URI));
    } catch (URISyntaxException e) {
        throw new RuntimeException(e);
    }
    return flowSpecBuilder.build();
}

Example 5 with FlowSpec

use of org.apache.gobblin.runtime.api.FlowSpec in project incubator-gobblin by apache.

the class MultiHopsFlowToJobSpecCompilerTest method testDijkstraPathFinding.

@Test(dependsOnMethods = "testWeightedGraphConstruction")
public void testDijkstraPathFinding() {
    FlowSpec flowSpec = initFlowSpec();
    TopologySpec topologySpec_1 = initTopologySpec(TOPOLOGY_SPEC_STORE_DIR, TEST_SOURCE_NAME, TEST_HOP_NAME_A, TEST_HOP_NAME_B, TEST_SINK_NAME);
    TopologySpec topologySpec_2 = initTopologySpec(TOPOLOGY_SPEC_STORE_DIR_SECOND, TEST_SOURCE_NAME, TEST_HOP_NAME_B, TEST_HOP_NAME_C, TEST_SINK_NAME);
    this.compilerWithTemplateCalague.onAddSpec(topologySpec_1);
    this.compilerWithTemplateCalague.onAddSpec(topologySpec_2);
    // Get the edge -> Change the weight -> Materialized the edge change back to graph -> compile again -> Assertion
    this.compilerWithTemplateCalague.compileFlow(flowSpec);
    DirectedWeightedMultigraph<ServiceNode, FlowEdge> weightedGraph = compilerWithTemplateCalague.getWeightedGraph();
    FlowEdge a2b = weightedGraph.getEdge(vertexHopA, vertexHopB);
    FlowEdge b2c = weightedGraph.getEdge(vertexHopB, vertexHopC);
    FlowEdge c2s = weightedGraph.getEdge(vertexHopC, vertexSink);
    weightedGraph.setEdgeWeight(a2b, 1.99);
    weightedGraph.setEdgeWeight(b2c, 0.1);
    weightedGraph.setEdgeWeight(c2s, 0.2);
    // Best route: Src - B(1) - C(0.1) - sink (0.2)
    this.compilerWithTemplateCalague.compileFlow(flowSpec);
    List<FlowEdge> edgeList = dijkstraBasedPathFindingHelper(vertexSource, vertexSink, weightedGraph);
    FlowEdge src2b = weightedGraph.getEdge(vertexSource, vertexHopB);
    FlowEdge b2C = weightedGraph.getEdge(vertexHopB, vertexHopC);
    FlowEdge c2sink = weightedGraph.getEdge(vertexHopC, vertexSink);
    Assert.assertEquals(edgeList.get(0).getEdgeIdentity(), src2b.getEdgeIdentity());
    Assert.assertEquals(edgeList.get(1).getEdgeIdentity(), b2C.getEdgeIdentity());
    Assert.assertEquals(edgeList.get(2).getEdgeIdentity(), c2sink.getEdgeIdentity());
    this.compilerWithTemplateCalague.onDeleteSpec(topologySpec_1.getUri(), "");
    this.compilerWithTemplateCalague.onDeleteSpec(topologySpec_2.getUri(), "");
}