Examples with Spec org.apache.gobblin.runtime.api.Spec used on opensource projects

Example 6 with Spec

use of org.apache.gobblin.runtime.api.Spec in project incubator-gobblin by apache.

the class TopologyCatalog method addListener.

@Override
public void addListener(SpecCatalogListener specListener) {
    Preconditions.checkNotNull(specListener);
    this.listeners.addListener(specListener);
    if (state() == Service.State.RUNNING) {
        for (Spec spec : getSpecs()) {
            SpecCatalogListener.AddSpecCallback addJobCallback = new SpecCatalogListener.AddSpecCallback(spec);
            this.listeners.callbackOneListener(addJobCallback, specListener);
        }
    }
}

Example 7 with Spec

use of org.apache.gobblin.runtime.api.Spec in project incubator-gobblin by apache.

the class FSSpecStore method getSpec.

@Override
public Spec getSpec(URI specUri) throws SpecNotFoundException {
    Preconditions.checkArgument(null != specUri, "Spec URI should not be null");
    Collection<Spec> specs = getAllVersionsOfSpec(specUri);
    Spec highestVersionSpec = null;
    for (Spec spec : specs) {
        if (null == highestVersionSpec) {
            highestVersionSpec = spec;
        } else if (null != spec.getVersion() && spec.getVersion().compareTo(spec.getVersion()) > 0) {
            highestVersionSpec = spec;
        }
    }
    if (null == highestVersionSpec) {
        throw new SpecNotFoundException(specUri);
    }
    return highestVersionSpec;
}

Example 8 with Spec

use of org.apache.gobblin.runtime.api.Spec in project incubator-gobblin by apache.

the class GitConfigMonitorTest method testForcedPushConfig.

@Test(dependsOnMethods = "testDeleteConfig")
public void testForcedPushConfig() throws IOException, GitAPIException, URISyntaxException {
    // push a new config file
    this.testGroupDir.mkdirs();
    this.testFlowFile.createNewFile();
    Files.write("flow.name=testFlow\nflow.group=testGroup\nparam1=value1\n", testFlowFile, Charsets.UTF_8);
    this.testFlowFile2.createNewFile();
    Files.write("flow.name=testFlow2\nflow.group=testGroup\nparam1=value2\n", testFlowFile2, Charsets.UTF_8);
    // add, commit, push
    this.gitForPush.add().addFilepattern(formConfigFilePath(this.testGroupDir.getName(), this.testFlowFile.getName())).call();
    this.gitForPush.add().addFilepattern(formConfigFilePath(this.testGroupDir.getName(), this.testFlowFile2.getName())).call();
    this.gitForPush.commit().setMessage("Fifth commit").call();
    this.gitForPush.push().setRemote("origin").setRefSpecs(this.masterRefSpec).call();
    this.gitConfigMonitor.processGitConfigChanges();
    Collection<Spec> specs = this.flowCatalog.getSpecs();
    Assert.assertTrue(specs.size() == 2);
    List<Spec> specList = Lists.newArrayList(specs);
    specList.sort(new Comparator<Spec>() {

        @Override
        public int compare(Spec o1, Spec o2) {
            return o1.getUri().compareTo(o2.getUri());
        }
    });
    FlowSpec spec = (FlowSpec) specList.get(0);
    Assert.assertEquals(spec.getUri(), new URI("gobblin-flow:/testGroup/testFlow"));
    Assert.assertEquals(spec.getConfig().getString(ConfigurationKeys.FLOW_NAME_KEY), "testFlow");
    Assert.assertEquals(spec.getConfig().getString(ConfigurationKeys.FLOW_GROUP_KEY), "testGroup");
    Assert.assertEquals(spec.getConfig().getString("param1"), "value1");
    spec = (FlowSpec) specList.get(1);
    Assert.assertEquals(spec.getUri(), new URI("gobblin-flow:/testGroup/testFlow2"));
    Assert.assertEquals(spec.getConfig().getString(ConfigurationKeys.FLOW_NAME_KEY), "testFlow2");
    Assert.assertEquals(spec.getConfig().getString(ConfigurationKeys.FLOW_GROUP_KEY), "testGroup");
    Assert.assertEquals(spec.getConfig().getString("param1"), "value2");
    // go back in time to cause conflict
    this.gitForPush.reset().setMode(ResetCommand.ResetType.HARD).setRef("HEAD~1").call();
    this.gitForPush.push().setForce(true).setRemote("origin").setRefSpecs(this.masterRefSpec).call();
    // add new files
    this.testGroupDir.mkdirs();
    this.testFlowFile2.createNewFile();
    Files.write("flow.name=testFlow2\nflow.group=testGroup\nparam1=value4\n", testFlowFile2, Charsets.UTF_8);
    this.testFlowFile3.createNewFile();
    Files.write("flow.name=testFlow3\nflow.group=testGroup\nparam1=value5\n", testFlowFile3, Charsets.UTF_8);
    // add, commit, push
    this.gitForPush.add().addFilepattern(formConfigFilePath(this.testGroupDir.getName(), this.testFlowFile2.getName())).call();
    this.gitForPush.add().addFilepattern(formConfigFilePath(this.testGroupDir.getName(), this.testFlowFile3.getName())).call();
    this.gitForPush.commit().setMessage("Sixth commit").call();
    this.gitForPush.push().setRemote("origin").setRefSpecs(this.masterRefSpec).call();
    this.gitConfigMonitor.processGitConfigChanges();
    specs = this.flowCatalog.getSpecs();
    Assert.assertTrue(specs.size() == 2);
    specList = Lists.newArrayList(specs);
    specList.sort(new Comparator<Spec>() {

        @Override
        public int compare(Spec o1, Spec o2) {
            return o1.getUri().compareTo(o2.getUri());
        }
    });
    spec = (FlowSpec) specList.get(0);
    Assert.assertEquals(spec.getUri(), new URI("gobblin-flow:/testGroup/testFlow2"));
    Assert.assertEquals(spec.getConfig().getString(ConfigurationKeys.FLOW_NAME_KEY), "testFlow2");
    Assert.assertEquals(spec.getConfig().getString(ConfigurationKeys.FLOW_GROUP_KEY), "testGroup");
    Assert.assertEquals(spec.getConfig().getString("param1"), "value4");
    spec = (FlowSpec) specList.get(1);
    Assert.assertEquals(spec.getUri(), new URI("gobblin-flow:/testGroup/testFlow3"));
    Assert.assertEquals(spec.getConfig().getString(ConfigurationKeys.FLOW_NAME_KEY), "testFlow3");
    Assert.assertEquals(spec.getConfig().getString(ConfigurationKeys.FLOW_GROUP_KEY), "testGroup");
    Assert.assertEquals(spec.getConfig().getString("param1"), "value5");
    // reset for next test case
    this.gitForPush.reset().setMode(ResetCommand.ResetType.HARD).setRef("HEAD~4").call();
    this.gitForPush.push().setForce(true).setRemote("origin").setRefSpecs(this.masterRefSpec).call();
    this.gitConfigMonitor.processGitConfigChanges();
    specs = this.flowCatalog.getSpecs();
    Assert.assertTrue(specs.size() == 0);
}

Example 9 with Spec

use of org.apache.gobblin.runtime.api.Spec in project incubator-gobblin by apache.

the class GitConfigMonitorTest method testUpdateConfig.

@Test(dependsOnMethods = "testAddConfig")
public void testUpdateConfig() throws IOException, GitAPIException, URISyntaxException {
    // push an updated config file
    Files.write("flow.name=testFlow\nflow.group=testGroup\nparam1=value2\n", testFlowFile, Charsets.UTF_8);
    // add, commit, push
    this.gitForPush.add().addFilepattern(formConfigFilePath(this.testGroupDir.getName(), this.testFlowFile.getName())).call();
    this.gitForPush.commit().setMessage("Third commit").call();
    this.gitForPush.push().setRemote("origin").setRefSpecs(this.masterRefSpec).call();
    this.gitConfigMonitor.processGitConfigChanges();
    Collection<Spec> specs = this.flowCatalog.getSpecs();
    Assert.assertTrue(specs.size() == 1);
    FlowSpec spec = (FlowSpec) (specs.iterator().next());
    Assert.assertEquals(spec.getUri(), new URI("gobblin-flow:/testGroup/testFlow"));
    Assert.assertEquals(spec.getConfig().getString(ConfigurationKeys.FLOW_NAME_KEY), "testFlow");
    Assert.assertEquals(spec.getConfig().getString(ConfigurationKeys.FLOW_GROUP_KEY), "testGroup");
    Assert.assertEquals(spec.getConfig().getString("param1"), "value2");
}

Example 10 with Spec

use of org.apache.gobblin.runtime.api.Spec in project incubator-gobblin by apache.

the class IdentityFlowToJobSpecCompilerTest method testCompilerWithoutTemplateCatalog.

@Test
public void testCompilerWithoutTemplateCatalog() {
    FlowSpec flowSpec = initFlowSpec();
    // Run compiler on flowSpec
    Map<Spec, SpecExecutor> specExecutorMapping = this.compilerWithoutTemplateCalague.compileFlow(flowSpec);
    // Assert pre-requisites
    Assert.assertNotNull(specExecutorMapping, "Expected non null mapping.");
    Assert.assertTrue(specExecutorMapping.size() == 1, "Exepected 1 executor for FlowSpec.");
    // Assert FlowSpec compilation
    Spec spec = specExecutorMapping.keySet().iterator().next();
    Assert.assertTrue(spec instanceof JobSpec, "Expected JobSpec compiled from FlowSpec.");
    // Assert JobSpec properties
    JobSpec jobSpec = (JobSpec) spec;
    Assert.assertTrue(!jobSpec.getConfig().hasPath("testProperty1"));
    Assert.assertTrue(!jobSpec.getConfig().hasPath("testProperty2"));
    Assert.assertTrue(!jobSpec.getConfig().hasPath("testProperty3"));
    Assert.assertEquals(jobSpec.getConfig().getString(ServiceConfigKeys.FLOW_SOURCE_IDENTIFIER_KEY), TEST_SOURCE_NAME);
    Assert.assertFalse(jobSpec.getConfig().hasPath(ConfigurationKeys.JOB_SCHEDULE_KEY));
    Assert.assertEquals(jobSpec.getConfig().getString(ConfigurationKeys.JOB_NAME_KEY), TEST_FLOW_NAME);
    Assert.assertEquals(jobSpec.getConfig().getString(ConfigurationKeys.JOB_GROUP_KEY), TEST_FLOW_GROUP);
    Assert.assertEquals(jobSpec.getConfig().getString(ConfigurationKeys.FLOW_NAME_KEY), TEST_FLOW_NAME);
    Assert.assertEquals(jobSpec.getConfig().getString(ConfigurationKeys.FLOW_GROUP_KEY), TEST_FLOW_GROUP);
    Assert.assertTrue(jobSpec.getConfig().hasPath(ConfigurationKeys.FLOW_EXECUTION_ID_KEY));
}