Examples with Spec org.apache.gobblin.runtime.api.Spec used on opensource projects

Example 16 with Spec

use of org.apache.gobblin.runtime.api.Spec in project incubator-gobblin by apache.

the class IdentityFlowToJobSpecCompiler method compileFlow.

@Override
public Map<Spec, SpecExecutor> compileFlow(Spec spec) {
    Preconditions.checkNotNull(spec);
    Preconditions.checkArgument(spec instanceof FlowSpec, "IdentityFlowToJobSpecCompiler only converts FlowSpec to JobSpec");
    long startTime = System.nanoTime();
    Map<Spec, SpecExecutor> specExecutorMap = Maps.newLinkedHashMap();
    FlowSpec flowSpec = (FlowSpec) spec;
    String source = flowSpec.getConfig().getString(ServiceConfigKeys.FLOW_SOURCE_IDENTIFIER_KEY);
    String destination = flowSpec.getConfig().getString(ServiceConfigKeys.FLOW_DESTINATION_IDENTIFIER_KEY);
    log.info(String.format("Compiling flow for source: %s and destination: %s", source, destination));
    JobSpec jobSpec = jobSpecGenerator(flowSpec);
    for (TopologySpec topologySpec : topologySpecMap.values()) {
        try {
            Map<ServiceNode, ServiceNode> capabilities = (Map<ServiceNode, ServiceNode>) topologySpec.getSpecExecutor().getCapabilities().get();
            for (Map.Entry<ServiceNode, ServiceNode> capability : capabilities.entrySet()) {
                log.info(String.format("Evaluating current JobSpec: %s against TopologySpec: %s with " + "capability of source: %s and destination: %s ", jobSpec.getUri(), topologySpec.getUri(), capability.getKey(), capability.getValue()));
                if (source.equals(capability.getKey().getNodeName()) && destination.equals(capability.getValue().getNodeName())) {
                    specExecutorMap.put(jobSpec, topologySpec.getSpecExecutor());
                    log.info(String.format("Current JobSpec: %s is executable on TopologySpec: %s. Added TopologySpec as candidate.", jobSpec.getUri(), topologySpec.getUri()));
                    log.info("Since we found a candidate executor, we will not try to compute more. " + "(Intended limitation for IdentityFlowToJobSpecCompiler)");
                    return specExecutorMap;
                }
            }
        } catch (InterruptedException | ExecutionException e) {
            Instrumented.markMeter(this.flowCompilationFailedMeter);
            throw new RuntimeException("Cannot determine topology capabilities", e);
        }
    }
    Instrumented.markMeter(this.flowCompilationSuccessFulMeter);
    Instrumented.updateTimer(this.flowCompilationTimer, System.nanoTime() - startTime, TimeUnit.NANOSECONDS);
    return specExecutorMap;
}

Example 17 with Spec

use of org.apache.gobblin.runtime.api.Spec in project incubator-gobblin by apache.

the class GitConfigMonitorTest method testAddConfig.

@Test
public void testAddConfig() throws IOException, GitAPIException, URISyntaxException {
    // push a new config file
    this.testGroupDir.mkdirs();
    this.testFlowFile.createNewFile();
    Files.write("flow.name=testFlow\nflow.group=testGroup\nparam1=value1\n", testFlowFile, Charsets.UTF_8);
    // add, commit, push
    this.gitForPush.add().addFilepattern(formConfigFilePath(this.testGroupDir.getName(), this.testFlowFile.getName())).call();
    this.gitForPush.commit().setMessage("Second commit").call();
    this.gitForPush.push().setRemote("origin").setRefSpecs(this.masterRefSpec).call();
    this.gitConfigMonitor.processGitConfigChanges();
    Collection<Spec> specs = this.flowCatalog.getSpecs();
    Assert.assertTrue(specs.size() == 1);
    FlowSpec spec = (FlowSpec) (specs.iterator().next());
    Assert.assertEquals(spec.getUri(), new URI("gobblin-flow:/testGroup/testFlow"));
    Assert.assertEquals(spec.getConfig().getString(ConfigurationKeys.FLOW_NAME_KEY), "testFlow");
    Assert.assertEquals(spec.getConfig().getString(ConfigurationKeys.FLOW_GROUP_KEY), "testGroup");
    Assert.assertEquals(spec.getConfig().getString("param1"), "value1");
}

Example 18 with Spec

use of org.apache.gobblin.runtime.api.Spec in project incubator-gobblin by apache.

the class GitConfigMonitorTest method testPollingConfig.

@Test(dependsOnMethods = "testForcedPushConfig")
public void testPollingConfig() throws IOException, GitAPIException, URISyntaxException, InterruptedException {
    // push a new config file
    this.testGroupDir.mkdirs();
    this.testFlowFile.createNewFile();
    Files.write("flow.name=testFlow\nflow.group=testGroup\nparam1=value20\n", testFlowFile, Charsets.UTF_8);
    // add, commit, push
    this.gitForPush.add().addFilepattern(formConfigFilePath(this.testGroupDir.getName(), this.testFlowFile.getName())).call();
    this.gitForPush.commit().setMessage("Seventh commit").call();
    this.gitForPush.push().setRemote("origin").setRefSpecs(this.masterRefSpec).call();
    Collection<Spec> specs = this.flowCatalog.getSpecs();
    Assert.assertTrue(specs.size() == 0);
    this.gitConfigMonitor.startAsync().awaitRunning();
    // polling is every 5 seconds, so wait twice as long and check
    TimeUnit.SECONDS.sleep(10);
    specs = this.flowCatalog.getSpecs();
    Assert.assertTrue(specs.size() == 1);
    FlowSpec spec = (FlowSpec) (specs.iterator().next());
    Assert.assertEquals(spec.getUri(), new URI("gobblin-flow:/testGroup/testFlow"));
    Assert.assertEquals(spec.getConfig().getString(ConfigurationKeys.FLOW_NAME_KEY), "testFlow");
    Assert.assertEquals(spec.getConfig().getString(ConfigurationKeys.FLOW_GROUP_KEY), "testGroup");
    Assert.assertEquals(spec.getConfig().getString("param1"), "value20");
}

Example 19 with Spec

use of org.apache.gobblin.runtime.api.Spec in project incubator-gobblin by apache.

the class GitConfigMonitorTest method testDeleteConfig.

@Test(dependsOnMethods = "testUpdateConfig")
public void testDeleteConfig() throws IOException, GitAPIException, URISyntaxException {
    // delete a config file
    testFlowFile.delete();
    // flow catalog has 1 entry before the config is deleted
    Collection<Spec> specs = this.flowCatalog.getSpecs();
    Assert.assertTrue(specs.size() == 1);
    // add, commit, push
    DirCache ac = this.gitForPush.rm().addFilepattern(formConfigFilePath(this.testGroupDir.getName(), this.testFlowFile.getName())).call();
    RevCommit cc = this.gitForPush.commit().setMessage("Fourth commit").call();
    this.gitForPush.push().setRemote("origin").setRefSpecs(this.masterRefSpec).call();
    this.gitConfigMonitor.processGitConfigChanges();
    specs = this.flowCatalog.getSpecs();
    Assert.assertTrue(specs.size() == 0);
}

Example 20 with Spec

use of org.apache.gobblin.runtime.api.Spec in project incubator-gobblin by apache.

the class GobblinServiceManagerTest method testGitCreate.

@Test(dependsOnMethods = "testDelete")
public void testGitCreate() throws Exception {
    // push a new config file
    File testFlowFile = new File(GIT_CLONE_DIR + "/gobblin-config/testGroup/testFlow.pull");
    testFlowFile.getParentFile().mkdirs();
    Files.write("flow.name=testFlow\nflow.group=testGroup\nparam1=value20\n", testFlowFile, Charsets.UTF_8);
    Collection<Spec> specs = this.gobblinServiceManager.flowCatalog.getSpecs();
    Assert.assertTrue(specs.size() == 0);
    // add, commit, push
    this.gitForPush.add().addFilepattern("gobblin-config/testGroup/testFlow.pull").call();
    this.gitForPush.commit().setMessage("second commit").call();
    this.gitForPush.push().setRemote("origin").setRefSpecs(new RefSpec("master")).call();
    // polling is every 5 seconds, so wait twice as long and check
    TimeUnit.SECONDS.sleep(10);
    specs = this.gobblinServiceManager.flowCatalog.getSpecs();
    Assert.assertTrue(specs.size() == 1);
    FlowSpec spec = (FlowSpec) (specs.iterator().next());
    Assert.assertEquals(spec.getUri(), new URI("gobblin-flow:/testGroup/testFlow"));
    Assert.assertEquals(spec.getConfig().getString(ConfigurationKeys.FLOW_NAME_KEY), "testFlow");
    Assert.assertEquals(spec.getConfig().getString(ConfigurationKeys.FLOW_GROUP_KEY), "testGroup");
    Assert.assertEquals(spec.getConfig().getString("param1"), "value20");
}