Examples with FlowSpec org.apache.gobblin.runtime.api.FlowSpec used on opensource projects

Example 11 with FlowSpec

use of org.apache.gobblin.runtime.api.FlowSpec in project incubator-gobblin by apache.

the class FlowCatalogTest method createFlowSpec.

@Test
public void createFlowSpec() {
    // List Current Specs
    Collection<Spec> specs = flowCatalog.getSpecs();
    logger.info("[Before Create] Number of specs: " + specs.size());
    int i = 0;
    for (Spec spec : specs) {
        FlowSpec flowSpec = (FlowSpec) spec;
        logger.info("[Before Create] Spec " + i++ + ": " + gson.toJson(flowSpec));
    }
    Assert.assertTrue(specs.size() == 0, "Spec store should be empty before addition");
    // Create and add Spec
    this.flowCatalog.put(flowSpec);
    // List Specs after adding
    specs = flowCatalog.getSpecs();
    logger.info("[After Create] Number of specs: " + specs.size());
    i = 0;
    for (Spec spec : specs) {
        flowSpec = (FlowSpec) spec;
        logger.info("[After Create] Spec " + i++ + ": " + gson.toJson(flowSpec));
    }
    Assert.assertTrue(specs.size() == 1, "Spec store should contain 1 Spec after addition");
}

Example 12 with FlowSpec

use of org.apache.gobblin.runtime.api.FlowSpec in project incubator-gobblin by apache.

the class IdentityFlowToJobSpecCompiler method compileFlow.

@Override
public Map<Spec, SpecExecutor> compileFlow(Spec spec) {
    Preconditions.checkNotNull(spec);
    Preconditions.checkArgument(spec instanceof FlowSpec, "IdentityFlowToJobSpecCompiler only converts FlowSpec to JobSpec");
    long startTime = System.nanoTime();
    Map<Spec, SpecExecutor> specExecutorMap = Maps.newLinkedHashMap();
    FlowSpec flowSpec = (FlowSpec) spec;
    String source = flowSpec.getConfig().getString(ServiceConfigKeys.FLOW_SOURCE_IDENTIFIER_KEY);
    String destination = flowSpec.getConfig().getString(ServiceConfigKeys.FLOW_DESTINATION_IDENTIFIER_KEY);
    log.info(String.format("Compiling flow for source: %s and destination: %s", source, destination));
    JobSpec jobSpec = jobSpecGenerator(flowSpec);
    for (TopologySpec topologySpec : topologySpecMap.values()) {
        try {
            Map<ServiceNode, ServiceNode> capabilities = (Map<ServiceNode, ServiceNode>) topologySpec.getSpecExecutor().getCapabilities().get();
            for (Map.Entry<ServiceNode, ServiceNode> capability : capabilities.entrySet()) {
                log.info(String.format("Evaluating current JobSpec: %s against TopologySpec: %s with " + "capability of source: %s and destination: %s ", jobSpec.getUri(), topologySpec.getUri(), capability.getKey(), capability.getValue()));
                if (source.equals(capability.getKey().getNodeName()) && destination.equals(capability.getValue().getNodeName())) {
                    specExecutorMap.put(jobSpec, topologySpec.getSpecExecutor());
                    log.info(String.format("Current JobSpec: %s is executable on TopologySpec: %s. Added TopologySpec as candidate.", jobSpec.getUri(), topologySpec.getUri()));
                    log.info("Since we found a candidate executor, we will not try to compute more. " + "(Intended limitation for IdentityFlowToJobSpecCompiler)");
                    return specExecutorMap;
                }
            }
        } catch (InterruptedException | ExecutionException e) {
            Instrumented.markMeter(this.flowCompilationFailedMeter);
            throw new RuntimeException("Cannot determine topology capabilities", e);
        }
    }
    Instrumented.markMeter(this.flowCompilationSuccessFulMeter);
    Instrumented.updateTimer(this.flowCompilationTimer, System.nanoTime() - startTime, TimeUnit.NANOSECONDS);
    return specExecutorMap;
}

Example 13 with FlowSpec

use of org.apache.gobblin.runtime.api.FlowSpec in project incubator-gobblin by apache.

the class GitConfigMonitor method removeSpec.

/**
 * remove a {@link FlowSpec} for a deleted or renamed flow config
 * @param change
 */
private void removeSpec(DiffEntry change) {
    if (checkConfigFilePath(change.getOldPath())) {
        Path configFilePath = new Path(this.repositoryDir, change.getOldPath());
        String flowName = FSSpecStore.getSpecName(configFilePath);
        String flowGroup = FSSpecStore.getSpecGroup(configFilePath);
        // build a dummy config to get the proper URI for delete
        Config dummyConfig = ConfigBuilder.create().addPrimitive(ConfigurationKeys.FLOW_GROUP_KEY, flowGroup).addPrimitive(ConfigurationKeys.FLOW_NAME_KEY, flowName).build();
        FlowSpec spec = FlowSpec.builder().withConfig(dummyConfig).withVersion(SPEC_VERSION).withDescription(SPEC_DESCRIPTION).build();
        this.flowCatalog.remove(spec.getUri());
    }
}

Example 14 with FlowSpec

use of org.apache.gobblin.runtime.api.FlowSpec in project incubator-gobblin by apache.

the class GitConfigMonitorTest method testAddConfig.

@Test
public void testAddConfig() throws IOException, GitAPIException, URISyntaxException {
    // push a new config file
    this.testGroupDir.mkdirs();
    this.testFlowFile.createNewFile();
    Files.write("flow.name=testFlow\nflow.group=testGroup\nparam1=value1\n", testFlowFile, Charsets.UTF_8);
    // add, commit, push
    this.gitForPush.add().addFilepattern(formConfigFilePath(this.testGroupDir.getName(), this.testFlowFile.getName())).call();
    this.gitForPush.commit().setMessage("Second commit").call();
    this.gitForPush.push().setRemote("origin").setRefSpecs(this.masterRefSpec).call();
    this.gitConfigMonitor.processGitConfigChanges();
    Collection<Spec> specs = this.flowCatalog.getSpecs();
    Assert.assertTrue(specs.size() == 1);
    FlowSpec spec = (FlowSpec) (specs.iterator().next());
    Assert.assertEquals(spec.getUri(), new URI("gobblin-flow:/testGroup/testFlow"));
    Assert.assertEquals(spec.getConfig().getString(ConfigurationKeys.FLOW_NAME_KEY), "testFlow");
    Assert.assertEquals(spec.getConfig().getString(ConfigurationKeys.FLOW_GROUP_KEY), "testGroup");
    Assert.assertEquals(spec.getConfig().getString("param1"), "value1");
}

Example 15 with FlowSpec

use of org.apache.gobblin.runtime.api.FlowSpec in project incubator-gobblin by apache.

the class GitConfigMonitorTest method testPollingConfig.

@Test(dependsOnMethods = "testForcedPushConfig")
public void testPollingConfig() throws IOException, GitAPIException, URISyntaxException, InterruptedException {
    // push a new config file
    this.testGroupDir.mkdirs();
    this.testFlowFile.createNewFile();
    Files.write("flow.name=testFlow\nflow.group=testGroup\nparam1=value20\n", testFlowFile, Charsets.UTF_8);
    // add, commit, push
    this.gitForPush.add().addFilepattern(formConfigFilePath(this.testGroupDir.getName(), this.testFlowFile.getName())).call();
    this.gitForPush.commit().setMessage("Seventh commit").call();
    this.gitForPush.push().setRemote("origin").setRefSpecs(this.masterRefSpec).call();
    Collection<Spec> specs = this.flowCatalog.getSpecs();
    Assert.assertTrue(specs.size() == 0);
    this.gitConfigMonitor.startAsync().awaitRunning();
    // polling is every 5 seconds, so wait twice as long and check
    TimeUnit.SECONDS.sleep(10);
    specs = this.flowCatalog.getSpecs();
    Assert.assertTrue(specs.size() == 1);
    FlowSpec spec = (FlowSpec) (specs.iterator().next());
    Assert.assertEquals(spec.getUri(), new URI("gobblin-flow:/testGroup/testFlow"));
    Assert.assertEquals(spec.getConfig().getString(ConfigurationKeys.FLOW_NAME_KEY), "testFlow");
    Assert.assertEquals(spec.getConfig().getString(ConfigurationKeys.FLOW_GROUP_KEY), "testGroup");
    Assert.assertEquals(spec.getConfig().getString("param1"), "value20");
}