Examples with InMemorySpecExecutor org.apache.gobblin.runtime.spec_executorInstance.InMemorySpecExecutor used on opensource projects

Example 1 with InMemorySpecExecutor

use of org.apache.gobblin.runtime.spec_executorInstance.InMemorySpecExecutor in project incubator-gobblin by apache.

the class IdentityFlowToJobSpecCompilerTest method initTopologySpec.

private TopologySpec initTopologySpec() {
    Properties properties = new Properties();
    properties.put("specStore.fs.dir", TOPOLOGY_SPEC_STORE_DIR);
    properties.put("specExecInstance.capabilities", TEST_SOURCE_NAME + ":" + TEST_SINK_NAME);
    Config config = ConfigUtils.propertiesToConfig(properties);
    SpecExecutor specExecutorInstance = new InMemorySpecExecutor(config);
    TopologySpec.Builder topologySpecBuilder = TopologySpec.builder(computeTopologySpecURI(SPEC_STORE_PARENT_DIR, TOPOLOGY_SPEC_STORE_DIR)).withConfig(config).withDescription(SPEC_DESCRIPTION).withVersion(SPEC_VERSION).withSpecExecutor(specExecutorInstance);
    return topologySpecBuilder.build();
}

Example 2 with InMemorySpecExecutor

use of org.apache.gobblin.runtime.spec_executorInstance.InMemorySpecExecutor in project incubator-gobblin by apache.

the class MultiHopsFlowToJobSpecCompilerTest method initTopologySpec.

// The topology is: Src - A - B - Dest
private TopologySpec initTopologySpec(String storeDir, String... args) {
    Properties properties = new Properties();
    properties.put("specStore.fs.dir", storeDir);
    String capabilitiesString = "";
    for (int i = 0; i < args.length - 1; i++) {
        capabilitiesString = capabilitiesString + (args[i] + ":" + args[i + 1] + ",");
    }
    Assert.assertEquals(capabilitiesString.charAt(capabilitiesString.length() - 1), ',');
    capabilitiesString = capabilitiesString.substring(0, capabilitiesString.length() - 1);
    properties.put("specExecInstance.capabilities", capabilitiesString);
    properties.put("executorAttrs", new Properties());
    Config config = ConfigUtils.propertiesToConfig(properties);
    SpecExecutor specExecutorInstance = new InMemorySpecExecutor(config);
    TopologySpec.Builder topologySpecBuilder = TopologySpec.builder(IdentityFlowToJobSpecCompilerTest.computeTopologySpecURI(SPEC_STORE_PARENT_DIR, storeDir)).withConfig(config).withDescription(SPEC_DESCRIPTION).withVersion(SPEC_VERSION).withSpecExecutor(specExecutorInstance);
    return topologySpecBuilder.build();
}

Example 3 with InMemorySpecExecutor

use of org.apache.gobblin.runtime.spec_executorInstance.InMemorySpecExecutor in project incubator-gobblin by apache.

the class OrchestratorTest method initTopologySpec.

private TopologySpec initTopologySpec() {
    Properties properties = new Properties();
    properties.put("specStore.fs.dir", TOPOLOGY_SPEC_STORE_DIR);
    properties.put("specExecInstance.capabilities", "source:destination");
    Config config = ConfigUtils.propertiesToConfig(properties);
    SpecExecutor specExecutorInstance = new InMemorySpecExecutor(config);
    TopologySpec.Builder topologySpecBuilder = TopologySpec.builder(computeTopologySpecURI(SPEC_STORE_PARENT_DIR, TOPOLOGY_SPEC_STORE_DIR)).withConfig(config).withDescription(SPEC_DESCRIPTION).withVersion(SPEC_VERSION).withSpecExecutor(specExecutorInstance);
    return topologySpecBuilder.build();
}

Example 4 with InMemorySpecExecutor

use of org.apache.gobblin.runtime.spec_executorInstance.InMemorySpecExecutor in project incubator-gobblin by apache.

the class FlowCatalogTest method initFlowSpec.

private FlowSpec initFlowSpec() {
    Properties properties = new Properties();
    properties.put("specStore.fs.dir", SPEC_STORE_DIR);
    properties.put("specExecInstance.capabilities", "source:destination");
    Config config = ConfigUtils.propertiesToConfig(properties);
    SpecExecutor specExecutorInstanceProducer = new InMemorySpecExecutor(config);
    FlowSpec.Builder flowSpecBuilder = null;
    try {
        flowSpecBuilder = FlowSpec.builder(computeFlowSpecURI()).withConfig(config).withDescription(SPEC_DESCRIPTION).withVersion(SPEC_VERSION).withTemplate(new URI("templateURI"));
    } catch (URISyntaxException e) {
        throw new RuntimeException(e);
    }
    return flowSpecBuilder.build();
}

Example 5 with InMemorySpecExecutor

use of org.apache.gobblin.runtime.spec_executorInstance.InMemorySpecExecutor in project incubator-gobblin by apache.

the class TopologyCatalogTest method initTopologySpec.

private TopologySpec initTopologySpec() {
    Properties properties = new Properties();
    properties.put("specStore.fs.dir", SPEC_STORE_DIR);
    properties.put("specExecInstance.capabilities", "source:destination");
    Config config = ConfigUtils.propertiesToConfig(properties);
    SpecExecutor specExecutorInstanceProducer = new InMemorySpecExecutor(config);
    TopologySpec.Builder topologySpecBuilder = TopologySpec.builder(computeTopologySpecURI()).withConfig(config).withDescription(SPEC_DESCRIPTION).withVersion(SPEC_VERSION).withSpecExecutor(specExecutorInstanceProducer);
    return topologySpecBuilder.build();
}