Examples with TopologySpec org.apache.gobblin.runtime.api.TopologySpec used on opensource projects

Example 11 with TopologySpec

use of org.apache.gobblin.runtime.api.TopologySpec in project incubator-gobblin by apache.

the class OrchestratorTest method initTopologySpec.

private TopologySpec initTopologySpec() {
    Properties properties = new Properties();
    properties.put("specStore.fs.dir", TOPOLOGY_SPEC_STORE_DIR);
    properties.put("specExecInstance.capabilities", "source:destination");
    Config config = ConfigUtils.propertiesToConfig(properties);
    SpecExecutor specExecutorInstance = new InMemorySpecExecutor(config);
    TopologySpec.Builder topologySpecBuilder = TopologySpec.builder(computeTopologySpecURI(SPEC_STORE_PARENT_DIR, TOPOLOGY_SPEC_STORE_DIR)).withConfig(config).withDescription(SPEC_DESCRIPTION).withVersion(SPEC_VERSION).withSpecExecutor(specExecutorInstance);
    return topologySpecBuilder.build();
}

Example 12 with TopologySpec

use of org.apache.gobblin.runtime.api.TopologySpec in project incubator-gobblin by apache.

the class ConfigBasedTopologySpecFactoryTest method testGetTopologies.

@Test
public void testGetTopologies() {
    Collection<TopologySpec> topologySpecs = _configBasedTopologySpecFactory.getTopologies();
    Assert.assertTrue(topologySpecs.size() == 2, "Expected 2 topologies but received: " + topologySpecs.size());
    Iterator<TopologySpec> topologySpecIterator = topologySpecs.iterator();
    TopologySpec topologySpec1 = topologySpecIterator.next();
    Assert.assertTrue(topologySpec1.getDescription().equals("Topology for cluster"), "Description did not match with construction");
    Assert.assertTrue(topologySpec1.getVersion().equals("1"), "Version did not match with construction");
    TopologySpec topologySpec2 = topologySpecIterator.next();
    Assert.assertTrue(topologySpec2.getDescription().equals("Topology for Azkaban"), "Description did not match with construction");
    Assert.assertTrue(topologySpec2.getVersion().equals("2"), "Version did not match with construction");
}

Example 13 with TopologySpec

use of org.apache.gobblin.runtime.api.TopologySpec in project incubator-gobblin by apache.

the class ConfigBasedTopologySpecFactory method getTopologies.

@Override
public Collection<TopologySpec> getTopologies() {
    if (!_config.hasPath(ServiceConfigKeys.TOPOLOGY_FACTORY_TOPOLOGY_NAMES_KEY)) {
        return Collections.EMPTY_LIST;
    }
    Collection<TopologySpec> topologySpecs = Lists.newArrayList();
    Collection<String> topologyNames = SPLIT_BY_COMMA.splitToList(_config.getString(ServiceConfigKeys.TOPOLOGY_FACTORY_TOPOLOGY_NAMES_KEY));
    for (String topologyName : topologyNames) {
        Preconditions.checkArgument(_config.hasPath(ServiceConfigKeys.TOPOLOGY_FACTORY_PREFIX + topologyName), "Config does not contain Topology Factory descriptor for Topology " + topologyName);
        Config topologyConfig = _config.getConfig(ServiceConfigKeys.TOPOLOGY_FACTORY_PREFIX + topologyName);
        String description = ConfigUtils.getString(topologyConfig, ServiceConfigKeys.TOPOLOGYSPEC_DESCRIPTION_KEY, "NA");
        String version = ConfigUtils.getString(topologyConfig, ServiceConfigKeys.TOPOLOGYSPEC_VERSION_KEY, "-1");
        String specExecutorClass = ServiceConfigKeys.DEFAULT_SPEC_EXECUTOR;
        if (topologyConfig.hasPath(ServiceConfigKeys.SPEC_EXECUTOR_KEY)) {
            specExecutorClass = topologyConfig.getString(ServiceConfigKeys.SPEC_EXECUTOR_KEY);
        }
        SpecExecutor specExecutor;
        try {
            _log.info("Using SpecProducer class name/alias " + specExecutorClass);
            specExecutor = (SpecExecutor) ConstructorUtils.invokeConstructor(Class.forName(_aliasResolver.resolve(specExecutorClass)), topologyConfig);
        } catch (NoSuchMethodException | IllegalAccessException | InvocationTargetException | InstantiationException | ClassNotFoundException e) {
            throw new RuntimeException(e);
        }
        TopologySpec.Builder topologySpecBuilder = TopologySpec.builder(topologyConfig.getString(ServiceConfigKeys.TOPOLOGYSPEC_URI_KEY)).withConfig(topologyConfig).withDescription(description).withVersion(version).withSpecExecutor(specExecutor);
        topologySpecs.add(topologySpecBuilder.build());
    }
    return topologySpecs;
}

Example 14 with TopologySpec

use of org.apache.gobblin.runtime.api.TopologySpec in project incubator-gobblin by apache.

the class TopologyCatalogTest method initTopologySpec.

private TopologySpec initTopologySpec() {
    Properties properties = new Properties();
    properties.put("specStore.fs.dir", SPEC_STORE_DIR);
    properties.put("specExecInstance.capabilities", "source:destination");
    Config config = ConfigUtils.propertiesToConfig(properties);
    SpecExecutor specExecutorInstanceProducer = new InMemorySpecExecutor(config);
    TopologySpec.Builder topologySpecBuilder = TopologySpec.builder(computeTopologySpecURI()).withConfig(config).withDescription(SPEC_DESCRIPTION).withVersion(SPEC_VERSION).withSpecExecutor(specExecutorInstanceProducer);
    return topologySpecBuilder.build();
}

Example 15 with TopologySpec

use of org.apache.gobblin.runtime.api.TopologySpec in project incubator-gobblin by apache.

the class TopologyCatalogTest method createTopologySpec.

@Test
public void createTopologySpec() {
    // List Current Specs
    Collection<Spec> specs = topologyCatalog.getSpecs();
    logger.info("[Before Create] Number of specs: " + specs.size());
    int i = 0;
    for (Spec spec : specs) {
        TopologySpec topologySpec = (TopologySpec) spec;
        logger.info("[Before Create] Spec " + i++ + ": " + gson.toJson(topologySpec));
    }
    Assert.assertTrue(specs.size() == 0, "Spec store should be empty before addition");
    // Create and add Spec
    this.topologyCatalog.put(topologySpec);
    // List Specs after adding
    specs = topologyCatalog.getSpecs();
    logger.info("[After Create] Number of specs: " + specs.size());
    i = 0;
    for (Spec spec : specs) {
        topologySpec = (TopologySpec) spec;
        logger.info("[After Create] Spec " + i++ + ": " + gson.toJson(topologySpec));
    }
    Assert.assertTrue(specs.size() == 1, "Spec store should contain 1 Spec after addition");
}