Examples with SpecExecutor org.apache.gobblin.runtime.api.SpecExecutor used on opensource projects

Example 11 with SpecExecutor

use of org.apache.gobblin.runtime.api.SpecExecutor in project incubator-gobblin by apache.

the class OrchestratorTest method createFlowSpec.

@Test(dependsOnMethods = "createTopologySpec")
public void createFlowSpec() throws Exception {
    // Since only 1 Topology with 1 SpecProducer has been added in previous test
    // .. it should be available and responsible for our new FlowSpec
    IdentityFlowToJobSpecCompiler specCompiler = (IdentityFlowToJobSpecCompiler) this.orchestrator.getSpecCompiler();
    SpecExecutor sei = specCompiler.getTopologySpecMap().values().iterator().next().getSpecExecutor();
    // List Current Specs
    Collection<Spec> specs = flowCatalog.getSpecs();
    logger.info("[Before Create] Number of specs: " + specs.size());
    int i = 0;
    for (Spec spec : specs) {
        FlowSpec flowSpec = (FlowSpec) spec;
        logger.info("[Before Create] Spec " + i++ + ": " + gson.toJson(flowSpec));
    }
    // Make sure FlowCatalog is empty
    Assert.assertTrue(specs.size() == 0, "Spec store should be empty before addition");
    // Make sure FlowCatalog Listener is empty
    Assert.assertTrue(((List) (sei.getProducer().get().listSpecs().get())).size() == 0, "SpecProducer should not know about " + "any Flow before addition");
    // Create and add Spec
    this.flowCatalog.put(flowSpec);
    // List Specs after adding
    specs = flowCatalog.getSpecs();
    logger.info("[After Create] Number of specs: " + specs.size());
    i = 0;
    for (Spec spec : specs) {
        flowSpec = (FlowSpec) spec;
        logger.info("[After Create] Spec " + i++ + ": " + gson.toJson(flowSpec));
    }
    // Make sure FlowCatalog has the added Flow
    Assert.assertTrue(specs.size() == 1, "Spec store should contain 1 Spec after addition");
    // Orchestrator is a no-op listener for any new FlowSpecs
    Assert.assertTrue(((List) (sei.getProducer().get().listSpecs().get())).size() == 0, "SpecProducer should contain 0 " + "Spec after addition");
}

Example 12 with SpecExecutor

use of org.apache.gobblin.runtime.api.SpecExecutor in project incubator-gobblin by apache.

the class MultiHopsFlowToJobSpecCompiler method userSpecifiedPathVerificator.

// If path specified not existed, return false;
// else return true.
private boolean userSpecifiedPathVerificator(Map<Spec, SpecExecutor> specExecutorInstanceMap, FlowSpec flowSpec) {
    Map<Spec, SpecExecutor> tmpSpecExecutorInstanceMap = new HashMap<>();
    List<String> userSpecfiedPath = Arrays.asList(optionalUserSpecifiedPath.get().split(","));
    for (int i = 0; i < userSpecfiedPath.size() - 1; i++) {
        ServiceNode sourceNode = new BaseServiceNodeImpl(userSpecfiedPath.get(i));
        ServiceNode targetNode = new BaseServiceNodeImpl(userSpecfiedPath.get(i + 1));
        if (weightedGraph.containsVertex(sourceNode) && weightedGraph.containsVertex(targetNode) && weightedGraph.containsEdge(sourceNode, targetNode)) {
            tmpSpecExecutorInstanceMap.put(convertHopToJobSpec(sourceNode, targetNode, flowSpec), (((LoadBasedFlowEdgeImpl) weightedGraph.getEdge(sourceNode, targetNode)).getSpecExecutorInstance()));
        } else {
            log.error("User Specified Path is invalid");
            return false;
        }
    }
    specExecutorInstanceMap.putAll(tmpSpecExecutorInstanceMap);
    return true;
}

Example 13 with SpecExecutor

use of org.apache.gobblin.runtime.api.SpecExecutor in project incubator-gobblin by apache.

the class ConfigBasedTopologySpecFactory method getTopologies.

@Override
public Collection<TopologySpec> getTopologies() {
    if (!_config.hasPath(ServiceConfigKeys.TOPOLOGY_FACTORY_TOPOLOGY_NAMES_KEY)) {
        return Collections.EMPTY_LIST;
    }
    Collection<TopologySpec> topologySpecs = Lists.newArrayList();
    Collection<String> topologyNames = SPLIT_BY_COMMA.splitToList(_config.getString(ServiceConfigKeys.TOPOLOGY_FACTORY_TOPOLOGY_NAMES_KEY));
    for (String topologyName : topologyNames) {
        Preconditions.checkArgument(_config.hasPath(ServiceConfigKeys.TOPOLOGY_FACTORY_PREFIX + topologyName), "Config does not contain Topology Factory descriptor for Topology " + topologyName);
        Config topologyConfig = _config.getConfig(ServiceConfigKeys.TOPOLOGY_FACTORY_PREFIX + topologyName);
        String description = ConfigUtils.getString(topologyConfig, ServiceConfigKeys.TOPOLOGYSPEC_DESCRIPTION_KEY, "NA");
        String version = ConfigUtils.getString(topologyConfig, ServiceConfigKeys.TOPOLOGYSPEC_VERSION_KEY, "-1");
        String specExecutorClass = ServiceConfigKeys.DEFAULT_SPEC_EXECUTOR;
        if (topologyConfig.hasPath(ServiceConfigKeys.SPEC_EXECUTOR_KEY)) {
            specExecutorClass = topologyConfig.getString(ServiceConfigKeys.SPEC_EXECUTOR_KEY);
        }
        SpecExecutor specExecutor;
        try {
            _log.info("Using SpecProducer class name/alias " + specExecutorClass);
            specExecutor = (SpecExecutor) ConstructorUtils.invokeConstructor(Class.forName(_aliasResolver.resolve(specExecutorClass)), topologyConfig);
        } catch (NoSuchMethodException | IllegalAccessException | InvocationTargetException | InstantiationException | ClassNotFoundException e) {
            throw new RuntimeException(e);
        }
        TopologySpec.Builder topologySpecBuilder = TopologySpec.builder(topologyConfig.getString(ServiceConfigKeys.TOPOLOGYSPEC_URI_KEY)).withConfig(topologyConfig).withDescription(description).withVersion(version).withSpecExecutor(specExecutor);
        topologySpecs.add(topologySpecBuilder.build());
    }
    return topologySpecs;
}

Example 14 with SpecExecutor

use of org.apache.gobblin.runtime.api.SpecExecutor in project incubator-gobblin by apache.

the class FlowCatalogTest method initFlowSpec.

private FlowSpec initFlowSpec() {
    Properties properties = new Properties();
    properties.put("specStore.fs.dir", SPEC_STORE_DIR);
    properties.put("specExecInstance.capabilities", "source:destination");
    Config config = ConfigUtils.propertiesToConfig(properties);
    SpecExecutor specExecutorInstanceProducer = new InMemorySpecExecutor(config);
    FlowSpec.Builder flowSpecBuilder = null;
    try {
        flowSpecBuilder = FlowSpec.builder(computeFlowSpecURI()).withConfig(config).withDescription(SPEC_DESCRIPTION).withVersion(SPEC_VERSION).withTemplate(new URI("templateURI"));
    } catch (URISyntaxException e) {
        throw new RuntimeException(e);
    }
    return flowSpecBuilder.build();
}

Example 15 with SpecExecutor

use of org.apache.gobblin.runtime.api.SpecExecutor in project incubator-gobblin by apache.

the class TopologyCatalogTest method initTopologySpec.

private TopologySpec initTopologySpec() {
    Properties properties = new Properties();
    properties.put("specStore.fs.dir", SPEC_STORE_DIR);
    properties.put("specExecInstance.capabilities", "source:destination");
    Config config = ConfigUtils.propertiesToConfig(properties);
    SpecExecutor specExecutorInstanceProducer = new InMemorySpecExecutor(config);
    TopologySpec.Builder topologySpecBuilder = TopologySpec.builder(computeTopologySpecURI()).withConfig(config).withDescription(SPEC_DESCRIPTION).withVersion(SPEC_VERSION).withSpecExecutor(specExecutorInstanceProducer);
    return topologySpecBuilder.build();
}