Examples with FlowSpec org.apache.gobblin.runtime.api.FlowSpec used on opensource projects

Example 26 with FlowSpec

use of org.apache.gobblin.runtime.api.FlowSpec in project incubator-gobblin by apache.

the class MultiHopsFlowToJobSpecCompiler method findPath.

// Basically a dijkstra path finding for connecting source and sink by multiple hops in between.
// If there's any user-specified prioritization, conduct the DFS and see if the user-specified path is available.
// there's no updates on TopologySpec, or user should be aware of the possibility
// that a topologySpec not being reflected in findPath.
private void findPath(Map<Spec, SpecExecutor> specExecutorInstanceMap, Spec spec) {
    inMemoryWeightGraphGenerator();
    FlowSpec flowSpec = (FlowSpec) spec;
    if (optionalUserSpecifiedPath.isPresent()) {
        log.info("Starting to evaluate user's specified path ... ");
        if (userSpecifiedPathVerificator(specExecutorInstanceMap, flowSpec)) {
            log.info("User specified path[ " + optionalUserSpecifiedPath.get() + "] successfully verified.");
            return;
        } else {
            log.error("Will not execute user specified path[ " + optionalUserSpecifiedPath.get() + "]");
            log.info("Start to execute FlowCompiler's algorithm for valid data movement path");
        }
    }
    ServiceNode sourceNode = new BaseServiceNodeImpl(flowSpec.getConfig().getString(ServiceConfigKeys.FLOW_SOURCE_IDENTIFIER_KEY));
    ServiceNode targetNode = new BaseServiceNodeImpl(flowSpec.getConfig().getString(ServiceConfigKeys.FLOW_DESTINATION_IDENTIFIER_KEY));
    List<FlowEdge> resultEdgePath = dijkstraBasedPathFindingHelper(sourceNode, targetNode, this.weightedGraph);
    for (int i = 0; i < resultEdgePath.size(); i++) {
        FlowEdge tmpFlowEdge = resultEdgePath.get(i);
        ServiceNode edgeSrcNode = ((LoadBasedFlowEdgeImpl) tmpFlowEdge).getSourceNode();
        ServiceNode edgeTgtNode = ((LoadBasedFlowEdgeImpl) tmpFlowEdge).getTargetNode();
        specExecutorInstanceMap.put(convertHopToJobSpec(edgeSrcNode, edgeTgtNode, flowSpec), ((LoadBasedFlowEdgeImpl) (resultEdgePath.get(i))).getSpecExecutorInstance());
    }
}

Example 27 with FlowSpec

use of org.apache.gobblin.runtime.api.FlowSpec in project incubator-gobblin by apache.

the class MultiHopsFlowToJobSpecCompiler method jobSpecURIGenerator.

/**
 * A naive implementation of generating a jobSpec's URI within a multi-hop logical Flow.
 */
@Override
public URI jobSpecURIGenerator(Object... objects) {
    FlowSpec flowSpec = (FlowSpec) objects[0];
    ServiceNode sourceNode = (ServiceNode) objects[1];
    ServiceNode targetNode = (ServiceNode) objects[2];
    try {
        return new URI(JobSpec.Builder.DEFAULT_JOB_CATALOG_SCHEME, flowSpec.getUri().getAuthority(), StringUtils.appendIfMissing(StringUtils.prependIfMissing(flowSpec.getUri().getPath(), "/"), "/") + sourceNode.getNodeName() + "-" + targetNode.getNodeName(), null);
    } catch (URISyntaxException e) {
        log.error("URI construction failed when jobSpec from " + sourceNode.getNodeName() + " to " + targetNode.getNodeName());
        throw new RuntimeException();
    }
}

Example 28 with FlowSpec

use of org.apache.gobblin.runtime.api.FlowSpec in project incubator-gobblin by apache.

the class GobblinServiceJobScheduler method setActive.

public synchronized void setActive(boolean isActive) {
    if (this.isActive == isActive) {
        // No-op if already in correct state
        return;
    }
    // Since we are going to change status to isActive=true, schedule all flows
    if (isActive) {
        // Need to set active first; otherwise in the STANDBY->ACTIVE transition,
        // the onAddSpec will forward specs to the leader, which is itself.
        this.isActive = isActive;
        if (this.flowCatalog.isPresent()) {
            Collection<Spec> specs = this.flowCatalog.get().getSpecsWithTimeUpdate();
            for (Spec spec : specs) {
                // Disable FLOW_RUN_IMMEDIATELY on service startup or leadership change
                if (spec instanceof FlowSpec) {
                    Spec modifiedSpec = disableFlowRunImmediatelyOnStart((FlowSpec) spec);
                    onAddSpec(modifiedSpec);
                } else {
                    onAddSpec(spec);
                }
            }
        }
    } else // Since we are going to change status to isActive=false, unschedule all flows
    {
        for (Spec spec : this.scheduledFlowSpecs.values()) {
            onDeleteSpec(spec.getUri(), spec.getVersion());
        }
        // Need to set active at the end; otherwise in the ACTIVE->STANDBY transition,
        // the onDeleteSpec will forward specs to the leader, which is itself.
        this.isActive = isActive;
    }
}

Example 29 with FlowSpec

use of org.apache.gobblin.runtime.api.FlowSpec in project incubator-gobblin by apache.

the class GobblinServiceJobScheduler method onAddSpec.

/**
 * {@inheritDoc}
 */
@Override
public void onAddSpec(Spec addedSpec) {
    if (this.helixManager.isPresent() && !this.helixManager.get().isConnected()) {
        // Specs in store will be notified when Scheduler is added as listener to FlowCatalog, so ignore
        // .. Specs if in cluster mode and Helix is not yet initialized
        _log.info("System not yet initialized. Skipping Spec Addition: " + addedSpec);
        return;
    }
    _log.info("New Flow Spec detected: " + addedSpec);
    if (addedSpec instanceof FlowSpec) {
        if (!isActive && helixManager.isPresent()) {
            _log.info("Scheduler running in slave mode, forward Spec add via Helix message to master: " + addedSpec);
            HelixUtils.sendUserDefinedMessage(ServiceConfigKeys.HELIX_FLOWSPEC_ADD, addedSpec.getUri().toString(), UUID.randomUUID().toString(), InstanceType.CONTROLLER, helixManager.get(), _log);
            return;
        }
        try {
            Properties jobConfig = new Properties();
            Properties flowSpecProperties = ((FlowSpec) addedSpec).getConfigAsProperties();
            jobConfig.putAll(this.properties);
            jobConfig.setProperty(ConfigurationKeys.JOB_NAME_KEY, addedSpec.getUri().toString());
            jobConfig.setProperty(ConfigurationKeys.JOB_GROUP_KEY, ((FlowSpec) addedSpec).getConfig().getValue(ConfigurationKeys.FLOW_GROUP_KEY).toString());
            jobConfig.setProperty(ConfigurationKeys.FLOW_RUN_IMMEDIATELY, ConfigUtils.getString(((FlowSpec) addedSpec).getConfig(), ConfigurationKeys.FLOW_RUN_IMMEDIATELY, "false"));
            if (flowSpecProperties.containsKey(ConfigurationKeys.JOB_SCHEDULE_KEY) && StringUtils.isNotBlank(flowSpecProperties.getProperty(ConfigurationKeys.JOB_SCHEDULE_KEY))) {
                jobConfig.setProperty(ConfigurationKeys.JOB_SCHEDULE_KEY, flowSpecProperties.getProperty(ConfigurationKeys.JOB_SCHEDULE_KEY));
            }
            this.scheduledFlowSpecs.put(addedSpec.getUri().toString(), addedSpec);
            if (jobConfig.containsKey(ConfigurationKeys.JOB_SCHEDULE_KEY)) {
                _log.info("Scheduling flow spec: " + addedSpec);
                scheduleJob(jobConfig, null);
                if (PropertiesUtils.getPropAsBoolean(jobConfig, ConfigurationKeys.FLOW_RUN_IMMEDIATELY, "false")) {
                    _log.info("RunImmediately requested, hence executing FlowSpec: " + addedSpec);
                    this.jobExecutor.execute(new NonScheduledJobRunner(jobConfig, null));
                }
            } else {
                _log.info("No FlowSpec schedule found, so running FlowSpec: " + addedSpec);
                this.jobExecutor.execute(new NonScheduledJobRunner(jobConfig, null));
            }
        } catch (JobException je) {
            _log.error("Failed to schedule or run FlowSpec " + addedSpec, je);
        }
    }
}

Example 30 with FlowSpec

use of org.apache.gobblin.runtime.api.FlowSpec in project incubator-gobblin by apache.

the class FlowCatalogTest method deleteFlowSpec.

@Test(dependsOnMethods = "testExist")
public void deleteFlowSpec() throws SpecNotFoundException {
    // List Current Specs
    Collection<Spec> specs = flowCatalog.getSpecs();
    logger.info("[Before Delete] Number of specs: " + specs.size());
    int i = 0;
    for (Spec spec : specs) {
        FlowSpec flowSpec = (FlowSpec) spec;
        logger.info("[Before Delete] Spec " + i++ + ": " + gson.toJson(flowSpec));
    }
    Assert.assertTrue(specs.size() == 1, "Spec store should initially have 1 Spec before deletion");
    this.flowCatalog.remove(flowSpec.getUri());
    // List Specs after adding
    specs = flowCatalog.getSpecs();
    logger.info("[After Delete] Number of specs: " + specs.size());
    i = 0;
    for (Spec spec : specs) {
        flowSpec = (FlowSpec) spec;
        logger.info("[After Delete] Spec " + i++ + ": " + gson.toJson(flowSpec));
    }
    Assert.assertTrue(specs.size() == 0, "Spec store should be empty after deletion");
}