Examples with Spec org.apache.gobblin.runtime.api.Spec used on opensource projects

Example 26 with Spec

use of org.apache.gobblin.runtime.api.Spec in project incubator-gobblin by apache.

the class MultiHopsFlowToJobSpecCompiler method userSpecifiedPathVerificator.

// If path specified not existed, return false;
// else return true.
private boolean userSpecifiedPathVerificator(Map<Spec, SpecExecutor> specExecutorInstanceMap, FlowSpec flowSpec) {
    Map<Spec, SpecExecutor> tmpSpecExecutorInstanceMap = new HashMap<>();
    List<String> userSpecfiedPath = Arrays.asList(optionalUserSpecifiedPath.get().split(","));
    for (int i = 0; i < userSpecfiedPath.size() - 1; i++) {
        ServiceNode sourceNode = new BaseServiceNodeImpl(userSpecfiedPath.get(i));
        ServiceNode targetNode = new BaseServiceNodeImpl(userSpecfiedPath.get(i + 1));
        if (weightedGraph.containsVertex(sourceNode) && weightedGraph.containsVertex(targetNode) && weightedGraph.containsEdge(sourceNode, targetNode)) {
            tmpSpecExecutorInstanceMap.put(convertHopToJobSpec(sourceNode, targetNode, flowSpec), (((LoadBasedFlowEdgeImpl) weightedGraph.getEdge(sourceNode, targetNode)).getSpecExecutorInstance()));
        } else {
            log.error("User Specified Path is invalid");
            return false;
        }
    }
    specExecutorInstanceMap.putAll(tmpSpecExecutorInstanceMap);
    return true;
}

Example 27 with Spec

use of org.apache.gobblin.runtime.api.Spec in project incubator-gobblin by apache.

the class GobblinServiceJobScheduler method onDeleteSpec.

/**
 * {@inheritDoc}
 */
@Override
public void onDeleteSpec(URI deletedSpecURI, String deletedSpecVersion) {
    if (this.helixManager.isPresent() && !this.helixManager.get().isConnected()) {
        // Specs in store will be notified when Scheduler is added as listener to FlowCatalog, so ignore
        // .. Specs if in cluster mode and Helix is not yet initialized
        _log.info("System not yet initialized. Skipping Spec Deletion: " + deletedSpecURI);
        return;
    }
    _log.info("Spec deletion detected: " + deletedSpecURI + "/" + deletedSpecVersion);
    if (!isActive && helixManager.isPresent()) {
        _log.info("Scheduler running in slave mode, forward Spec delete via Helix message to master: " + deletedSpecURI);
        HelixUtils.sendUserDefinedMessage(ServiceConfigKeys.HELIX_FLOWSPEC_REMOVE, deletedSpecURI.toString() + ":" + deletedSpecVersion, UUID.randomUUID().toString(), InstanceType.CONTROLLER, helixManager.get(), _log);
        return;
    }
    try {
        Spec deletedSpec = this.scheduledFlowSpecs.get(deletedSpecURI.toString());
        if (null != deletedSpec) {
            this.orchestrator.remove(deletedSpec);
            this.scheduledFlowSpecs.remove(deletedSpecURI.toString());
            unscheduleJob(deletedSpecURI.toString());
        } else {
            _log.warn(String.format("Spec with URI: %s was not found in cache. May be it was cleaned, if not please " + "clean it manually", deletedSpecURI));
        }
    } catch (JobException e) {
        _log.warn(String.format("Spec with URI: %s was not unscheduled cleaning", deletedSpecURI), e);
    }
}

Example 28 with Spec

use of org.apache.gobblin.runtime.api.Spec in project incubator-gobblin by apache.

the class GobblinServiceJobScheduler method setActive.

public synchronized void setActive(boolean isActive) {
    if (this.isActive == isActive) {
        // No-op if already in correct state
        return;
    }
    // Since we are going to change status to isActive=true, schedule all flows
    if (isActive) {
        // Need to set active first; otherwise in the STANDBY->ACTIVE transition,
        // the onAddSpec will forward specs to the leader, which is itself.
        this.isActive = isActive;
        if (this.flowCatalog.isPresent()) {
            Collection<Spec> specs = this.flowCatalog.get().getSpecsWithTimeUpdate();
            for (Spec spec : specs) {
                // Disable FLOW_RUN_IMMEDIATELY on service startup or leadership change
                if (spec instanceof FlowSpec) {
                    Spec modifiedSpec = disableFlowRunImmediatelyOnStart((FlowSpec) spec);
                    onAddSpec(modifiedSpec);
                } else {
                    onAddSpec(spec);
                }
            }
        }
    } else // Since we are going to change status to isActive=false, unschedule all flows
    {
        for (Spec spec : this.scheduledFlowSpecs.values()) {
            onDeleteSpec(spec.getUri(), spec.getVersion());
        }
        // Need to set active at the end; otherwise in the ACTIVE->STANDBY transition,
        // the onDeleteSpec will forward specs to the leader, which is itself.
        this.isActive = isActive;
    }
}

Example 29 with Spec

use of org.apache.gobblin.runtime.api.Spec in project incubator-gobblin by apache.

the class FlowCatalogTest method deleteFlowSpec.

@Test(dependsOnMethods = "testExist")
public void deleteFlowSpec() throws SpecNotFoundException {
    // List Current Specs
    Collection<Spec> specs = flowCatalog.getSpecs();
    logger.info("[Before Delete] Number of specs: " + specs.size());
    int i = 0;
    for (Spec spec : specs) {
        FlowSpec flowSpec = (FlowSpec) spec;
        logger.info("[Before Delete] Spec " + i++ + ": " + gson.toJson(flowSpec));
    }
    Assert.assertTrue(specs.size() == 1, "Spec store should initially have 1 Spec before deletion");
    this.flowCatalog.remove(flowSpec.getUri());
    // List Specs after adding
    specs = flowCatalog.getSpecs();
    logger.info("[After Delete] Number of specs: " + specs.size());
    i = 0;
    for (Spec spec : specs) {
        flowSpec = (FlowSpec) spec;
        logger.info("[After Delete] Spec " + i++ + ": " + gson.toJson(flowSpec));
    }
    Assert.assertTrue(specs.size() == 0, "Spec store should be empty after deletion");
}

Example 30 with Spec

use of org.apache.gobblin.runtime.api.Spec in project incubator-gobblin by apache.

the class TopologyCatalogTest method createTopologySpec.

@Test
public void createTopologySpec() {
    // List Current Specs
    Collection<Spec> specs = topologyCatalog.getSpecs();
    logger.info("[Before Create] Number of specs: " + specs.size());
    int i = 0;
    for (Spec spec : specs) {
        TopologySpec topologySpec = (TopologySpec) spec;
        logger.info("[Before Create] Spec " + i++ + ": " + gson.toJson(topologySpec));
    }
    Assert.assertTrue(specs.size() == 0, "Spec store should be empty before addition");
    // Create and add Spec
    this.topologyCatalog.put(topologySpec);
    // List Specs after adding
    specs = topologyCatalog.getSpecs();
    logger.info("[After Create] Number of specs: " + specs.size());
    i = 0;
    for (Spec spec : specs) {
        topologySpec = (TopologySpec) spec;
        logger.info("[After Create] Spec " + i++ + ": " + gson.toJson(topologySpec));
    }
    Assert.assertTrue(specs.size() == 1, "Spec store should contain 1 Spec after addition");
}