Examples with FlowSpec org.apache.gobblin.runtime.api.FlowSpec used on opensource projects

Example 21 with FlowSpec

use of org.apache.gobblin.runtime.api.FlowSpec in project incubator-gobblin by apache.

the class OrchestratorTest method deleteFlowSpec.

@Test(dependsOnMethods = "createFlowSpec")
public void deleteFlowSpec() throws Exception {
    // Since only 1 Flow has been added in previous test it should be available
    IdentityFlowToJobSpecCompiler specCompiler = (IdentityFlowToJobSpecCompiler) this.orchestrator.getSpecCompiler();
    SpecExecutor sei = specCompiler.getTopologySpecMap().values().iterator().next().getSpecExecutor();
    // List Current Specs
    Collection<Spec> specs = flowCatalog.getSpecs();
    logger.info("[Before Delete] Number of specs: " + specs.size());
    int i = 0;
    for (Spec spec : specs) {
        FlowSpec flowSpec = (FlowSpec) spec;
        logger.info("[Before Delete] Spec " + i++ + ": " + gson.toJson(flowSpec));
    }
    // Make sure FlowCatalog has the previously added Flow
    Assert.assertTrue(specs.size() == 1, "Spec store should contain 1 Flow that was added in last test");
    // Orchestrator is a no-op listener for any new FlowSpecs, so no FlowSpecs should be around
    int specsInSEI = ((List) (sei.getProducer().get().listSpecs().get())).size();
    Assert.assertTrue(specsInSEI == 0, "SpecProducer should contain 0 " + "Spec after addition because Orchestrator is a no-op listener for any new FlowSpecs");
    // Remove the flow
    this.flowCatalog.remove(flowSpec.getUri());
    // List Specs after adding
    specs = flowCatalog.getSpecs();
    logger.info("[After Delete] Number of specs: " + specs.size());
    i = 0;
    for (Spec spec : specs) {
        flowSpec = (FlowSpec) spec;
        logger.info("[After Delete] Spec " + i++ + ": " + gson.toJson(flowSpec));
    }
    // Make sure FlowCatalog has the Flow removed
    Assert.assertTrue(specs.size() == 0, "Spec store should not contain Spec after deletion");
    // Make sure FlowCatalog Listener knows about the deletion
    specsInSEI = ((List) (sei.getProducer().get().listSpecs().get())).size();
    Assert.assertTrue(specsInSEI == 0, "SpecProducer should not contain " + "Spec after deletion");
}

Example 22 with FlowSpec

use of org.apache.gobblin.runtime.api.FlowSpec in project incubator-gobblin by apache.

the class OrchestratorTest method initFlowSpec.

private FlowSpec initFlowSpec() {
    Properties properties = new Properties();
    properties.put("specStore.fs.dir", FLOW_SPEC_STORE_DIR);
    properties.put("specExecInstance.capabilities", "source:destination");
    properties.put("gobblin.flow.sourceIdentifier", "source");
    properties.put("gobblin.flow.destinationIdentifier", "destination");
    Config config = ConfigUtils.propertiesToConfig(properties);
    FlowSpec.Builder flowSpecBuilder = null;
    try {
        flowSpecBuilder = FlowSpec.builder(computeTopologySpecURI(SPEC_STORE_PARENT_DIR, FLOW_SPEC_GROUP_DIR)).withConfig(config).withDescription(SPEC_DESCRIPTION).withVersion(SPEC_VERSION).withTemplate(new URI("templateURI"));
    } catch (URISyntaxException e) {
        throw new RuntimeException(e);
    }
    return flowSpecBuilder.build();
}

Example 23 with FlowSpec

use of org.apache.gobblin.runtime.api.FlowSpec in project incubator-gobblin by apache.

the class OrchestratorTest method createFlowSpec.

@Test(dependsOnMethods = "createTopologySpec")
public void createFlowSpec() throws Exception {
    // Since only 1 Topology with 1 SpecProducer has been added in previous test
    // .. it should be available and responsible for our new FlowSpec
    IdentityFlowToJobSpecCompiler specCompiler = (IdentityFlowToJobSpecCompiler) this.orchestrator.getSpecCompiler();
    SpecExecutor sei = specCompiler.getTopologySpecMap().values().iterator().next().getSpecExecutor();
    // List Current Specs
    Collection<Spec> specs = flowCatalog.getSpecs();
    logger.info("[Before Create] Number of specs: " + specs.size());
    int i = 0;
    for (Spec spec : specs) {
        FlowSpec flowSpec = (FlowSpec) spec;
        logger.info("[Before Create] Spec " + i++ + ": " + gson.toJson(flowSpec));
    }
    // Make sure FlowCatalog is empty
    Assert.assertTrue(specs.size() == 0, "Spec store should be empty before addition");
    // Make sure FlowCatalog Listener is empty
    Assert.assertTrue(((List) (sei.getProducer().get().listSpecs().get())).size() == 0, "SpecProducer should not know about " + "any Flow before addition");
    // Create and add Spec
    this.flowCatalog.put(flowSpec);
    // List Specs after adding
    specs = flowCatalog.getSpecs();
    logger.info("[After Create] Number of specs: " + specs.size());
    i = 0;
    for (Spec spec : specs) {
        flowSpec = (FlowSpec) spec;
        logger.info("[After Create] Spec " + i++ + ": " + gson.toJson(flowSpec));
    }
    // Make sure FlowCatalog has the added Flow
    Assert.assertTrue(specs.size() == 1, "Spec store should contain 1 Spec after addition");
    // Orchestrator is a no-op listener for any new FlowSpecs
    Assert.assertTrue(((List) (sei.getProducer().get().listSpecs().get())).size() == 0, "SpecProducer should contain 0 " + "Spec after addition");
}

Example 24 with FlowSpec

use of org.apache.gobblin.runtime.api.FlowSpec in project incubator-gobblin by apache.

the class GobblinServiceJobSchedulerTest method testDisableFlowRunImmediatelyOnStart.

@Test
public void testDisableFlowRunImmediatelyOnStart() throws Exception {
    Properties properties = new Properties();
    properties.setProperty(ConfigurationKeys.FLOW_RUN_IMMEDIATELY, "true");
    properties.setProperty(ConfigurationKeys.JOB_SCHEDULE_KEY, TEST_SCHEDULE);
    properties.setProperty(ConfigurationKeys.JOB_GROUP_KEY, TEST_GROUP_NAME);
    properties.setProperty(ConfigurationKeys.JOB_NAME_KEY, TEST_FLOW_NAME);
    Config config = ConfigFactory.parseProperties(properties);
    FlowSpec spec = FlowSpec.builder().withTemplate(new URI(TEST_TEMPLATE_URI)).withVersion("version").withConfigAsProperties(properties).withConfig(config).build();
    FlowSpec modifiedSpec = (FlowSpec) GobblinServiceJobScheduler.disableFlowRunImmediatelyOnStart(spec);
    for (URI templateURI : modifiedSpec.getTemplateURIs().get()) {
        Assert.assertEquals(templateURI.toString(), TEST_TEMPLATE_URI);
    }
    Assert.assertEquals(modifiedSpec.getVersion(), "version");
    Config modifiedConfig = modifiedSpec.getConfig();
    Assert.assertFalse(modifiedConfig.getBoolean(ConfigurationKeys.FLOW_RUN_IMMEDIATELY));
    Assert.assertEquals(modifiedConfig.getString(ConfigurationKeys.JOB_GROUP_KEY), TEST_GROUP_NAME);
    Assert.assertEquals(modifiedConfig.getString(ConfigurationKeys.JOB_NAME_KEY), TEST_FLOW_NAME);
}

Example 25 with FlowSpec

use of org.apache.gobblin.runtime.api.FlowSpec in project incubator-gobblin by apache.

the class MultiHopsFlowToJobSpecCompiler method userSpecifiedPathVerificator.

// If path specified not existed, return false;
// else return true.
private boolean userSpecifiedPathVerificator(Map<Spec, SpecExecutor> specExecutorInstanceMap, FlowSpec flowSpec) {
    Map<Spec, SpecExecutor> tmpSpecExecutorInstanceMap = new HashMap<>();
    List<String> userSpecfiedPath = Arrays.asList(optionalUserSpecifiedPath.get().split(","));
    for (int i = 0; i < userSpecfiedPath.size() - 1; i++) {
        ServiceNode sourceNode = new BaseServiceNodeImpl(userSpecfiedPath.get(i));
        ServiceNode targetNode = new BaseServiceNodeImpl(userSpecfiedPath.get(i + 1));
        if (weightedGraph.containsVertex(sourceNode) && weightedGraph.containsVertex(targetNode) && weightedGraph.containsEdge(sourceNode, targetNode)) {
            tmpSpecExecutorInstanceMap.put(convertHopToJobSpec(sourceNode, targetNode, flowSpec), (((LoadBasedFlowEdgeImpl) weightedGraph.getEdge(sourceNode, targetNode)).getSpecExecutorInstance()));
        } else {
            log.error("User Specified Path is invalid");
            return false;
        }
    }
    specExecutorInstanceMap.putAll(tmpSpecExecutorInstanceMap);
    return true;
}