use of org.apache.gobblin.runtime.api.Spec in project incubator-gobblin by apache.
the class IdentityFlowToJobSpecCompilerTest method testNoJobSpecCompilation.
@Test
public void testNoJobSpecCompilation() {
FlowSpec flowSpec = initFlowSpec(TEST_FLOW_GROUP, TEST_FLOW_NAME, "unsupportedSource", "unsupportedSink");
// Run compiler on flowSpec
Map<Spec, SpecExecutor> specExecutorMapping = this.compilerWithTemplateCalague.compileFlow(flowSpec);
// Assert pre-requisites
Assert.assertNotNull(specExecutorMapping, "Expected non null mapping.");
Assert.assertTrue(specExecutorMapping.size() == 0, "Exepected 1 executor for FlowSpec.");
}
use of org.apache.gobblin.runtime.api.Spec in project incubator-gobblin by apache.
the class IdentityFlowToJobSpecCompilerTest method testCompilerWithTemplateCatalog.
@Test
public void testCompilerWithTemplateCatalog() {
FlowSpec flowSpec = initFlowSpec();
// Run compiler on flowSpec
Map<Spec, SpecExecutor> specExecutorMapping = this.compilerWithTemplateCalague.compileFlow(flowSpec);
// Assert pre-requisites
Assert.assertNotNull(specExecutorMapping, "Expected non null mapping.");
Assert.assertTrue(specExecutorMapping.size() == 1, "Exepected 1 executor for FlowSpec.");
// Assert FlowSpec compilation
Spec spec = specExecutorMapping.keySet().iterator().next();
Assert.assertTrue(spec instanceof JobSpec, "Expected JobSpec compiled from FlowSpec.");
// Assert JobSpec properties
JobSpec jobSpec = (JobSpec) spec;
Assert.assertEquals(jobSpec.getConfig().getString("testProperty1"), "testValue1");
Assert.assertEquals(jobSpec.getConfig().getString("testProperty2"), "test.Value1");
Assert.assertEquals(jobSpec.getConfig().getString("testProperty3"), "100");
Assert.assertEquals(jobSpec.getConfig().getString(ServiceConfigKeys.FLOW_SOURCE_IDENTIFIER_KEY), TEST_SOURCE_NAME);
Assert.assertFalse(jobSpec.getConfig().hasPath(ConfigurationKeys.JOB_SCHEDULE_KEY));
Assert.assertEquals(jobSpec.getConfig().getString(ConfigurationKeys.JOB_NAME_KEY), TEST_FLOW_NAME);
Assert.assertEquals(jobSpec.getConfig().getString(ConfigurationKeys.JOB_GROUP_KEY), TEST_FLOW_GROUP);
Assert.assertEquals(jobSpec.getConfig().getString(ConfigurationKeys.FLOW_NAME_KEY), TEST_FLOW_NAME);
Assert.assertEquals(jobSpec.getConfig().getString(ConfigurationKeys.FLOW_GROUP_KEY), TEST_FLOW_GROUP);
Assert.assertTrue(jobSpec.getConfig().hasPath(ConfigurationKeys.FLOW_EXECUTION_ID_KEY));
}
use of org.apache.gobblin.runtime.api.Spec in project incubator-gobblin by apache.
the class OrchestratorTest method createTopologySpec.
@Test
public void createTopologySpec() {
IdentityFlowToJobSpecCompiler specCompiler = (IdentityFlowToJobSpecCompiler) this.orchestrator.getSpecCompiler();
// List Current Specs
Collection<Spec> specs = topologyCatalog.getSpecs();
logger.info("[Before Create] Number of specs: " + specs.size());
int i = 0;
for (Spec spec : specs) {
TopologySpec topologySpec = (TopologySpec) spec;
logger.info("[Before Create] Spec " + i++ + ": " + gson.toJson(topologySpec));
}
// Make sure TopologyCatalog is empty
Assert.assertTrue(specs.size() == 0, "Spec store should be empty before addition");
// Make sure TopologyCatalog Listener is empty
Assert.assertTrue(specCompiler.getTopologySpecMap().size() == 0, "SpecCompiler should not know about any Topology " + "before addition");
// Create and add Spec
this.topologyCatalog.put(topologySpec);
// List Specs after adding
specs = topologyCatalog.getSpecs();
logger.info("[After Create] Number of specs: " + specs.size());
i = 0;
for (Spec spec : specs) {
topologySpec = (TopologySpec) spec;
logger.info("[After Create] Spec " + i++ + ": " + gson.toJson(topologySpec));
}
// Make sure TopologyCatalog has the added Topology
Assert.assertTrue(specs.size() == 1, "Spec store should contain 1 Spec after addition");
// Make sure TopologyCatalog Listener knows about added Topology
Assert.assertTrue(specCompiler.getTopologySpecMap().size() == 1, "SpecCompiler should contain 1 Spec after addition");
}
use of org.apache.gobblin.runtime.api.Spec in project incubator-gobblin by apache.
the class OrchestratorTest method deleteFlowSpec.
@Test(dependsOnMethods = "createFlowSpec")
public void deleteFlowSpec() throws Exception {
// Since only 1 Flow has been added in previous test it should be available
IdentityFlowToJobSpecCompiler specCompiler = (IdentityFlowToJobSpecCompiler) this.orchestrator.getSpecCompiler();
SpecExecutor sei = specCompiler.getTopologySpecMap().values().iterator().next().getSpecExecutor();
// List Current Specs
Collection<Spec> specs = flowCatalog.getSpecs();
logger.info("[Before Delete] Number of specs: " + specs.size());
int i = 0;
for (Spec spec : specs) {
FlowSpec flowSpec = (FlowSpec) spec;
logger.info("[Before Delete] Spec " + i++ + ": " + gson.toJson(flowSpec));
}
// Make sure FlowCatalog has the previously added Flow
Assert.assertTrue(specs.size() == 1, "Spec store should contain 1 Flow that was added in last test");
// Orchestrator is a no-op listener for any new FlowSpecs, so no FlowSpecs should be around
int specsInSEI = ((List) (sei.getProducer().get().listSpecs().get())).size();
Assert.assertTrue(specsInSEI == 0, "SpecProducer should contain 0 " + "Spec after addition because Orchestrator is a no-op listener for any new FlowSpecs");
// Remove the flow
this.flowCatalog.remove(flowSpec.getUri());
// List Specs after adding
specs = flowCatalog.getSpecs();
logger.info("[After Delete] Number of specs: " + specs.size());
i = 0;
for (Spec spec : specs) {
flowSpec = (FlowSpec) spec;
logger.info("[After Delete] Spec " + i++ + ": " + gson.toJson(flowSpec));
}
// Make sure FlowCatalog has the Flow removed
Assert.assertTrue(specs.size() == 0, "Spec store should not contain Spec after deletion");
// Make sure FlowCatalog Listener knows about the deletion
specsInSEI = ((List) (sei.getProducer().get().listSpecs().get())).size();
Assert.assertTrue(specsInSEI == 0, "SpecProducer should not contain " + "Spec after deletion");
}
use of org.apache.gobblin.runtime.api.Spec in project incubator-gobblin by apache.
the class OrchestratorTest method createFlowSpec.
@Test(dependsOnMethods = "createTopologySpec")
public void createFlowSpec() throws Exception {
// Since only 1 Topology with 1 SpecProducer has been added in previous test
// .. it should be available and responsible for our new FlowSpec
IdentityFlowToJobSpecCompiler specCompiler = (IdentityFlowToJobSpecCompiler) this.orchestrator.getSpecCompiler();
SpecExecutor sei = specCompiler.getTopologySpecMap().values().iterator().next().getSpecExecutor();
// List Current Specs
Collection<Spec> specs = flowCatalog.getSpecs();
logger.info("[Before Create] Number of specs: " + specs.size());
int i = 0;
for (Spec spec : specs) {
FlowSpec flowSpec = (FlowSpec) spec;
logger.info("[Before Create] Spec " + i++ + ": " + gson.toJson(flowSpec));
}
// Make sure FlowCatalog is empty
Assert.assertTrue(specs.size() == 0, "Spec store should be empty before addition");
// Make sure FlowCatalog Listener is empty
Assert.assertTrue(((List) (sei.getProducer().get().listSpecs().get())).size() == 0, "SpecProducer should not know about " + "any Flow before addition");
// Create and add Spec
this.flowCatalog.put(flowSpec);
// List Specs after adding
specs = flowCatalog.getSpecs();
logger.info("[After Create] Number of specs: " + specs.size());
i = 0;
for (Spec spec : specs) {
flowSpec = (FlowSpec) spec;
logger.info("[After Create] Spec " + i++ + ": " + gson.toJson(flowSpec));
}
// Make sure FlowCatalog has the added Flow
Assert.assertTrue(specs.size() == 1, "Spec store should contain 1 Spec after addition");
// Orchestrator is a no-op listener for any new FlowSpecs
Assert.assertTrue(((List) (sei.getProducer().get().listSpecs().get())).size() == 0, "SpecProducer should contain 0 " + "Spec after addition");
}
Aggregations