Examples with AddSpecResponse org.apache.gobblin.runtime.spec_catalog.AddSpecResponse used on opensource projects

Example 1 with AddSpecResponse

use of org.apache.gobblin.runtime.spec_catalog.AddSpecResponse in project incubator-gobblin by apache.

the class GobblinServiceJobSchedulerTest method testJobSchedulerUnschedule.

/**
 * Test that flowSpecs that throw compilation errors do not block the scheduling of other flowSpecs
 */
@Test
public void testJobSchedulerUnschedule() throws Exception {
    // Mock a FlowCatalog.
    File specDir = Files.createTempDir();
    Properties properties = new Properties();
    properties.setProperty(FLOWSPEC_STORE_DIR_KEY, specDir.getAbsolutePath());
    FlowCatalog flowCatalog = new FlowCatalog(ConfigUtils.propertiesToConfig(properties));
    ServiceBasedAppLauncher serviceLauncher = new ServiceBasedAppLauncher(properties, "GaaSJobSchedulerTest");
    // Assume that the catalog can store corrupted flows
    SpecCatalogListener mockListener = Mockito.mock(SpecCatalogListener.class);
    when(mockListener.getName()).thenReturn(ServiceConfigKeys.GOBBLIN_SERVICE_JOB_SCHEDULER_LISTENER_CLASS);
    when(mockListener.onAddSpec(any())).thenReturn(new AddSpecResponse(""));
    flowCatalog.addListener(mockListener);
    serviceLauncher.addService(flowCatalog);
    serviceLauncher.start();
    FlowSpec flowSpec0 = FlowCatalogTest.initFlowSpec(specDir.getAbsolutePath(), URI.create("spec0"));
    FlowSpec flowSpec1 = FlowCatalogTest.initFlowSpec(specDir.getAbsolutePath(), URI.create("spec1"));
    FlowSpec flowSpec2 = FlowCatalogTest.initFlowSpec(specDir.getAbsolutePath(), URI.create("spec2"));
    // Ensure that these flows are scheduled
    flowCatalog.put(flowSpec0, true);
    flowCatalog.put(flowSpec1, true);
    flowCatalog.put(flowSpec2, true);
    Assert.assertEquals(flowCatalog.getSpecs().size(), 3);
    Orchestrator mockOrchestrator = Mockito.mock(Orchestrator.class);
    SchedulerService schedulerService = new SchedulerService(new Properties());
    // Mock a GaaS scheduler.
    TestGobblinServiceJobScheduler scheduler = new TestGobblinServiceJobScheduler("testscheduler", ConfigFactory.empty(), Optional.of(flowCatalog), null, mockOrchestrator, schedulerService);
    schedulerService.startAsync().awaitRunning();
    scheduler.startUp();
    SpecCompiler mockCompiler = Mockito.mock(SpecCompiler.class);
    Mockito.when(mockOrchestrator.getSpecCompiler()).thenReturn(mockCompiler);
    Mockito.doAnswer((Answer<Void>) a -> {
        scheduler.isCompilerHealthy = true;
        return null;
    }).when(mockCompiler).awaitHealthy();
    scheduler.setActive(true);
    AssertWithBackoff.create().timeoutMs(20000).maxSleepMs(2000).backoffFactor(2).assertTrue(new Predicate<Void>() {

        @Override
        public boolean apply(Void input) {
            Map<String, Spec> scheduledFlowSpecs = scheduler.scheduledFlowSpecs;
            if (scheduledFlowSpecs != null && scheduledFlowSpecs.size() == 3) {
                return scheduler.scheduledFlowSpecs.containsKey("spec0") && scheduler.scheduledFlowSpecs.containsKey("spec1") && scheduler.scheduledFlowSpecs.containsKey("spec2");
            } else {
                return false;
            }
        }
    }, "Waiting all flowSpecs to be scheduled");
    // set scheduler to be inactive and unschedule flows
    scheduler.setActive(false);
    Collection<Invocation> invocations = Mockito.mockingDetails(mockOrchestrator).getInvocations();
    for (Invocation invocation : invocations) {
        // ensure that orchestrator is not calling remove
        Assert.assertFalse(invocation.getMethod().getName().equals("remove"));
    }
    Assert.assertEquals(scheduler.scheduledFlowSpecs.size(), 0);
    Assert.assertEquals(schedulerService.getScheduler().getJobGroupNames().size(), 0);
}

Example 2 with AddSpecResponse

use of org.apache.gobblin.runtime.spec_catalog.AddSpecResponse in project incubator-gobblin by apache.

the class GobblinServiceJobSchedulerTest method testJobSchedulerInitWithFailedSpec.

/**
 * Test that flowSpecs that throw compilation errors do not block the scheduling of other flowSpecs
 */
@Test
public void testJobSchedulerInitWithFailedSpec() throws Exception {
    // Mock a FlowCatalog.
    File specDir = Files.createTempDir();
    Properties properties = new Properties();
    properties.setProperty(FLOWSPEC_STORE_DIR_KEY, specDir.getAbsolutePath());
    FlowCatalog flowCatalog = new FlowCatalog(ConfigUtils.propertiesToConfig(properties));
    ServiceBasedAppLauncher serviceLauncher = new ServiceBasedAppLauncher(properties, "GaaSJobSchedulerTest");
    // Assume that the catalog can store corrupted flows
    SpecCatalogListener mockListener = Mockito.mock(SpecCatalogListener.class);
    when(mockListener.getName()).thenReturn(ServiceConfigKeys.GOBBLIN_SERVICE_JOB_SCHEDULER_LISTENER_CLASS);
    when(mockListener.onAddSpec(any())).thenReturn(new AddSpecResponse(""));
    flowCatalog.addListener(mockListener);
    serviceLauncher.addService(flowCatalog);
    serviceLauncher.start();
    FlowSpec flowSpec0 = FlowCatalogTest.initFlowSpec(specDir.getAbsolutePath(), URI.create("spec0"), MockedSpecCompiler.UNCOMPILABLE_FLOW);
    FlowSpec flowSpec1 = FlowCatalogTest.initFlowSpec(specDir.getAbsolutePath(), URI.create("spec1"));
    FlowSpec flowSpec2 = FlowCatalogTest.initFlowSpec(specDir.getAbsolutePath(), URI.create("spec2"));
    // Ensure that these flows are scheduled
    flowCatalog.put(flowSpec0, true);
    flowCatalog.put(flowSpec1, true);
    flowCatalog.put(flowSpec2, true);
    Assert.assertEquals(flowCatalog.getSpecs().size(), 3);
    Orchestrator mockOrchestrator = Mockito.mock(Orchestrator.class);
    // Mock a GaaS scheduler.
    TestGobblinServiceJobScheduler scheduler = new TestGobblinServiceJobScheduler("testscheduler", ConfigFactory.empty(), Optional.of(flowCatalog), null, mockOrchestrator, null);
    SpecCompiler mockCompiler = Mockito.mock(SpecCompiler.class);
    Mockito.when(mockOrchestrator.getSpecCompiler()).thenReturn(mockCompiler);
    Mockito.doAnswer((Answer<Void>) a -> {
        scheduler.isCompilerHealthy = true;
        return null;
    }).when(mockCompiler).awaitHealthy();
    scheduler.setActive(true);
    AssertWithBackoff.create().timeoutMs(20000).maxSleepMs(2000).backoffFactor(2).assertTrue(new Predicate<Void>() {

        @Override
        public boolean apply(Void input) {
            Map<String, Spec> scheduledFlowSpecs = scheduler.scheduledFlowSpecs;
            if (scheduledFlowSpecs != null && scheduledFlowSpecs.size() == 2) {
                return scheduler.scheduledFlowSpecs.containsKey("spec1") && scheduler.scheduledFlowSpecs.containsKey("spec2");
            } else {
                return false;
            }
        }
    }, "Waiting all flowSpecs to be scheduled");
}

Example 3 with AddSpecResponse

use of org.apache.gobblin.runtime.spec_catalog.AddSpecResponse in project incubator-gobblin by apache.

the class GitConfigMonitorTest method setup.

@BeforeClass
public void setup() throws Exception {
    cleanUpDir(TEST_DIR);
    // Create a bare repository
    RepositoryCache.FileKey fileKey = RepositoryCache.FileKey.exact(remoteDir, FS.DETECTED);
    this.remoteRepo = fileKey.open(false);
    this.remoteRepo.create(true);
    this.gitForPush = Git.cloneRepository().setURI(this.remoteRepo.getDirectory().getAbsolutePath()).setDirectory(cloneDir).call();
    // push an empty commit as a base for detecting changes
    this.gitForPush.commit().setMessage("First commit").call();
    this.gitForPush.push().setRemote("origin").setRefSpecs(this.masterRefSpec).call();
    this.config = ConfigBuilder.create().addPrimitive(GitConfigMonitor.GIT_CONFIG_MONITOR_PREFIX + "." + ConfigurationKeys.GIT_MONITOR_REPO_URI, this.remoteRepo.getDirectory().getAbsolutePath()).addPrimitive(GitConfigMonitor.GIT_CONFIG_MONITOR_PREFIX + "." + ConfigurationKeys.GIT_MONITOR_REPO_DIR, TEST_DIR + "/jobConfig").addPrimitive(FlowCatalog.FLOWSPEC_STORE_DIR_KEY, TEST_DIR + "flowCatalog").addPrimitive(ConfigurationKeys.GIT_MONITOR_POLLING_INTERVAL, 5).build();
    this.flowCatalog = new FlowCatalog(config);
    this.mockListener = mock(SpecCatalogListener.class);
    when(mockListener.getName()).thenReturn(ServiceConfigKeys.GOBBLIN_SERVICE_JOB_SCHEDULER_LISTENER_CLASS);
    when(mockListener.onAddSpec(any())).thenReturn(new AddSpecResponse(""));
    this.flowCatalog.addListener(mockListener);
    this.flowCatalog.startAsync().awaitRunning();
    this.gitConfigMonitor = new GitConfigMonitor(this.config, this.flowCatalog);
    this.gitConfigMonitor.setActive(true);
}

Example 4 with AddSpecResponse

use of org.apache.gobblin.runtime.spec_catalog.AddSpecResponse in project incubator-gobblin by apache.

the class OrchestratorTest method setup.

@BeforeClass
public void setup() throws Exception {
    cleanUpDir(TOPOLOGY_SPEC_STORE_DIR);
    cleanUpDir(FLOW_SPEC_STORE_DIR);
    Properties orchestratorProperties = new Properties();
    Properties topologyProperties = new Properties();
    topologyProperties.put("specStore.fs.dir", TOPOLOGY_SPEC_STORE_DIR);
    Properties flowProperties = new Properties();
    flowProperties.put("specStore.fs.dir", FLOW_SPEC_STORE_DIR);
    this.serviceLauncher = new ServiceBasedAppLauncher(orchestratorProperties, "OrchestratorCatalogTest");
    this.topologyCatalog = new TopologyCatalog(ConfigUtils.propertiesToConfig(topologyProperties), Optional.of(logger));
    this.serviceLauncher.addService(topologyCatalog);
    this.flowCatalog = new FlowCatalog(ConfigUtils.propertiesToConfig(flowProperties), Optional.of(logger));
    this.mockListener = mock(SpecCatalogListener.class);
    when(mockListener.getName()).thenReturn(ServiceConfigKeys.GOBBLIN_SERVICE_JOB_SCHEDULER_LISTENER_CLASS);
    when(mockListener.onAddSpec(any())).thenReturn(new AddSpecResponse(""));
    this.flowCatalog.addListener(mockListener);
    this.serviceLauncher.addService(flowCatalog);
    this.orchestrator = new Orchestrator(ConfigUtils.propertiesToConfig(orchestratorProperties), mock(FlowStatusGenerator.class), Optional.of(this.topologyCatalog), Optional.<DagManager>absent(), Optional.of(logger));
    this.topologyCatalog.addListener(orchestrator);
    this.flowCatalog.addListener(orchestrator);
    // Start application
    this.serviceLauncher.start();
    // Create Spec to play with
    this.topologySpec = initTopologySpec();
    this.flowSpec = initFlowSpec();
}

Example 5 with AddSpecResponse

use of org.apache.gobblin.runtime.spec_catalog.AddSpecResponse in project incubator-gobblin by apache.

the class BaseFlowToJobSpecCompiler method onAddSpec.

@Override
public synchronized AddSpecResponse onAddSpec(Spec addedSpec) {
    TopologySpec spec = (TopologySpec) addedSpec;
    log.info("Loading topology {}", spec.toLongString());
    for (Map.Entry entry : spec.getConfigAsProperties().entrySet()) {
        log.info("topo: {} --> {}", entry.getKey(), entry.getValue());
    }
    topologySpecMap.put(addedSpec.getUri(), (TopologySpec) addedSpec);
    return new AddSpecResponse(null);
}