Examples with AddSpecResponse org.apache.gobblin.runtime.spec_catalog.AddSpecResponse used on opensource projects

Example 6 with AddSpecResponse

use of org.apache.gobblin.runtime.spec_catalog.AddSpecResponse in project incubator-gobblin by apache.

the class OrchestratorTest method setup.

@BeforeClass
public void setup() throws Exception {
    cleanUpDir(TOPOLOGY_SPEC_STORE_DIR);
    cleanUpDir(FLOW_SPEC_STORE_DIR);
    Properties orchestratorProperties = new Properties();
    Properties topologyProperties = new Properties();
    topologyProperties.put("specStore.fs.dir", TOPOLOGY_SPEC_STORE_DIR);
    Properties flowProperties = new Properties();
    flowProperties.put("specStore.fs.dir", FLOW_SPEC_STORE_DIR);
    this.serviceLauncher = new ServiceBasedAppLauncher(orchestratorProperties, "OrchestratorCatalogTest");
    this.topologyCatalog = new TopologyCatalog(ConfigUtils.propertiesToConfig(topologyProperties), Optional.of(logger));
    this.serviceLauncher.addService(topologyCatalog);
    this.flowCatalog = new FlowCatalog(ConfigUtils.propertiesToConfig(flowProperties), Optional.of(logger));
    this.mockListener = mock(SpecCatalogListener.class);
    when(mockListener.getName()).thenReturn(ServiceConfigKeys.GOBBLIN_SERVICE_JOB_SCHEDULER_LISTENER_CLASS);
    when(mockListener.onAddSpec(any())).thenReturn(new AddSpecResponse(""));
    this.flowCatalog.addListener(mockListener);
    this.serviceLauncher.addService(flowCatalog);
    this.orchestrator = new Orchestrator(ConfigUtils.propertiesToConfig(orchestratorProperties), mock(FlowStatusGenerator.class), Optional.of(this.topologyCatalog), Optional.<DagManager>absent(), Optional.of(logger));
    this.topologyCatalog.addListener(orchestrator);
    this.flowCatalog.addListener(orchestrator);
    // Start application
    this.serviceLauncher.start();
    // Create Spec to play with
    this.topologySpec = initTopologySpec();
    this.flowSpec = initFlowSpec();
}

Example 7 with AddSpecResponse

use of org.apache.gobblin.runtime.spec_catalog.AddSpecResponse in project incubator-gobblin by apache.

the class BaseFlowToJobSpecCompiler method onAddSpec.

@Override
public synchronized AddSpecResponse onAddSpec(Spec addedSpec) {
    TopologySpec spec = (TopologySpec) addedSpec;
    log.info("Loading topology {}", spec.toLongString());
    for (Map.Entry entry : spec.getConfigAsProperties().entrySet()) {
        log.info("topo: {} --> {}", entry.getKey(), entry.getValue());
    }
    topologySpecMap.put(addedSpec.getUri(), (TopologySpec) addedSpec);
    return new AddSpecResponse(null);
}

Example 8 with AddSpecResponse

use of org.apache.gobblin.runtime.spec_catalog.AddSpecResponse in project incubator-gobblin by apache.

the class GobblinServiceJobSchedulerTest method testJobSchedulerInit.

/**
 * Test whenever JobScheduler is calling setActive, the FlowSpec is loading into scheduledFlowSpecs (eventually)
 */
@Test
public void testJobSchedulerInit() throws Exception {
    // Mock a FlowCatalog.
    File specDir = Files.createTempDir();
    Properties properties = new Properties();
    properties.setProperty(FLOWSPEC_STORE_DIR_KEY, specDir.getAbsolutePath());
    FlowCatalog flowCatalog = new FlowCatalog(ConfigUtils.propertiesToConfig(properties));
    SpecCatalogListener mockListener = Mockito.mock(SpecCatalogListener.class);
    when(mockListener.getName()).thenReturn(ServiceConfigKeys.GOBBLIN_SERVICE_JOB_SCHEDULER_LISTENER_CLASS);
    when(mockListener.onAddSpec(any())).thenReturn(new AddSpecResponse(""));
    flowCatalog.addListener(mockListener);
    ServiceBasedAppLauncher serviceLauncher = new ServiceBasedAppLauncher(properties, "GaaSJobSchedulerTest");
    serviceLauncher.addService(flowCatalog);
    serviceLauncher.start();
    FlowSpec flowSpec0 = FlowCatalogTest.initFlowSpec(specDir.getAbsolutePath(), URI.create("spec0"));
    FlowSpec flowSpec1 = FlowCatalogTest.initFlowSpec(specDir.getAbsolutePath(), URI.create("spec1"));
    flowCatalog.put(flowSpec0, true);
    flowCatalog.put(flowSpec1, true);
    Assert.assertEquals(flowCatalog.getSpecs().size(), 2);
    Orchestrator mockOrchestrator = Mockito.mock(Orchestrator.class);
    // Mock a GaaS scheduler.
    TestGobblinServiceJobScheduler scheduler = new TestGobblinServiceJobScheduler("testscheduler", ConfigFactory.empty(), Optional.of(flowCatalog), null, mockOrchestrator, null);
    SpecCompiler mockCompiler = Mockito.mock(SpecCompiler.class);
    Mockito.when(mockOrchestrator.getSpecCompiler()).thenReturn(mockCompiler);
    Mockito.doAnswer((Answer<Void>) a -> {
        scheduler.isCompilerHealthy = true;
        return null;
    }).when(mockCompiler).awaitHealthy();
    scheduler.setActive(true);
    AssertWithBackoff.create().timeoutMs(20000).maxSleepMs(2000).backoffFactor(2).assertTrue(new Predicate<Void>() {

        @Override
        public boolean apply(Void input) {
            Map<String, Spec> scheduledFlowSpecs = scheduler.scheduledFlowSpecs;
            if (scheduledFlowSpecs != null && scheduledFlowSpecs.size() == 2) {
                return scheduler.scheduledFlowSpecs.containsKey("spec0") && scheduler.scheduledFlowSpecs.containsKey("spec1");
            } else {
                return false;
            }
        }
    }, "Waiting all flowSpecs to be scheduled");
}

Example 9 with AddSpecResponse

use of org.apache.gobblin.runtime.spec_catalog.AddSpecResponse in project incubator-gobblin by apache.

the class GobblinServiceJobScheduler method onAddSpec.

/**
 * @param addedSpec spec to be added
 * @return add spec response, which contains <code>null</code> if there is an error
 */
@Override
public AddSpecResponse onAddSpec(Spec addedSpec) {
    if (this.helixManager.isPresent() && !this.helixManager.get().isConnected()) {
        // Specs in store will be notified when Scheduler is added as listener to FlowCatalog, so ignore
        // .. Specs if in cluster mode and Helix is not yet initialized
        _log.info("System not yet initialized. Skipping Spec Addition: " + addedSpec);
        return null;
    }
    _log.info("New Flow Spec detected: " + addedSpec);
    if (!(addedSpec instanceof FlowSpec)) {
        return null;
    }
    FlowSpec flowSpec = (FlowSpec) addedSpec;
    URI flowSpecUri = flowSpec.getUri();
    Properties jobConfig = createJobConfig(flowSpec);
    boolean isExplain = flowSpec.isExplain();
    String response = null;
    // always try to compile the flow to verify if it is compilable
    Dag<JobExecutionPlan> dag = this.orchestrator.getSpecCompiler().compileFlow(flowSpec);
    // If dag is null then a compilation error has occurred
    if (dag != null && !dag.isEmpty()) {
        response = dag.toString();
    }
    boolean compileSuccess = FlowCatalog.isCompileSuccessful(response);
    if (isExplain || !compileSuccess || !this.isActive) {
        // todo: in case of a scheduled job, we should also check if the job schedule is a valid cron schedule
        // so it can be scheduled
        _log.info("Ignoring the spec {}. isExplain: {}, compileSuccess: {}, master: {}", addedSpec, isExplain, compileSuccess, this.isActive);
        return new AddSpecResponse<>(response);
    }
    // todo : we should probably not schedule a flow if it is a runOnce flow
    this.scheduledFlowSpecs.put(flowSpecUri.toString(), addedSpec);
    if (jobConfig.containsKey(ConfigurationKeys.JOB_SCHEDULE_KEY)) {
        _log.info("{} Scheduling flow spec: {} ", this.serviceName, addedSpec);
        try {
            scheduleJob(jobConfig, null);
        } catch (JobException je) {
            _log.error("{} Failed to schedule or run FlowSpec {}", serviceName, addedSpec, je);
            this.scheduledFlowSpecs.remove(addedSpec.getUri().toString());
            return null;
        }
        if (PropertiesUtils.getPropAsBoolean(jobConfig, ConfigurationKeys.FLOW_RUN_IMMEDIATELY, "false")) {
            _log.info("RunImmediately requested, hence executing FlowSpec: " + addedSpec);
            this.jobExecutor.execute(new NonScheduledJobRunner(flowSpecUri, false, jobConfig, null));
        }
    } else {
        _log.info("No FlowSpec schedule found, so running FlowSpec: " + addedSpec);
        this.jobExecutor.execute(new NonScheduledJobRunner(flowSpecUri, true, jobConfig, null));
    }
    return new AddSpecResponse<>(response);
}

Example 10 with AddSpecResponse

use of org.apache.gobblin.runtime.spec_catalog.AddSpecResponse in project incubator-gobblin by apache.

the class FlowConfigTest method setUp.

@BeforeClass
public void setUp() throws Exception {
    ConfigBuilder configBuilder = ConfigBuilder.create();
    _testDirectory = Files.createTempDir();
    configBuilder.addPrimitive(ConfigurationKeys.JOB_CONFIG_FILE_DIR_KEY, _testDirectory.getAbsolutePath()).addPrimitive(FSSpecStore.SPECSTORE_FS_DIR_KEY, TEST_SPEC_STORE_DIR);
    cleanUpDir(TEST_SPEC_STORE_DIR);
    Config config = configBuilder.build();
    final FlowCatalog flowCatalog = new FlowCatalog(config);
    final SpecCatalogListener mockListener = mock(SpecCatalogListener.class);
    when(mockListener.getName()).thenReturn(ServiceConfigKeys.GOBBLIN_SERVICE_JOB_SCHEDULER_LISTENER_CLASS);
    when(mockListener.onAddSpec(any())).thenReturn(new AddSpecResponse(""));
    flowCatalog.addListener(mockListener);
    flowCatalog.startAsync();
    flowCatalog.awaitRunning();
    Injector injector = Guice.createInjector(new Module() {

        @Override
        public void configure(Binder binder) {
            binder.bind(FlowConfigsResourceHandler.class).toInstance(new FlowConfigResourceLocalHandler(flowCatalog));
            // indicate that we are in unit testing since the resource is being blocked until flow catalog changes have
            // been made
            binder.bindConstant().annotatedWith(Names.named(FlowConfigsResource.INJECT_READY_TO_USE)).to(Boolean.TRUE);
            binder.bind(RequesterService.class).toInstance(new NoopRequesterService(config));
        }
    });
    _server = EmbeddedRestliServer.builder().resources(Lists.<Class<? extends BaseResource>>newArrayList(FlowConfigsResource.class)).injector(injector).build();
    _server.startAsync();
    _server.awaitRunning();
    Map<String, String> transportClientProperties = Maps.newHashMap();
    transportClientProperties.put(HttpClientFactory.HTTP_REQUEST_TIMEOUT, "10000");
    _client = new FlowConfigClient(String.format("http://localhost:%s/", _server.getPort()), transportClientProperties);
}