Examples with JobSpec org.apache.gobblin.runtime.api.JobSpec used on opensource projects

Example 46 with JobSpec

use of org.apache.gobblin.runtime.api.JobSpec in project incubator-gobblin by apache.

the class ScheduledJobConfigurationManager method fetchJobSpecs.

/**
 * TODO: Change cluster code to handle Spec. Right now all job properties are needed to be in config and template is not honored
 * TODO: Materialized JobSpec and make use of ResolvedJobSpec
 * @throws ExecutionException
 * @throws InterruptedException
 */
private void fetchJobSpecs() throws ExecutionException, InterruptedException {
    List<Pair<SpecExecutor.Verb, Spec>> changesSpecs = (List<Pair<SpecExecutor.Verb, Spec>>) this._specConsumer.changedSpecs().get();
    for (Pair<SpecExecutor.Verb, Spec> entry : changesSpecs) {
        SpecExecutor.Verb verb = entry.getKey();
        if (verb.equals(SpecExecutor.Verb.ADD)) {
            // Handle addition
            JobSpec jobSpec = (JobSpec) entry.getValue();
            postNewJobConfigArrival(jobSpec.getUri().toString(), jobSpec.getConfigAsProperties());
            jobSpecs.put(entry.getValue().getUri(), (JobSpec) entry.getValue());
        } else if (verb.equals(SpecExecutor.Verb.UPDATE)) {
            // Handle update
            JobSpec jobSpec = (JobSpec) entry.getValue();
            postUpdateJobConfigArrival(jobSpec.getUri().toString(), jobSpec.getConfigAsProperties());
            jobSpecs.put(entry.getValue().getUri(), (JobSpec) entry.getValue());
        } else if (verb.equals(SpecExecutor.Verb.DELETE)) {
            // Handle delete
            Spec anonymousSpec = (Spec) entry.getValue();
            postDeleteJobConfigArrival(anonymousSpec.getUri().toString(), new Properties());
            jobSpecs.remove(entry.getValue().getUri());
        }
    }
}

Example 47 with JobSpec

use of org.apache.gobblin.runtime.api.JobSpec in project incubator-gobblin by apache.

the class StreamingKafkaSpecExecutorTest method testUpdateSpec.

@Test(dependsOnMethods = "testAddSpec")
public void testUpdateSpec() throws Exception {
    // update is only treated as an update for existing job specs
    String updatedSpecUriString = "/foo/bar/addedSpec";
    Spec spec = initJobSpec(updatedSpecUriString);
    WriteResponse writeResponse = (WriteResponse) _seip.updateSpec(spec).get();
    log.info("WriteResponse: " + writeResponse);
    List<Pair<SpecExecutor.Verb, Spec>> consumedEvent = _seic.changedSpecs().get();
    Assert.assertTrue(consumedEvent.size() == 1, "Consumption did not match production");
    Map.Entry<SpecExecutor.Verb, Spec> consumedSpecAction = consumedEvent.get(0);
    Assert.assertTrue(consumedSpecAction.getKey().equals(SpecExecutor.Verb.UPDATE), "Verb did not match");
    Assert.assertTrue(consumedSpecAction.getValue().getUri().toString().equals(updatedSpecUriString), "Expected URI did not match");
    Assert.assertTrue(consumedSpecAction.getValue() instanceof JobSpec, "Expected JobSpec");
}

Example 48 with JobSpec

use of org.apache.gobblin.runtime.api.JobSpec in project incubator-gobblin by apache.

the class SimpleKafkaSpecExecutorTest method testAddSpec.

@Test
public void testAddSpec() throws Exception {
    _closer = Closer.create();
    _properties = new Properties();
    // Properties for Producer
    _properties.setProperty(KafkaWriterConfigurationKeys.KAFKA_TOPIC, TOPIC);
    _properties.setProperty(KafkaWriterConfigurationKeys.KAFKA_PRODUCER_CONFIG_PREFIX + "bootstrap.servers", _kafkaBrokers);
    _properties.setProperty(KafkaWriterConfigurationKeys.KAFKA_PRODUCER_CONFIG_PREFIX + "value.serializer", "org.apache.kafka.common.serialization.ByteArraySerializer");
    // Properties for Consumer
    _properties.setProperty(ConfigurationKeys.KAFKA_BROKERS, _kafkaBrokers);
    _properties.setProperty(SimpleKafkaSpecExecutor.SPEC_KAFKA_TOPICS_KEY, TOPIC);
    // SEI Producer
    _seip = _closer.register(new SimpleKafkaSpecProducer(ConfigUtils.propertiesToConfig(_properties)));
    String addedSpecUriString = "/foo/bar/addedSpec";
    Spec spec = initJobSpec(addedSpecUriString);
    WriteResponse writeResponse = (WriteResponse) _seip.addSpec(spec).get();
    log.info("WriteResponse: " + writeResponse);
    try {
        Thread.sleep(1000);
    } catch (InterruptedException ex) {
        Thread.currentThread().interrupt();
    }
    _seic = _closer.register(new SimpleKafkaSpecConsumer(ConfigUtils.propertiesToConfig(_properties)));
    List<Pair<SpecExecutor.Verb, Spec>> consumedEvent = _seic.changedSpecs().get();
    Assert.assertTrue(consumedEvent.size() == 1, "Consumption did not match production");
    Map.Entry<SpecExecutor.Verb, Spec> consumedSpecAction = consumedEvent.get(0);
    Assert.assertTrue(consumedSpecAction.getKey().equals(SpecExecutor.Verb.ADD), "Verb did not match");
    Assert.assertTrue(consumedSpecAction.getValue().getUri().toString().equals(addedSpecUriString), "Expected URI did not match");
    Assert.assertTrue(consumedSpecAction.getValue() instanceof JobSpec, "Expected JobSpec");
}

Example 49 with JobSpec

use of org.apache.gobblin.runtime.api.JobSpec in project incubator-gobblin by apache.

the class AzkabanProjectConfigTest method testProjectNameWithReallyLongName.

@Test
public void testProjectNameWithReallyLongName() throws Exception {
    String expectedProjectName = "randomPrefixWithReallyLongName_http___localhost_8000__55490420";
    Properties properties = new Properties();
    properties.setProperty("gobblin.service.azkaban.project.namePrefix", "randomPrefixWithReallyLongName");
    JobSpec jobSpec = new JobSpec(new URI("http://localhost:8000/context/that-keeps-expanding-and-explanding"), "0.0", "test job spec", ConfigUtils.propertiesToConfig(properties), properties, Optional.absent());
    AzkabanProjectConfig azkabanProjectConfig = new AzkabanProjectConfig(jobSpec);
    String actualProjectName = azkabanProjectConfig.getAzkabanProjectName();
    Assert.assertEquals(actualProjectName, expectedProjectName);
}

Example 50 with JobSpec

use of org.apache.gobblin.runtime.api.JobSpec in project incubator-gobblin by apache.

the class AzkabanProjectConfigTest method testProjectNameWithConfig.

@Test
public void testProjectNameWithConfig() throws Exception {
    String expectedProjectName = "randomPrefix_http___localhost_8000_context";
    Properties properties = new Properties();
    properties.setProperty("gobblin.service.azkaban.project.namePrefix", "randomPrefix");
    JobSpec jobSpec = new JobSpec(new URI("http://localhost:8000/context"), "0.0", "test job spec", ConfigUtils.propertiesToConfig(properties), properties, Optional.absent());
    AzkabanProjectConfig azkabanProjectConfig = new AzkabanProjectConfig(jobSpec);
    String actualProjectName = azkabanProjectConfig.getAzkabanProjectName();
    Assert.assertEquals(actualProjectName, expectedProjectName);
}