Examples with JobSpec org.apache.gobblin.runtime.api.JobSpec used on opensource projects

Example 6 with JobSpec

use of org.apache.gobblin.runtime.api.JobSpec in project incubator-gobblin by apache.

the class StreamingKafkaSpecExecutorTest method testDeleteSpec.

@Test(dependsOnMethods = "testUpdateSpec")
public void testDeleteSpec() throws Exception {
    // delete needs to be on a job spec that exists to get notification
    String deletedSpecUriString = "/foo/bar/addedSpec";
    WriteResponse writeResponse = (WriteResponse) _seip.deleteSpec(new URI(deletedSpecUriString)).get();
    log.info("WriteResponse: " + writeResponse);
    List<Pair<SpecExecutor.Verb, Spec>> consumedEvent = _seic.changedSpecs().get();
    Assert.assertTrue(consumedEvent.size() == 1, "Consumption did not match production");
    Map.Entry<SpecExecutor.Verb, Spec> consumedSpecAction = consumedEvent.get(0);
    Assert.assertTrue(consumedSpecAction.getKey().equals(SpecExecutor.Verb.DELETE), "Verb did not match");
    Assert.assertTrue(consumedSpecAction.getValue().getUri().toString().equals(deletedSpecUriString), "Expected URI did not match");
    Assert.assertTrue(consumedSpecAction.getValue() instanceof JobSpec, "Expected JobSpec");
}

Example 7 with JobSpec

use of org.apache.gobblin.runtime.api.JobSpec in project incubator-gobblin by apache.

the class SimpleKafkaSpecExecutorTest method testDeleteSpec.

@Test(dependsOnMethods = "testUpdateSpec")
public void testDeleteSpec() throws Exception {
    String deletedSpecUriString = "/foo/bar/deletedSpec";
    WriteResponse writeResponse = (WriteResponse) _seip.deleteSpec(new URI(deletedSpecUriString)).get();
    log.info("WriteResponse: " + writeResponse);
    try {
        Thread.sleep(1000);
    } catch (InterruptedException ex) {
        Thread.currentThread().interrupt();
    }
    List<Pair<SpecExecutor.Verb, Spec>> consumedEvent = _seic.changedSpecs().get();
    Assert.assertTrue(consumedEvent.size() == 1, "Consumption did not match production");
    Map.Entry<SpecExecutor.Verb, Spec> consumedSpecAction = consumedEvent.get(0);
    Assert.assertTrue(consumedSpecAction.getKey().equals(SpecExecutor.Verb.DELETE), "Verb did not match");
    Assert.assertTrue(consumedSpecAction.getValue().getUri().toString().equals(deletedSpecUriString), "Expected URI did not match");
    Assert.assertTrue(consumedSpecAction.getValue() instanceof JobSpec, "Expected JobSpec");
}

Example 8 with JobSpec

use of org.apache.gobblin.runtime.api.JobSpec in project incubator-gobblin by apache.

the class SimpleKafkaSpecExecutorTest method testUpdateSpec.

@Test(dependsOnMethods = "testAddSpec")
public void testUpdateSpec() throws Exception {
    String updatedSpecUriString = "/foo/bar/updatedSpec";
    Spec spec = initJobSpec(updatedSpecUriString);
    WriteResponse writeResponse = (WriteResponse) _seip.updateSpec(spec).get();
    log.info("WriteResponse: " + writeResponse);
    try {
        Thread.sleep(1000);
    } catch (InterruptedException ex) {
        Thread.currentThread().interrupt();
    }
    List<Pair<SpecExecutor.Verb, Spec>> consumedEvent = _seic.changedSpecs().get();
    Assert.assertTrue(consumedEvent.size() == 1, "Consumption did not match production");
    Map.Entry<SpecExecutor.Verb, Spec> consumedSpecAction = consumedEvent.get(0);
    Assert.assertTrue(consumedSpecAction.getKey().equals(SpecExecutor.Verb.UPDATE), "Verb did not match");
    Assert.assertTrue(consumedSpecAction.getValue().getUri().toString().equals(updatedSpecUriString), "Expected URI did not match");
    Assert.assertTrue(consumedSpecAction.getValue() instanceof JobSpec, "Expected JobSpec");
}

Example 9 with JobSpec

use of org.apache.gobblin.runtime.api.JobSpec in project incubator-gobblin by apache.

the class MutableJobCatalogBase method put.

@Override
public void put(JobSpec jobSpec) {
    Preconditions.checkState(allowMutationsBeforeStartup() || state() == State.RUNNING, String.format("%s is not running.", this.getClass().getName()));
    Preconditions.checkNotNull(jobSpec);
    JobSpec oldSpec = doPut(jobSpec);
    if (null == oldSpec) {
        this.listeners.onAddJob(jobSpec);
    } else {
        this.listeners.onUpdateJob(jobSpec);
    }
}

Example 10 with JobSpec

use of org.apache.gobblin.runtime.api.JobSpec in project incubator-gobblin by apache.

the class NonObservingFSJobCatalog method remove.

/**
 * Allow user to programmatically delete a new JobSpec.
 * This method is designed to be reentrant.
 * @param jobURI The relative Path that specified by user, need to make it into complete path.
 */
@Override
public synchronized void remove(URI jobURI) {
    Preconditions.checkState(state() == State.RUNNING, String.format("%s is not running.", this.getClass().getName()));
    try {
        long startTime = System.currentTimeMillis();
        JobSpec jobSpec = getJobSpec(jobURI);
        Path jobSpecPath = getPathForURI(this.jobConfDirPath, jobURI);
        if (fs.exists(jobSpecPath)) {
            fs.delete(jobSpecPath, false);
            this.mutableMetrics.updateRemoveJobTime(startTime);
        } else {
            LOGGER.warn("No file with URI:" + jobSpecPath + " is found. Deletion failed.");
        }
        this.listeners.onDeleteJob(jobURI, jobSpec.getVersion());
    } catch (IOException e) {
        throw new RuntimeException("When removing a JobConf. file, issues unexpected happen:" + e.getMessage());
    } catch (SpecNotFoundException e) {
        LOGGER.warn("No file with URI:" + jobURI + " is found. Deletion failed.");
    }
}