Examples with JobTemplate org.apache.gobblin.runtime.api.JobTemplate used on opensource projects

Example 1 with JobTemplate

use of org.apache.gobblin.runtime.api.JobTemplate in project incubator-gobblin by apache.

the class InheritingJobTemplate method getRawTemplateConfigHelper.

private Config getRawTemplateConfigHelper(Set<JobTemplate> alreadyInheritedTemplates) throws SpecNotFoundException, TemplateException {
    Config rawTemplate = getLocalRawTemplate();
    for (JobTemplate template : Lists.reverse(this.superTemplates)) {
        if (!alreadyInheritedTemplates.contains(template)) {
            alreadyInheritedTemplates.add(template);
            Config thisFallback = template instanceof InheritingJobTemplate ? ((InheritingJobTemplate) template).getRawTemplateConfigHelper(alreadyInheritedTemplates) : template.getRawTemplateConfig();
            rawTemplate = rawTemplate.withFallback(thisFallback);
        }
    }
    return rawTemplate;
}

Example 2 with JobTemplate

use of org.apache.gobblin.runtime.api.JobTemplate in project incubator-gobblin by apache.

the class InheritingJobTemplate method getRequiredConfigListHelper.

private Set<String> getRequiredConfigListHelper(Set<JobTemplate> alreadyLoadedTemplates) throws SpecNotFoundException, TemplateException {
    Set<String> requiredConfigs = Sets.newHashSet(getLocallyRequiredConfigList());
    for (JobTemplate template : this.superTemplates) {
        if (!alreadyLoadedTemplates.contains(template)) {
            alreadyLoadedTemplates.add(template);
            requiredConfigs.addAll(template instanceof InheritingJobTemplate ? ((InheritingJobTemplate) template).getRequiredConfigListHelper(alreadyLoadedTemplates) : template.getRequiredConfigList());
        }
    }
    return requiredConfigs;
}

Example 3 with JobTemplate

use of org.apache.gobblin.runtime.api.JobTemplate in project incubator-gobblin by apache.

the class InheritingJobTemplateTest method testSimpleInheritance.

@Test
public void testSimpleInheritance() throws Exception {
    TestTemplate template1 = new TestTemplate(new URI("template1"), Lists.<JobTemplate>newArrayList(), ImmutableMap.of("key1", "value1"), Lists.newArrayList("required"));
    TestTemplate template2 = new TestTemplate(new URI("template2"), Lists.<JobTemplate>newArrayList(template1), ImmutableMap.of("key2", "value2"), Lists.newArrayList("required2"));
    Collection<String> required = template2.getRequiredConfigList();
    Assert.assertEquals(required.size(), 2);
    Assert.assertTrue(required.contains("required"));
    Assert.assertTrue(required.contains("required2"));
    Config rawTemplate = template2.getRawTemplateConfig();
    Assert.assertEquals(rawTemplate.getString("key1"), "value1");
    Assert.assertEquals(rawTemplate.getString("key2"), "value2");
    Config resolved = template2.getResolvedConfig(ConfigFactory.parseMap(ImmutableMap.of("required", "r1", "required2", "r2")));
    Assert.assertEquals(resolved.getString("key1"), "value1");
    Assert.assertEquals(resolved.getString("key2"), "value2");
    Assert.assertEquals(resolved.getString("required"), "r1");
    Assert.assertEquals(resolved.getString("required2"), "r2");
    try {
        // should throw exception because missing required property
        resolved = template2.getResolvedConfig(ConfigFactory.parseMap(ImmutableMap.of("required", "r1")));
        Assert.fail();
    } catch (JobTemplate.TemplateException te) {
    // expected
    }
}

Example 4 with JobTemplate

use of org.apache.gobblin.runtime.api.JobTemplate in project incubator-gobblin by apache.

the class PackagedTemplatesJobCatalogDecoratorTest method test.

@Test
public void test() throws Exception {
    JobCatalogWithTemplates underlying = Mockito.mock(JobCatalogWithTemplates.class);
    JobCatalogWithTemplates catalog = new PackagedTemplatesJobCatalogDecorator(underlying);
    JobTemplate classTemplate = catalog.getTemplate(new URI(PackagedTemplatesJobCatalogDecorator.CLASS + "://" + TestTemplate.class.getName()));
    Assert.assertEquals(classTemplate.getClass(), TestTemplate.class);
    try {
        catalog.getTemplate(new URI(PackagedTemplatesJobCatalogDecorator.CLASS + "://" + "non.existing.class"));
        Assert.fail();
    } catch (SpecNotFoundException exc) {
    // expect exception
    }
    JobTemplate resourceTemplate = catalog.getTemplate(new URI(PackagedTemplatesJobCatalogDecorator.RESOURCE + ":///templates/test.template"));
    Assert.assertEquals(resourceTemplate.getClass(), ResourceBasedJobTemplate.class);
    Assert.assertEquals(resourceTemplate.getRequiredConfigList().size(), 3);
    URI uri = new URI("scheme:///templates/test.template");
    try {
        catalog.getTemplate(uri);
        Assert.fail();
    } catch (SpecNotFoundException exc) {
    // expect exception
    }
    Mockito.verify(underlying).getTemplate(uri);
}

Example 5 with JobTemplate

use of org.apache.gobblin.runtime.api.JobTemplate in project incubator-gobblin by apache.

the class InheritingJobTemplate method resolveTemplates.

private void resolveTemplates(Map<URI, JobTemplate> loadedTemplates) throws SpecNotFoundException, TemplateException {
    if (this.resolved) {
        return;
    }
    this.superTemplates = Lists.newArrayList();
    for (URI uri : this.superTemplateUris) {
        if (!loadedTemplates.containsKey(uri)) {
            JobTemplate newTemplate = this.catalog.getTemplate(uri);
            loadedTemplates.put(uri, newTemplate);
            if (newTemplate instanceof InheritingJobTemplate) {
                ((InheritingJobTemplate) newTemplate).resolveTemplates(loadedTemplates);
            }
        }
        this.superTemplates.add(loadedTemplates.get(uri));
    }
    this.resolved = true;
}