Examples with PackagedTemplatesJobCatalogDecorator org.apache.gobblin.runtime.job_catalog.PackagedTemplatesJobCatalogDecorator used on opensource projects

Example 1 with PackagedTemplatesJobCatalogDecorator

use of org.apache.gobblin.runtime.job_catalog.PackagedTemplatesJobCatalogDecorator in project incubator-gobblin by apache.

the class SchedulerUtils method resolveTemplate.

private static Properties resolveTemplate(Properties jobProps) throws IOException {
    try {
        if (jobProps.containsKey(ConfigurationKeys.JOB_TEMPLATE_PATH)) {
            Config jobConfig = ConfigUtils.propertiesToConfig(jobProps);
            Properties resolvedProps = ConfigUtils.configToProperties((ResourceBasedJobTemplate.forResourcePath(jobProps.getProperty(ConfigurationKeys.JOB_TEMPLATE_PATH), new PackagedTemplatesJobCatalogDecorator())).getResolvedConfig(jobConfig));
            return resolvedProps;
        } else {
            return jobProps;
        }
    } catch (JobTemplate.TemplateException | SpecNotFoundException | URISyntaxException exc) {
        throw new IOException(exc);
    }
}

Example 2 with PackagedTemplatesJobCatalogDecorator

use of org.apache.gobblin.runtime.job_catalog.PackagedTemplatesJobCatalogDecorator in project incubator-gobblin by apache.

the class TemplateTest method testResolvingConfig.

// Testing the resolving of userCustomized attributes and template is correct.
@Test
public void testResolvingConfig() throws Exception {
    Config jobProps = ConfigFactory.parseProperties(this.userProp);
    Assert.assertEquals(ConfigUtils.configToProperties(jobProps).size(), 6);
    jobProps = ResourceBasedJobTemplate.forResourcePath(jobProps.getString(ConfigurationKeys.JOB_TEMPLATE_PATH), new PackagedTemplatesJobCatalogDecorator()).getResolvedConfig(jobProps);
    // Remove job.template in userSpecified file and gobblin.template.required_attributes in template
    Assert.assertEquals(ConfigUtils.configToProperties(jobProps).size(), 8);
    Properties targetResolvedJobProps = new Properties();
    targetResolvedJobProps.setProperty("a", "1");
    targetResolvedJobProps.setProperty("templated0", "2");
    targetResolvedJobProps.setProperty("templated1", "y");
    targetResolvedJobProps.setProperty("required0", "r0");
    targetResolvedJobProps.setProperty("required1", "r1");
    targetResolvedJobProps.setProperty("required2", "r2");
    targetResolvedJobProps.setProperty(ConfigurationKeys.JOB_TEMPLATE_PATH, "templates/test.template");
    targetResolvedJobProps.setProperty(ConfigurationKeys.REQUIRED_ATRRIBUTES_LIST, "required0,required1,required2");
    Assert.assertEquals(targetResolvedJobProps, ConfigUtils.configToProperties(jobProps));
}

Example 3 with PackagedTemplatesJobCatalogDecorator

use of org.apache.gobblin.runtime.job_catalog.PackagedTemplatesJobCatalogDecorator in project incubator-gobblin by apache.

the class TemplateTest method testRequiredAttrList.

@Test
public void testRequiredAttrList() throws Exception {
    Properties jobProps = this.userProp;
    Collection<String> requiredConfigList = ResourceBasedJobTemplate.forURI(new URI(jobProps.getProperty(ConfigurationKeys.JOB_TEMPLATE_PATH)), new PackagedTemplatesJobCatalogDecorator()).getRequiredConfigList();
    Assert.assertEquals(requiredConfigList.size(), 3);
    Assert.assertTrue(requiredConfigList.contains("required0"));
    Assert.assertTrue(requiredConfigList.contains("required1"));
    Assert.assertTrue(requiredConfigList.contains("required2"));
}

Example 4 with PackagedTemplatesJobCatalogDecorator

use of org.apache.gobblin.runtime.job_catalog.PackagedTemplatesJobCatalogDecorator in project incubator-gobblin by apache.

the class PullFileToConfigConverter method convert.

public void convert() throws IOException {
    Config baseConfig = ConfigFactory.parseString(DO_NOT_OVERRIDE_KEY + ": []");
    FileSystem pullFileFs = pullFileRootPath.getFileSystem(new Configuration());
    FileSystem outputFs = this.outputPath.getFileSystem(new Configuration());
    Config sysConfig = ConfigFactory.parseFile(this.sysConfigPath);
    PullFileLoader pullFileLoader = new PullFileLoader(this.pullFileRootPath, pullFileFs, PullFileLoader.DEFAULT_JAVA_PROPS_PULL_FILE_EXTENSIONS, PullFileLoader.DEFAULT_HOCON_PULL_FILE_EXTENSIONS);
    PackagedTemplatesJobCatalogDecorator catalog = new PackagedTemplatesJobCatalogDecorator();
    ConfigResolveOptions configResolveOptions = ConfigResolveOptions.defaults();
    configResolveOptions = configResolveOptions.setAllowUnresolved(true);
    ResourceBasedJobTemplate template;
    Config templateConfig;
    try {
        template = (ResourceBasedJobTemplate) catalog.getTemplate(templateURI.toUri());
        templateConfig = sysConfig.withFallback(template.getRawTemplateConfig()).withFallback(baseConfig).resolve(configResolveOptions);
    } catch (SpecNotFoundException | JobTemplate.TemplateException exc) {
        throw new IOException(exc);
    }
    Set<String> doNotOverride = templateConfig.hasPath(DO_NOT_OVERRIDE_KEY) ? Sets.newHashSet(templateConfig.getStringList(DO_NOT_OVERRIDE_KEY)) : Sets.<String>newHashSet();
    ConfigRenderOptions configRenderOptions = ConfigRenderOptions.defaults();
    configRenderOptions = configRenderOptions.setComments(false);
    configRenderOptions = configRenderOptions.setOriginComments(false);
    configRenderOptions = configRenderOptions.setFormatted(true);
    configRenderOptions = configRenderOptions.setJson(false);
    for (FileStatus pullFile : pullFileFs.globStatus(this.fileGlobToConvert)) {
        Config pullFileConfig = pullFileLoader.loadPullFile(pullFile.getPath(), ConfigFactory.empty(), true).resolve();
        Map<String, String> outputConfigMap = Maps.newHashMap();
        outputConfigMap.put(ConfigurationKeys.JOB_TEMPLATE_PATH, this.templateURI.toString());
        boolean somethingChanged;
        do {
            somethingChanged = false;
            Config currentOutputConfig = ConfigFactory.parseMap(outputConfigMap);
            Config currentResolvedConfig = currentOutputConfig.withFallback(templateConfig).resolve(configResolveOptions);
            for (Map.Entry<Object, Object> entry : ConfigUtils.configToProperties(pullFileConfig).entrySet()) {
                String key = (String) entry.getKey();
                String value = (String) entry.getValue();
                try {
                    if ((!currentResolvedConfig.hasPath(key)) || (!currentResolvedConfig.getString(key).equals(value) && !doNotOverride.contains(key))) {
                        if (!FILTER_KEYS.contains(key)) {
                            somethingChanged = true;
                            outputConfigMap.put(key, value);
                        }
                    }
                } catch (ConfigException.NotResolved nre) {
                // path is unresolved in config, will try again next iteration
                }
            }
        } while (somethingChanged);
        try {
            Config outputConfig = ConfigFactory.parseMap(outputConfigMap);
            Config currentResolvedConfig = outputConfig.withFallback(templateConfig).resolve();
            String rendered = outputConfig.root().render(configRenderOptions);
            Path newPath = PathUtils.removeExtension(pullFile.getPath(), PullFileLoader.DEFAULT_JAVA_PROPS_PULL_FILE_EXTENSIONS.toArray(new String[] {}));
            newPath = PathUtils.addExtension(newPath, "conf");
            newPath = new Path(this.outputPath, newPath.getName());
            FSDataOutputStream os = outputFs.create(newPath);
            os.write(rendered.getBytes(Charsets.UTF_8));
            os.close();
        } catch (ConfigException.NotResolved nre) {
            throw new IOException("Not all configuration keys were resolved in pull file " + pullFile.getPath(), nre);
        }
    }
}

Example 5 with PackagedTemplatesJobCatalogDecorator

use of org.apache.gobblin.runtime.job_catalog.PackagedTemplatesJobCatalogDecorator in project incubator-gobblin by apache.

the class ResolvedJobSpec method resolveConfig.

private static Config resolveConfig(JobSpec jobSpec, JobCatalog catalog) throws SpecNotFoundException, JobTemplate.TemplateException {
    Optional<URI> templateURIOpt = jobSpec.getTemplateURI();
    if (templateURIOpt.isPresent()) {
        JobCatalogWithTemplates catalogWithTemplates = new PackagedTemplatesJobCatalogDecorator(catalog);
        JobTemplate template = catalogWithTemplates.getTemplate(templateURIOpt.get());
        return template.getResolvedConfig(jobSpec.getConfig()).resolve();
    } else {
        return jobSpec.getConfig().resolve();
    }
}