Examples with HOCONInputStreamJobTemplate org.apache.gobblin.runtime.template.HOCONInputStreamJobTemplate used on opensource projects

Example 1 with HOCONInputStreamJobTemplate

use of org.apache.gobblin.runtime.template.HOCONInputStreamJobTemplate in project incubator-gobblin by apache.

the class FSFlowTemplateCatalog method getJobTemplatesForFlow.

/**
 * @param flowTemplateDirURI Relative URI of the flow template directory
 * @return a list of {@link JobTemplate}s for a given flow identified by its {@link URI}.
 * @throws IOException
 * @throws SpecNotFoundException
 * @throws JobTemplate.TemplateException
 */
public List<JobTemplate> getJobTemplatesForFlow(URI flowTemplateDirURI) throws IOException, SpecNotFoundException, JobTemplate.TemplateException, URISyntaxException {
    PathFilter extensionFilter = file -> {
        for (String extension : JOB_FILE_EXTENSIONS) {
            if (file.getName().endsWith(extension)) {
                return true;
            }
        }
        return false;
    };
    if (!validateTemplateURI(flowTemplateDirURI)) {
        throw new JobTemplate.TemplateException(String.format("The FlowTemplate %s is not valid", flowTemplateDirURI));
    }
    List<JobTemplate> jobTemplates = new ArrayList<>();
    String templateCatalogDir = this.sysConfig.getString(ServiceConfigKeys.TEMPLATE_CATALOGS_FULLY_QUALIFIED_PATH_KEY);
    // Flow templates are located under templateCatalogDir/flowEdgeTemplates
    Path flowTemplateDirPath = PathUtils.mergePaths(new Path(templateCatalogDir), new Path(flowTemplateDirURI));
    // Job files (with extension .job) are located under templateCatalogDir/flowEdgeTemplates/jobs directory.
    Path jobFilePath = new Path(flowTemplateDirPath, JOBS_DIR_NAME);
    FileSystem fs = FileSystem.get(jobFilePath.toUri(), new Configuration());
    for (FileStatus fileStatus : fs.listStatus(jobFilePath, extensionFilter)) {
        Config jobConfig = loadHoconFileAtPath(fileStatus.getPath());
        // Check if the .job file has an underlying job template
        if (jobConfig.hasPath(GOBBLIN_JOB_TEMPLATE_KEY)) {
            URI jobTemplateRelativeUri = new URI(jobConfig.getString(GOBBLIN_JOB_TEMPLATE_KEY));
            if (!jobTemplateRelativeUri.getScheme().equals(FS_SCHEME)) {
                throw new RuntimeException("Expected scheme " + FS_SCHEME + " got unsupported scheme " + flowTemplateDirURI.getScheme());
            }
            Path fullJobTemplatePath = PathUtils.mergePaths(new Path(templateCatalogDir), new Path(jobTemplateRelativeUri));
            jobConfig = jobConfig.withFallback(loadHoconFileAtPath(fullJobTemplatePath));
        }
        jobTemplates.add(new HOCONInputStreamJobTemplate(jobConfig, fileStatus.getPath().toUri(), this));
    }
    return jobTemplates;
}