Examples with SpecNotFoundException org.apache.gobblin.runtime.api.SpecNotFoundException used on opensource projects

Example 6 with SpecNotFoundException

use of org.apache.gobblin.runtime.api.SpecNotFoundException in project incubator-gobblin by apache.

the class MultiHopsFlowToJobSpecCompiler method buildJobSpec.

/**
 * Generate JobSpec based on the #templateURI that user specified.
 */
private JobSpec buildJobSpec(ServiceNode sourceNode, ServiceNode targetNode, URI templateURI, FlowSpec flowSpec) {
    JobSpec jobSpec;
    JobSpec.Builder jobSpecBuilder = JobSpec.builder(jobSpecURIGenerator(flowSpec, sourceNode, targetNode)).withConfig(flowSpec.getConfig()).withDescription(flowSpec.getDescription()).withVersion(flowSpec.getVersion());
    if (templateURI != null) {
        jobSpecBuilder.withTemplate(templateURI);
        try {
            jobSpec = new ResolvedJobSpec(jobSpecBuilder.build(), templateCatalog.get());
            log.info("Resolved JobSpec properties are: " + jobSpec.getConfigAsProperties());
        } catch (SpecNotFoundException | JobTemplate.TemplateException e) {
            throw new RuntimeException("Could not resolve template in JobSpec from TemplateCatalog", e);
        }
    } else {
        jobSpec = jobSpecBuilder.build();
        log.info("Unresolved JobSpec properties are: " + jobSpec.getConfigAsProperties());
    }
    // Remove schedule
    jobSpec.setConfig(jobSpec.getConfig().withoutPath(ConfigurationKeys.JOB_SCHEDULE_KEY));
    // Add job.name and job.group
    if (flowSpec.getConfig().hasPath(ConfigurationKeys.FLOW_NAME_KEY)) {
        jobSpec.setConfig(jobSpec.getConfig().withValue(ConfigurationKeys.JOB_NAME_KEY, ConfigValueFactory.fromAnyRef(flowSpec.getConfig().getValue(ConfigurationKeys.FLOW_NAME_KEY).unwrapped().toString() + "-" + sourceNode.getNodeName() + "-" + targetNode.getNodeName())));
    }
    if (flowSpec.getConfig().hasPath(ConfigurationKeys.FLOW_GROUP_KEY)) {
        jobSpec.setConfig(jobSpec.getConfig().withValue(ConfigurationKeys.JOB_GROUP_KEY, flowSpec.getConfig().getValue(ConfigurationKeys.FLOW_GROUP_KEY)));
    }
    // Add flow execution id for this compilation
    long flowExecutionId = System.currentTimeMillis();
    jobSpec.setConfig(jobSpec.getConfig().withValue(ConfigurationKeys.FLOW_EXECUTION_ID_KEY, ConfigValueFactory.fromAnyRef(flowExecutionId)));
    // Reset properties in Spec from Config
    jobSpec.setConfigAsProperties(ConfigUtils.configToProperties(jobSpec.getConfig()));
    return jobSpec;
}

Example 7 with SpecNotFoundException

use of org.apache.gobblin.runtime.api.SpecNotFoundException in project incubator-gobblin by apache.

the class BaseFlowToJobSpecCompiler method jobSpecGenerator.

/**
 * Naive implementation of generating jobSpec, which fetch the first available template,
 * in an exemplified single-hop FlowCompiler implementation.
 * @param flowSpec
 * @return
 */
protected JobSpec jobSpecGenerator(FlowSpec flowSpec) {
    JobSpec jobSpec;
    JobSpec.Builder jobSpecBuilder = JobSpec.builder(jobSpecURIGenerator(flowSpec)).withConfig(flowSpec.getConfig()).withDescription(flowSpec.getDescription()).withVersion(flowSpec.getVersion());
    if (flowSpec.getTemplateURIs().isPresent() && templateCatalog.isPresent()) {
        // Only first template uri will be honored for Identity
        jobSpecBuilder = jobSpecBuilder.withTemplate(flowSpec.getTemplateURIs().get().iterator().next());
        try {
            jobSpec = new ResolvedJobSpec(jobSpecBuilder.build(), templateCatalog.get());
            log.info("Resolved JobSpec properties are: " + jobSpec.getConfigAsProperties());
        } catch (SpecNotFoundException | JobTemplate.TemplateException e) {
            throw new RuntimeException("Could not resolve template in JobSpec from TemplateCatalog", e);
        }
    } else {
        jobSpec = jobSpecBuilder.build();
        log.info("Unresolved JobSpec properties are: " + jobSpec.getConfigAsProperties());
    }
    // Remove schedule
    jobSpec.setConfig(jobSpec.getConfig().withoutPath(ConfigurationKeys.JOB_SCHEDULE_KEY));
    // Add job.name and job.group
    if (flowSpec.getConfig().hasPath(ConfigurationKeys.FLOW_NAME_KEY)) {
        jobSpec.setConfig(jobSpec.getConfig().withValue(ConfigurationKeys.JOB_NAME_KEY, flowSpec.getConfig().getValue(ConfigurationKeys.FLOW_NAME_KEY)));
    }
    if (flowSpec.getConfig().hasPath(ConfigurationKeys.FLOW_GROUP_KEY)) {
        jobSpec.setConfig(jobSpec.getConfig().withValue(ConfigurationKeys.JOB_GROUP_KEY, flowSpec.getConfig().getValue(ConfigurationKeys.FLOW_GROUP_KEY)));
    }
    // Add flow execution id for this compilation
    long flowExecutionId = System.currentTimeMillis();
    jobSpec.setConfig(jobSpec.getConfig().withValue(ConfigurationKeys.FLOW_EXECUTION_ID_KEY, ConfigValueFactory.fromAnyRef(flowExecutionId)));
    // Reset properties in Spec from Config
    jobSpec.setConfigAsProperties(ConfigUtils.configToProperties(jobSpec.getConfig()));
    return jobSpec;
}

Example 8 with SpecNotFoundException

use of org.apache.gobblin.runtime.api.SpecNotFoundException in project incubator-gobblin by apache.

the class PackagedTemplatesJobCatalogDecoratorTest method test.

@Test
public void test() throws Exception {
    JobCatalogWithTemplates underlying = Mockito.mock(JobCatalogWithTemplates.class);
    JobCatalogWithTemplates catalog = new PackagedTemplatesJobCatalogDecorator(underlying);
    JobTemplate classTemplate = catalog.getTemplate(new URI(PackagedTemplatesJobCatalogDecorator.CLASS + "://" + TestTemplate.class.getName()));
    Assert.assertEquals(classTemplate.getClass(), TestTemplate.class);
    try {
        catalog.getTemplate(new URI(PackagedTemplatesJobCatalogDecorator.CLASS + "://" + "non.existing.class"));
        Assert.fail();
    } catch (SpecNotFoundException exc) {
    // expect exception
    }
    JobTemplate resourceTemplate = catalog.getTemplate(new URI(PackagedTemplatesJobCatalogDecorator.RESOURCE + ":///templates/test.template"));
    Assert.assertEquals(resourceTemplate.getClass(), ResourceBasedJobTemplate.class);
    Assert.assertEquals(resourceTemplate.getRequiredConfigList().size(), 3);
    URI uri = new URI("scheme:///templates/test.template");
    try {
        catalog.getTemplate(uri);
        Assert.fail();
    } catch (SpecNotFoundException exc) {
    // expect exception
    }
    Mockito.verify(underlying).getTemplate(uri);
}

Example 9 with SpecNotFoundException

use of org.apache.gobblin.runtime.api.SpecNotFoundException in project incubator-gobblin by apache.

the class PullFileToConfigConverter method convert.

public void convert() throws IOException {
    Config baseConfig = ConfigFactory.parseString(DO_NOT_OVERRIDE_KEY + ": []");
    FileSystem pullFileFs = pullFileRootPath.getFileSystem(new Configuration());
    FileSystem outputFs = this.outputPath.getFileSystem(new Configuration());
    Config sysConfig = ConfigFactory.parseFile(this.sysConfigPath);
    PullFileLoader pullFileLoader = new PullFileLoader(this.pullFileRootPath, pullFileFs, PullFileLoader.DEFAULT_JAVA_PROPS_PULL_FILE_EXTENSIONS, PullFileLoader.DEFAULT_HOCON_PULL_FILE_EXTENSIONS);
    PackagedTemplatesJobCatalogDecorator catalog = new PackagedTemplatesJobCatalogDecorator();
    ConfigResolveOptions configResolveOptions = ConfigResolveOptions.defaults();
    configResolveOptions = configResolveOptions.setAllowUnresolved(true);
    ResourceBasedJobTemplate template;
    Config templateConfig;
    try {
        template = (ResourceBasedJobTemplate) catalog.getTemplate(templateURI.toUri());
        templateConfig = sysConfig.withFallback(template.getRawTemplateConfig()).withFallback(baseConfig).resolve(configResolveOptions);
    } catch (SpecNotFoundException | JobTemplate.TemplateException exc) {
        throw new IOException(exc);
    }
    Set<String> doNotOverride = templateConfig.hasPath(DO_NOT_OVERRIDE_KEY) ? Sets.newHashSet(templateConfig.getStringList(DO_NOT_OVERRIDE_KEY)) : Sets.<String>newHashSet();
    ConfigRenderOptions configRenderOptions = ConfigRenderOptions.defaults();
    configRenderOptions = configRenderOptions.setComments(false);
    configRenderOptions = configRenderOptions.setOriginComments(false);
    configRenderOptions = configRenderOptions.setFormatted(true);
    configRenderOptions = configRenderOptions.setJson(false);
    for (FileStatus pullFile : pullFileFs.globStatus(this.fileGlobToConvert)) {
        Config pullFileConfig = pullFileLoader.loadPullFile(pullFile.getPath(), ConfigFactory.empty(), true).resolve();
        Map<String, String> outputConfigMap = Maps.newHashMap();
        outputConfigMap.put(ConfigurationKeys.JOB_TEMPLATE_PATH, this.templateURI.toString());
        boolean somethingChanged;
        do {
            somethingChanged = false;
            Config currentOutputConfig = ConfigFactory.parseMap(outputConfigMap);
            Config currentResolvedConfig = currentOutputConfig.withFallback(templateConfig).resolve(configResolveOptions);
            for (Map.Entry<Object, Object> entry : ConfigUtils.configToProperties(pullFileConfig).entrySet()) {
                String key = (String) entry.getKey();
                String value = (String) entry.getValue();
                try {
                    if ((!currentResolvedConfig.hasPath(key)) || (!currentResolvedConfig.getString(key).equals(value) && !doNotOverride.contains(key))) {
                        if (!FILTER_KEYS.contains(key)) {
                            somethingChanged = true;
                            outputConfigMap.put(key, value);
                        }
                    }
                } catch (ConfigException.NotResolved nre) {
                // path is unresolved in config, will try again next iteration
                }
            }
        } while (somethingChanged);
        try {
            Config outputConfig = ConfigFactory.parseMap(outputConfigMap);
            Config currentResolvedConfig = outputConfig.withFallback(templateConfig).resolve();
            String rendered = outputConfig.root().render(configRenderOptions);
            Path newPath = PathUtils.removeExtension(pullFile.getPath(), PullFileLoader.DEFAULT_JAVA_PROPS_PULL_FILE_EXTENSIONS.toArray(new String[] {}));
            newPath = PathUtils.addExtension(newPath, "conf");
            newPath = new Path(this.outputPath, newPath.getName());
            FSDataOutputStream os = outputFs.create(newPath);
            os.write(rendered.getBytes(Charsets.UTF_8));
            os.close();
        } catch (ConfigException.NotResolved nre) {
            throw new IOException("Not all configuration keys were resolved in pull file " + pullFile.getPath(), nre);
        }
    }
}

Example 10 with SpecNotFoundException

use of org.apache.gobblin.runtime.api.SpecNotFoundException in project incubator-gobblin by apache.

the class FlowConfigsResource method get.

/**
 * Retrieve the flow configuration with the given key
 * @param key flow config id key containing group name and flow name
 * @return {@link FlowConfig} with flow configuration
 */
@Override
public FlowConfig get(ComplexResourceKey<FlowId, EmptyRecord> key) {
    String flowGroup = key.getKey().getFlowGroup();
    String flowName = key.getKey().getFlowName();
    LOG.info("Get called with flowGroup " + flowGroup + " flowName " + flowName);
    try {
        URI flowCatalogURI = new URI("gobblin-flow", null, "/", null, null);
        URI flowUri = new URI(flowCatalogURI.getScheme(), flowCatalogURI.getAuthority(), "/" + flowGroup + "/" + flowName, null, null);
        FlowSpec spec = (FlowSpec) getFlowCatalog().getSpec(flowUri);
        FlowConfig flowConfig = new FlowConfig();
        Properties flowProps = spec.getConfigAsProperties();
        Schedule schedule = null;
        if (flowProps.containsKey(ConfigurationKeys.JOB_SCHEDULE_KEY)) {
            schedule = new Schedule();
            schedule.setCronSchedule(flowProps.getProperty(ConfigurationKeys.JOB_SCHEDULE_KEY));
        }
        if (flowProps.containsKey(ConfigurationKeys.JOB_TEMPLATE_PATH)) {
            flowConfig.setTemplateUris(flowProps.getProperty(ConfigurationKeys.JOB_TEMPLATE_PATH));
        } else if (spec.getTemplateURIs().isPresent()) {
            flowConfig.setTemplateUris(StringUtils.join(spec.getTemplateURIs().get(), ","));
        } else {
            flowConfig.setTemplateUris("NA");
        }
        if (schedule != null) {
            if (flowProps.containsKey(ConfigurationKeys.FLOW_RUN_IMMEDIATELY)) {
                schedule.setRunImmediately(Boolean.valueOf(flowProps.getProperty(ConfigurationKeys.FLOW_RUN_IMMEDIATELY)));
            }
            flowConfig.setSchedule(schedule);
        }
        // remove keys that were injected as part of flowSpec creation
        flowProps.remove(ConfigurationKeys.JOB_SCHEDULE_KEY);
        flowProps.remove(ConfigurationKeys.JOB_TEMPLATE_PATH);
        StringMap flowPropsAsStringMap = new StringMap();
        flowPropsAsStringMap.putAll(Maps.fromProperties(flowProps));
        return flowConfig.setId(new FlowId().setFlowGroup(flowGroup).setFlowName(flowName)).setProperties(flowPropsAsStringMap);
    } catch (URISyntaxException e) {
        logAndThrowRestLiServiceException(HttpStatus.S_400_BAD_REQUEST, "bad URI " + flowName, e);
    } catch (SpecNotFoundException e) {
        logAndThrowRestLiServiceException(HttpStatus.S_404_NOT_FOUND, "Flow requested does not exist: " + flowName, null);
    }
    return null;
}