Examples with FlowSpec org.apache.gobblin.runtime.api.FlowSpec used on opensource projects

Example 31 with FlowSpec

use of org.apache.gobblin.runtime.api.FlowSpec in project incubator-gobblin by apache.

the class FlowCatalogTest method initFlowSpec.

private FlowSpec initFlowSpec() {
    Properties properties = new Properties();
    properties.put("specStore.fs.dir", SPEC_STORE_DIR);
    properties.put("specExecInstance.capabilities", "source:destination");
    Config config = ConfigUtils.propertiesToConfig(properties);
    SpecExecutor specExecutorInstanceProducer = new InMemorySpecExecutor(config);
    FlowSpec.Builder flowSpecBuilder = null;
    try {
        flowSpecBuilder = FlowSpec.builder(computeFlowSpecURI()).withConfig(config).withDescription(SPEC_DESCRIPTION).withVersion(SPEC_VERSION).withTemplate(new URI("templateURI"));
    } catch (URISyntaxException e) {
        throw new RuntimeException(e);
    }
    return flowSpecBuilder.build();
}

Example 32 with FlowSpec

use of org.apache.gobblin.runtime.api.FlowSpec in project incubator-gobblin by apache.

the class FlowConfigsResource method get.

/**
 * Retrieve the flow configuration with the given key
 * @param key flow config id key containing group name and flow name
 * @return {@link FlowConfig} with flow configuration
 */
@Override
public FlowConfig get(ComplexResourceKey<FlowId, EmptyRecord> key) {
    String flowGroup = key.getKey().getFlowGroup();
    String flowName = key.getKey().getFlowName();
    LOG.info("Get called with flowGroup " + flowGroup + " flowName " + flowName);
    try {
        URI flowCatalogURI = new URI("gobblin-flow", null, "/", null, null);
        URI flowUri = new URI(flowCatalogURI.getScheme(), flowCatalogURI.getAuthority(), "/" + flowGroup + "/" + flowName, null, null);
        FlowSpec spec = (FlowSpec) getFlowCatalog().getSpec(flowUri);
        FlowConfig flowConfig = new FlowConfig();
        Properties flowProps = spec.getConfigAsProperties();
        Schedule schedule = null;
        if (flowProps.containsKey(ConfigurationKeys.JOB_SCHEDULE_KEY)) {
            schedule = new Schedule();
            schedule.setCronSchedule(flowProps.getProperty(ConfigurationKeys.JOB_SCHEDULE_KEY));
        }
        if (flowProps.containsKey(ConfigurationKeys.JOB_TEMPLATE_PATH)) {
            flowConfig.setTemplateUris(flowProps.getProperty(ConfigurationKeys.JOB_TEMPLATE_PATH));
        } else if (spec.getTemplateURIs().isPresent()) {
            flowConfig.setTemplateUris(StringUtils.join(spec.getTemplateURIs().get(), ","));
        } else {
            flowConfig.setTemplateUris("NA");
        }
        if (schedule != null) {
            if (flowProps.containsKey(ConfigurationKeys.FLOW_RUN_IMMEDIATELY)) {
                schedule.setRunImmediately(Boolean.valueOf(flowProps.getProperty(ConfigurationKeys.FLOW_RUN_IMMEDIATELY)));
            }
            flowConfig.setSchedule(schedule);
        }
        // remove keys that were injected as part of flowSpec creation
        flowProps.remove(ConfigurationKeys.JOB_SCHEDULE_KEY);
        flowProps.remove(ConfigurationKeys.JOB_TEMPLATE_PATH);
        StringMap flowPropsAsStringMap = new StringMap();
        flowPropsAsStringMap.putAll(Maps.fromProperties(flowProps));
        return flowConfig.setId(new FlowId().setFlowGroup(flowGroup).setFlowName(flowName)).setProperties(flowPropsAsStringMap);
    } catch (URISyntaxException e) {
        logAndThrowRestLiServiceException(HttpStatus.S_400_BAD_REQUEST, "bad URI " + flowName, e);
    } catch (SpecNotFoundException e) {
        logAndThrowRestLiServiceException(HttpStatus.S_404_NOT_FOUND, "Flow requested does not exist: " + flowName, null);
    }
    return null;
}