Examples with FlowConfig org.apache.gobblin.service.FlowConfig used on opensource projects

Example 16 with FlowConfig

use of org.apache.gobblin.service.FlowConfig in project incubator-gobblin by apache.

the class FlowConfigUtilsTest method testFlowConfigWithoutSchedule.

public void testFlowConfigWithoutSchedule() {
    FlowConfig flowConfig = new FlowConfig().setId(new FlowId().setFlowName("SN_CRMSYNC").setFlowGroup("DYNAMICS-USER-123456789"));
    flowConfig.setTemplateUris("FS:///my.template");
    Properties properties = new Properties();
    properties.put("gobblin.flow.sourceIdentifier", "dynamicsCrm");
    properties.put("gobblin.flow.destinationIdentifier", "espresso");
    flowConfig.setProperties(new StringMap(Maps.fromProperties(properties)));
    testFlowSpec(flowConfig);
    testSerDer(flowConfig);
}

Example 17 with FlowConfig

use of org.apache.gobblin.service.FlowConfig in project incubator-gobblin by apache.

the class GobblinServiceHATest method testUpdate.

@Test(dependsOnMethods = "testGet")
public void testUpdate() throws Exception {
    logger.info("+++++++++++++++++++ testUpdate START");
    // Update on one node and retrieve from another
    FlowId flowId = new FlowId().setFlowGroup(TEST_GROUP_NAME_1).setFlowName(TEST_FLOW_NAME_1);
    Map<String, String> flowProperties = Maps.newHashMap();
    flowProperties.put("param1", "value1b");
    flowProperties.put("param2", "value2b");
    flowProperties.put(ServiceConfigKeys.FLOW_SOURCE_IDENTIFIER_KEY, TEST_SOURCE_NAME);
    flowProperties.put(ServiceConfigKeys.FLOW_DESTINATION_IDENTIFIER_KEY, TEST_SINK_NAME);
    FlowConfig flowConfig = new FlowConfig().setId(new FlowId().setFlowGroup(TEST_GROUP_NAME_1).setFlowName(TEST_FLOW_NAME_1)).setTemplateUris(TEST_TEMPLATE_URI_1).setSchedule(new Schedule().setCronSchedule(TEST_SCHEDULE_1)).setProperties(new StringMap(flowProperties));
    this.node1FlowConfigClient.updateFlowConfig(flowConfig);
    FlowConfig retrievedFlowConfig = this.node2FlowConfigClient.getFlowConfig(flowId);
    Assert.assertEquals(retrievedFlowConfig.getId().getFlowGroup(), TEST_GROUP_NAME_1);
    Assert.assertEquals(retrievedFlowConfig.getId().getFlowName(), TEST_FLOW_NAME_1);
    Assert.assertEquals(retrievedFlowConfig.getSchedule().getCronSchedule(), TEST_SCHEDULE_1);
    Assert.assertEquals(retrievedFlowConfig.getTemplateUris(), TEST_TEMPLATE_URI_1);
    Assert.assertFalse(retrievedFlowConfig.getSchedule().isRunImmediately());
    Assert.assertEquals(retrievedFlowConfig.getProperties().get("param1"), "value1b");
    Assert.assertEquals(retrievedFlowConfig.getProperties().get("param2"), "value2b");
    logger.info("+++++++++++++++++++ testUpdate END");
}

Example 18 with FlowConfig

use of org.apache.gobblin.service.FlowConfig in project incubator-gobblin by apache.

the class GobblinServiceHATest method testGet.

@Test(dependsOnMethods = "testCreateAgain")
public void testGet() throws Exception {
    logger.info("+++++++++++++++++++ testGet START");
    FlowId flowId1 = new FlowId().setFlowGroup(TEST_GROUP_NAME_1).setFlowName(TEST_FLOW_NAME_1);
    FlowConfig flowConfig1 = this.node1FlowConfigClient.getFlowConfig(flowId1);
    Assert.assertEquals(flowConfig1.getId().getFlowGroup(), TEST_GROUP_NAME_1);
    Assert.assertEquals(flowConfig1.getId().getFlowName(), TEST_FLOW_NAME_1);
    Assert.assertEquals(flowConfig1.getSchedule().getCronSchedule(), TEST_SCHEDULE_1);
    Assert.assertEquals(flowConfig1.getTemplateUris(), TEST_TEMPLATE_URI_1);
    Assert.assertTrue(flowConfig1.getSchedule().isRunImmediately());
    Assert.assertEquals(flowConfig1.getProperties().get("param1"), "value1");
    flowConfig1 = this.node2FlowConfigClient.getFlowConfig(flowId1);
    Assert.assertEquals(flowConfig1.getId().getFlowGroup(), TEST_GROUP_NAME_1);
    Assert.assertEquals(flowConfig1.getId().getFlowName(), TEST_FLOW_NAME_1);
    Assert.assertEquals(flowConfig1.getSchedule().getCronSchedule(), TEST_SCHEDULE_1);
    Assert.assertEquals(flowConfig1.getTemplateUris(), TEST_TEMPLATE_URI_1);
    Assert.assertTrue(flowConfig1.getSchedule().isRunImmediately());
    Assert.assertEquals(flowConfig1.getProperties().get("param1"), "value1");
    logger.info("+++++++++++++++++++ testGet END");
}

Example 19 with FlowConfig

use of org.apache.gobblin.service.FlowConfig in project incubator-gobblin by apache.

the class GobblinServiceHATest method testBadUpdate.

@Test(dependsOnMethods = "testBadDelete")
public void testBadUpdate() throws Exception {
    logger.info("+++++++++++++++++++ testBadUpdate START");
    Map<String, String> flowProperties = Maps.newHashMap();
    flowProperties.put("param1", "value1b");
    flowProperties.put("param2", "value2b");
    FlowConfig flowConfig = new FlowConfig().setId(new FlowId().setFlowGroup(TEST_DUMMY_GROUP_NAME_1).setFlowName(TEST_DUMMY_FLOW_NAME_1)).setTemplateUris(TEST_TEMPLATE_URI_1).setSchedule(new Schedule().setCronSchedule(TEST_SCHEDULE_1)).setProperties(new StringMap(flowProperties));
    try {
        this.node1FlowConfigClient.updateFlowConfig(flowConfig);
    } catch (RestLiResponseException e) {
        Assert.assertEquals(e.getStatus(), HttpStatus.NOT_FOUND_404);
    }
    try {
        this.node2FlowConfigClient.updateFlowConfig(flowConfig);
    } catch (RestLiResponseException e) {
        Assert.assertEquals(e.getStatus(), HttpStatus.NOT_FOUND_404);
    }
    logger.info("+++++++++++++++++++ testBadUpdate END");
}

Example 20 with FlowConfig

use of org.apache.gobblin.service.FlowConfig in project incubator-gobblin by apache.

the class FlowConfigUtils method deserializeFlowConfig.

public static FlowConfig deserializeFlowConfig(String serialized) throws IOException {
    Properties properties = PropertiesUtils.deserialize(serialized);
    FlowConfig flowConfig = new FlowConfig().setId(new FlowId().setFlowName(properties.getProperty(FLOWCONFIG_ID_NAME)).setFlowGroup(properties.getProperty(FLOWCONFIG_ID_GROUP)));
    if (properties.containsKey(FLOWCONFIG_SCHEDULE_CRON)) {
        flowConfig.setSchedule(new Schedule().setCronSchedule(properties.getProperty(FLOWCONFIG_SCHEDULE_CRON)).setRunImmediately(Boolean.valueOf(properties.getProperty(FLOWCONFIG_SCHEDULE_RUN_IMMEDIATELY))));
    }
    if (properties.containsKey(FLOWCONFIG_TEMPLATEURIS)) {
        flowConfig.setTemplateUris(properties.getProperty(FLOWCONFIG_TEMPLATEURIS));
    }
    properties.remove(FLOWCONFIG_ID_NAME);
    properties.remove(FLOWCONFIG_ID_GROUP);
    properties.remove(FLOWCONFIG_SCHEDULE_CRON);
    properties.remove(FLOWCONFIG_SCHEDULE_RUN_IMMEDIATELY);
    properties.remove(FLOWCONFIG_TEMPLATEURIS);
    flowConfig.setProperties(new StringMap(Maps.fromProperties(properties)));
    return flowConfig;
}