Examples with FlowConfig org.apache.gobblin.service.FlowConfig used on opensource projects

Example 6 with FlowConfig

use of org.apache.gobblin.service.FlowConfig in project incubator-gobblin by apache.

the class GobblinServiceManager method testGobblinService.

// TODO: Remove after adding test cases
@SuppressWarnings("DLS_DEAD_LOCAL_STORE")
private static void testGobblinService(GobblinServiceManager gobblinServiceManager) {
    FlowConfigClient client = new FlowConfigClient(String.format("http://localhost:%s/", gobblinServiceManager.restliServer.getPort()));
    Map<String, String> flowProperties = Maps.newHashMap();
    flowProperties.put("param1", "value1");
    final String TEST_GROUP_NAME = "testGroup1";
    final String TEST_FLOW_NAME = "testFlow1";
    final String TEST_SCHEDULE = "0 1/0 * ? * *";
    final String TEST_TEMPLATE_URI = "FS:///templates/test.template";
    FlowConfig flowConfig = new FlowConfig().setId(new FlowId().setFlowGroup(TEST_GROUP_NAME).setFlowName(TEST_FLOW_NAME)).setTemplateUris(TEST_TEMPLATE_URI).setSchedule(new Schedule().setCronSchedule(TEST_SCHEDULE).setRunImmediately(true)).setProperties(new StringMap(flowProperties));
    try {
        client.createFlowConfig(flowConfig);
    } catch (RemoteInvocationException e) {
        throw new RuntimeException(e);
    }
}

Example 7 with FlowConfig

use of org.apache.gobblin.service.FlowConfig in project incubator-gobblin by apache.

the class GobblinServiceHATest method testCreate.

@Test
public void testCreate() throws Exception {
    logger.info("+++++++++++++++++++ testCreate START");
    Map<String, String> flowProperties = Maps.newHashMap();
    flowProperties.put("param1", "value1");
    flowProperties.put(ServiceConfigKeys.FLOW_SOURCE_IDENTIFIER_KEY, TEST_SOURCE_NAME);
    flowProperties.put(ServiceConfigKeys.FLOW_DESTINATION_IDENTIFIER_KEY, TEST_SINK_NAME);
    FlowConfig flowConfig1 = new FlowConfig().setId(new FlowId().setFlowGroup(TEST_GROUP_NAME_1).setFlowName(TEST_FLOW_NAME_1)).setTemplateUris(TEST_TEMPLATE_URI_1).setSchedule(new Schedule().setCronSchedule(TEST_SCHEDULE_1).setRunImmediately(true)).setProperties(new StringMap(flowProperties));
    FlowConfig flowConfig2 = new FlowConfig().setId(new FlowId().setFlowGroup(TEST_GROUP_NAME_2).setFlowName(TEST_FLOW_NAME_2)).setTemplateUris(TEST_TEMPLATE_URI_2).setSchedule(new Schedule().setCronSchedule(TEST_SCHEDULE_2).setRunImmediately(true)).setProperties(new StringMap(flowProperties));
    // Try create on both nodes
    long schedulingStartTime = System.currentTimeMillis();
    this.node1FlowConfigClient.createFlowConfig(flowConfig1);
    this.node2FlowConfigClient.createFlowConfig(flowConfig2);
    // Check if created on master
    GobblinServiceManager master;
    if (this.node1GobblinServiceManager.isLeader()) {
        master = this.node1GobblinServiceManager;
        logger.info("#### node 1 is manager");
    } else if (this.node2GobblinServiceManager.isLeader()) {
        master = this.node2GobblinServiceManager;
        logger.info("#### node 2 is manager");
    } else {
        Assert.fail("No leader found in service cluster");
        return;
    }
    int attempt = 0;
    boolean assertSuccess = false;
    // Below while-loop will read all flow specs, but some of them are being persisted.
    // We have seen CRC file java.io.EOFException when reading and writing at the same time.
    // Wait for a few seconds to guarantee all the flow specs are persisted.
    Thread.sleep(3000);
    while (attempt < 800) {
        int masterJobs = master.flowCatalog.getSpecs().size();
        if (masterJobs == 2) {
            assertSuccess = true;
            break;
        }
        Thread.sleep(5);
        attempt++;
    }
    long schedulingEndTime = System.currentTimeMillis();
    logger.info("Total scheduling time in ms: " + (schedulingEndTime - schedulingStartTime));
    Assert.assertTrue(assertSuccess, "Flow that was created is not reflecting in FlowCatalog");
    logger.info("+++++++++++++++++++ testCreate END");
}

Example 8 with FlowConfig

use of org.apache.gobblin.service.FlowConfig in project incubator-gobblin by apache.

the class GobblinServiceHATest method testDelete.

@Test(dependsOnMethods = "testUpdate")
public void testDelete() throws Exception {
    logger.info("+++++++++++++++++++ testDelete START");
    FlowId flowId = new FlowId().setFlowGroup(TEST_GROUP_NAME_1).setFlowName(TEST_FLOW_NAME_1);
    // make sure flow config exists
    FlowConfig flowConfig = this.node1FlowConfigClient.getFlowConfig(flowId);
    Assert.assertEquals(flowConfig.getId().getFlowGroup(), TEST_GROUP_NAME_1);
    Assert.assertEquals(flowConfig.getId().getFlowName(), TEST_FLOW_NAME_1);
    this.node1FlowConfigClient.deleteFlowConfig(flowId);
    // Check if deletion is reflected on both nodes
    try {
        this.node1FlowConfigClient.getFlowConfig(flowId);
        Assert.fail("Get should have gotten a 404 error");
    } catch (RestLiResponseException e) {
        Assert.assertEquals(e.getStatus(), HttpStatus.NOT_FOUND_404);
    }
    try {
        this.node2FlowConfigClient.getFlowConfig(flowId);
        Assert.fail("Get should have gotten a 404 error");
    } catch (RestLiResponseException e) {
        Assert.assertEquals(e.getStatus(), HttpStatus.NOT_FOUND_404);
    }
    logger.info("+++++++++++++++++++ testDelete END");
}

Example 9 with FlowConfig

use of org.apache.gobblin.service.FlowConfig in project incubator-gobblin by apache.

the class GobblinServiceHATest method testCreateAgain.

@Test(dependsOnMethods = "testCreate")
public void testCreateAgain() throws Exception {
    logger.info("+++++++++++++++++++ testCreateAgain START");
    Map<String, String> flowProperties = Maps.newHashMap();
    flowProperties.put("param1", "value1");
    flowProperties.put(ServiceConfigKeys.FLOW_SOURCE_IDENTIFIER_KEY, TEST_SOURCE_NAME);
    flowProperties.put(ServiceConfigKeys.FLOW_DESTINATION_IDENTIFIER_KEY, TEST_SINK_NAME);
    FlowConfig flowConfig1 = new FlowConfig().setId(new FlowId().setFlowGroup(TEST_GROUP_NAME_1).setFlowName(TEST_FLOW_NAME_1)).setTemplateUris(TEST_TEMPLATE_URI_1).setSchedule(new Schedule().setCronSchedule(TEST_SCHEDULE_1).setRunImmediately(true)).setProperties(new StringMap(flowProperties));
    FlowConfig flowConfig2 = new FlowConfig().setId(new FlowId().setFlowGroup(TEST_GROUP_NAME_2).setFlowName(TEST_FLOW_NAME_2)).setTemplateUris(TEST_TEMPLATE_URI_2).setSchedule(new Schedule().setCronSchedule(TEST_SCHEDULE_2).setRunImmediately(true)).setProperties(new StringMap(flowProperties));
    // Try create on both nodes
    try {
        this.node1FlowConfigClient.createFlowConfig(flowConfig1);
    } catch (RestLiResponseException e) {
        Assert.fail("Create Again should pass without complaining that the spec already exists.");
    }
    try {
        this.node2FlowConfigClient.createFlowConfig(flowConfig2);
    } catch (RestLiResponseException e) {
        Assert.fail("Create Again should pass without complaining that the spec already exists.");
    }
    logger.info("+++++++++++++++++++ testCreateAgain END");
}

Example 10 with FlowConfig

use of org.apache.gobblin.service.FlowConfig in project incubator-gobblin by apache.

the class GobblinServiceRedirectTest method testCreate.

@Test
public void testCreate() throws Exception {
    Map<String, String> flowProperties = Maps.newHashMap();
    flowProperties.put("param1", "value1");
    flowProperties.put(ServiceConfigKeys.FLOW_SOURCE_IDENTIFIER_KEY, TEST_SOURCE_NAME);
    flowProperties.put(ServiceConfigKeys.FLOW_DESTINATION_IDENTIFIER_KEY, TEST_SINK_NAME);
    FlowConfig flowConfig1 = new FlowConfig().setId(new FlowId().setFlowGroup(TEST_GROUP_NAME_1).setFlowName(TEST_FLOW_NAME_1)).setTemplateUris(TEST_TEMPLATE_URI_1).setSchedule(new Schedule().setCronSchedule(TEST_SCHEDULE_1).setRunImmediately(true)).setProperties(new StringMap(flowProperties));
    FlowConfig flowConfig2 = new FlowConfig().setId(new FlowId().setFlowGroup(TEST_GROUP_NAME_2).setFlowName(TEST_FLOW_NAME_2)).setTemplateUris(TEST_TEMPLATE_URI_2).setSchedule(new Schedule().setCronSchedule(TEST_SCHEDULE_2).setRunImmediately(true)).setProperties(new StringMap(flowProperties));
    GobblinServiceManager leader;
    FlowConfigClient leaderClient;
    FlowConfigClient slaveClient;
    if (this.node1GobblinServiceManager.isLeader()) {
        leader = this.node1GobblinServiceManager;
        leaderClient = this.node1FlowConfigClient;
        slaveClient = this.node2FlowConfigClient;
    } else {
        leader = this.node2GobblinServiceManager;
        leaderClient = this.node2FlowConfigClient;
        slaveClient = this.node1FlowConfigClient;
    }
    // Try create on leader, should be successful
    leaderClient.createFlowConfig(flowConfig1);
    // Try create on slave, should throw an error with leader URL
    try {
        slaveClient.createFlowConfig(flowConfig2);
    } catch (RestLiResponseException e) {
        Assert.assertTrue(e.hasErrorDetails());
        Assert.assertTrue(e.getErrorDetails().containsKey(ServiceConfigKeys.LEADER_URL));
        String expectedUrl = PREFIX + InetAddress.getLocalHost().getHostName() + ":" + leader.restliServer.getPort() + "/" + SERVICE_NAME;
        Assert.assertEquals(e.getErrorDetails().get(ServiceConfigKeys.LEADER_URL), expectedUrl);
        return;
    }
    throw new RuntimeException("Slave should have thrown an error");
}