Examples with Communication com.alibaba.datax.core.statistics.communication.Communication used on opensource projects

Example 26 with Communication

use of com.alibaba.datax.core.statistics.communication.Communication in project DataX by alibaba.

the class TaskGroupContainerTest method testLongTimeWriter.

@Test
public void testLongTimeWriter() {
    this.configuration.set("plugin.writer.fakewriter.class", FakeOneReader.class.getName());
    this.configuration.set("plugin.writer.fakewriter.class", FakeLongTimeWriter.class.getName());
    this.configuration.set(CoreConstant.DATAX_CORE_CONTAINER_TASKGROUP_CHANNEL, 1);
    Configuration jobContent = this.configuration.getListConfiguration(CoreConstant.DATAX_JOB_CONTENT).get(0);
    List<Configuration> jobContents = new ArrayList<Configuration>();
    jobContents.add(jobContent);
    this.configuration.set(CoreConstant.DATAX_JOB_CONTENT, jobContents);
    TaskGroupContainer taskGroupContainer = new TaskGroupContainer(this.configuration);
    taskGroupContainer.start();
    Assert.assertTrue(State.SUCCEEDED == taskGroupContainer.getContainerCommunicator().collect().getState());
    Communication res = null;
    try {
        Method com = TaskGroupContainer.class.getDeclaredMethod("reportTaskGroupCommunication", Communication.class, int.class);
        com.setAccessible(true);
        res = (Communication) com.invoke(taskGroupContainer, new Communication(), 1);
        System.out.println("TaskGroup => " + CommunicationTool.Stringify.getSnapshot(res));
    } catch (Exception e) {
        e.printStackTrace();
    }
    Assert.assertTrue(res != null);
    Assert.assertEquals(res.getLongCounter(CommunicationTool.TOTAL_READ_RECORDS).longValue(), 40);
    Assert.assertEquals(res.getLongCounter(CommunicationTool.TRANSFORMER_SUCCEED_RECORDS).longValue(), 0);
    Assert.assertEquals(res.getLongCounter(CommunicationTool.TRANSFORMER_FAILED_RECORDS).longValue(), 0);
    Assert.assertEquals(res.getLongCounter(CommunicationTool.TRANSFORMER_FILTER_RECORDS).longValue(), 0);
    Assert.assertEquals(res.getLongCounter(CommunicationTool.TRANSFORMER_USED_TIME).longValue(), 0);
    System.out.println(res.getMessage());
}

Example 27 with Communication

use of com.alibaba.datax.core.statistics.communication.Communication in project DataX by alibaba.

the class TaskGroupContainerTest method testStart.

@Test
public void testStart() throws InterruptedException {
    TaskGroupContainer taskGroupContainer = new TaskGroupContainer(this.configuration);
    taskGroupContainer.start();
    AbstractContainerCommunicator collector = taskGroupContainer.getContainerCommunicator();
    while (true) {
        State totalTaskState = collector.collectState();
        if (totalTaskState.isRunning()) {
            Thread.sleep(1000);
        } else {
            break;
        }
    }
    Communication totalTaskCommunication = collector.collect();
    List<String> messages = totalTaskCommunication.getMessage("bazhen-reader");
    Assert.assertTrue(!messages.isEmpty());
    messages = totalTaskCommunication.getMessage("bazhen-writer");
    Assert.assertTrue(!messages.isEmpty());
    messages = totalTaskCommunication.getMessage("bazhen");
    Assert.assertNull(messages);
    State state = totalTaskCommunication.getState();
    Assert.assertTrue("task finished", state.equals(State.SUCCEEDED));
}

Example 28 with Communication

use of com.alibaba.datax.core.statistics.communication.Communication in project DataX by alibaba.

the class TaskGroupContainerTest method testFakeTransformer.

@Test
public void testFakeTransformer() {
    TransformerRegistry.registTransformer(new FakeSubstrTransformer());
    TransformerRegistry.registTransformer(new FakeReplaceTransformer());
    TransformerRegistry.registTransformer(new FakeGroovyTransformer());
    LoadUtil.bind(configurationFakeTransformer);
    this.configurationFakeTransformer.set("plugin.writer.fakewriter.class", FakeOneReader.class.getName());
    this.configurationFakeTransformer.set("plugin.writer.fakewriter.class", FakeLongTimeWriter.class.getName());
    this.configurationFakeTransformer.set(CoreConstant.DATAX_CORE_CONTAINER_TASKGROUP_CHANNEL, 1);
    Configuration jobContent = this.configurationFakeTransformer.getListConfiguration(CoreConstant.DATAX_JOB_CONTENT).get(0);
    List<Configuration> jobContents = new ArrayList<Configuration>();
    jobContents.add(jobContent);
    this.configurationFakeTransformer.set(CoreConstant.DATAX_JOB_CONTENT, jobContents);
    TaskGroupContainer taskGroupContainer = new TaskGroupContainer(this.configurationFakeTransformer);
    taskGroupContainer.start();
    Assert.assertTrue(State.SUCCEEDED == taskGroupContainer.getContainerCommunicator().collect().getState());
    Communication res = null;
    try {
        Method com = TaskGroupContainer.class.getDeclaredMethod("reportTaskGroupCommunication", Communication.class, int.class);
        com.setAccessible(true);
        res = (Communication) com.invoke(taskGroupContainer, new Communication(), 1);
        System.out.println("TaskGroup => " + CommunicationTool.Stringify.getSnapshot(res));
    } catch (Exception e) {
        e.printStackTrace();
    }
    Assert.assertTrue(res != null);
    Assert.assertEquals(res.getLongCounter(CommunicationTool.TOTAL_READ_RECORDS).longValue(), 30);
    Assert.assertEquals(res.getLongCounter(CommunicationTool.TRANSFORMER_SUCCEED_RECORDS).longValue(), 10);
    Assert.assertEquals(res.getLongCounter(CommunicationTool.TRANSFORMER_FAILED_RECORDS).longValue(), 0);
    Assert.assertEquals(res.getLongCounter(CommunicationTool.TRANSFORMER_FILTER_RECORDS).longValue(), 10);
}

Example 29 with Communication

use of com.alibaba.datax.core.statistics.communication.Communication in project DataX by alibaba.

the class ErrorRecordLimitTest method testCheckRecordLimit2.

@Test
public void testCheckRecordLimit2() throws Exception {
    ErrorRecordChecker errLimit = new ErrorRecordChecker(1L, 0.5);
    errLimit.checkRecordLimit(new Communication() {

        {
            this.setLongCounter(CommunicationTool.WRITE_FAILED_RECORDS, 1);
        }
    });
}

Example 30 with Communication

use of com.alibaba.datax.core.statistics.communication.Communication in project DataX by alibaba.

the class ErrorRecordLimitTest method testCheckRecordLimit.

@Test(expected = DataXException.class)
public void testCheckRecordLimit() throws Exception {
    ErrorRecordChecker errLimit = new ErrorRecordChecker(0L, 0.5);
    errLimit.checkRecordLimit(new Communication() {

        {
            this.setLongCounter(CommunicationTool.WRITE_FAILED_RECORDS, 1);
        }
    });
}