Examples with RpcPipeKey com.alibaba.otter.node.etl.common.pipe.impl.rpc.RpcPipeKey used on opensource projects

Example 1 with RpcPipeKey

use of com.alibaba.otter.node.etl.common.pipe.impl.rpc.RpcPipeKey in project otter by alibaba.

the class RowDataPipeDelegate method get.

public DbBatch get(List<PipeKey> keys) {
    Assert.notNull(keys);
    DbBatch dbBatch = new DbBatch();
    Future<File> future = null;
    for (final PipeKey key : keys) {
        if (key == null) {
            // 忽略空的key
            continue;
        }
        if (key instanceof MemoryPipeKey) {
            dbBatch = rowDataMemoryPipe.get((MemoryPipeKey) key);
            // 直接返回
            return dbBatch;
        } else if (key instanceof HttpPipeKey) {
            if (key.getDataType().isDbBatch()) {
                // 区分一下数据下载
                dbBatch = rowDataHttpPipe.get((HttpPipeKey) key);
            } else {
                future = executorService.submit(new Callable<File>() {

                    public File call() throws Exception {
                        try {
                            HttpPipeKey pipeKey = (HttpPipeKey) key;
                            MDC.put(OtterConstants.splitPipelineLogFileKey, String.valueOf(pipeKey.getIdentity().getPipelineId()));
                            return attachmentHttpPipe.get(pipeKey);
                        } finally {
                            MDC.remove(OtterConstants.splitPipelineLogFileKey);
                        }
                    }
                });
            }
        } else if (key instanceof RpcPipeKey) {
            dbBatch = rowDataRpcPipe.get((RpcPipeKey) key);
        } else {
            throw new PipeException("unknow_PipeKey", key.toString());
        }
    }
    if (future != null && dbBatch != null) {
        try {
            dbBatch.setRoot(future.get());
        } catch (Exception e) {
            throw new PipeException(e);
        }
    }
    return dbBatch;
}

Example 2 with RpcPipeKey

use of com.alibaba.otter.node.etl.common.pipe.impl.rpc.RpcPipeKey in project otter by alibaba.

the class RpcPipeTest method test_ok.

@Test
public void test_ok() {
    final DbBatch source = new DbBatch();
    RowBatch rowBatch = new RowBatch();
    Identity identity = new Identity();
    identity.setChannelId(100L);
    identity.setPipelineId(100L);
    identity.setProcessId(100L);
    rowBatch.setIdentity(identity);
    source.setRowBatch(rowBatch);
    final RowDataRpcPipe pipe = new RowDataRpcPipe();
    try {
        pipe.afterPropertiesSet();
    } catch (Exception e) {
        want.fail();
    }
    Mockit.setUpMock(NodeCommmunicationClient.class, new Object() {

        @Mock
        public Object call(Long nid, final Event event) {
            try {
                return TestUtils.invokeMethod(pipe, "onGet", event);
            } catch (Exception e) {
                want.fail();
            }
            return null;
        }
    });
    Mockit.setUpMock(RowDataRpcPipe.class, new Object() {

        @Mock
        private Long getNid() {
            return 1L;
        }
    });
    pipe.setNodeCommmunicationClient(new NodeCommmunicationClient());
    RpcPipeKey key = pipe.put(source);
    DbBatch target = pipe.get(key);
    // identify相等
    want.bool(source.getRowBatch().getIdentity().equals(target.getRowBatch().getIdentity())).is(true);
}

Example 3 with RpcPipeKey

use of com.alibaba.otter.node.etl.common.pipe.impl.rpc.RpcPipeKey in project otter by alibaba.

the class RpcPipeTest method test_timeout.

@Test
public void test_timeout() {
    final DbBatch source = new DbBatch();
    RowBatch rowBatch = new RowBatch();
    Identity identity = new Identity();
    identity.setChannelId(100L);
    identity.setPipelineId(100L);
    identity.setProcessId(100L);
    rowBatch.setIdentity(identity);
    source.setRowBatch(rowBatch);
    final RowDataRpcPipe pipe = new RowDataRpcPipe();
    // 1s后超时
    pipe.setTimeout(1 * 1000L);
    try {
        pipe.afterPropertiesSet();
    } catch (Exception e) {
        want.fail();
    }
    Mockit.setUpMock(NodeCommmunicationClient.class, new Object() {

        @Mock
        public Object call(Long nid, final Event event) {
            try {
                return TestUtils.invokeMethod(pipe, "onGet", event);
            } catch (Exception e) {
                want.fail();
            }
            return null;
        }
    });
    Mockit.setUpMock(RowDataRpcPipe.class, new Object() {

        @Mock
        private Long getNid() {
            return 1L;
        }
    });
    pipe.setNodeCommmunicationClient(new NodeCommmunicationClient());
    RpcPipeKey key = pipe.put(source);
    try {
        Thread.sleep(1500L);
    } catch (InterruptedException e) {
        want.fail();
    }
    DbBatch target = pipe.get(key);
    // 返回结果为空
    want.bool(target == null).is(true);
}