Examples with MemoryPipeKey com.alibaba.otter.node.etl.common.pipe.impl.memory.MemoryPipeKey used on opensource projects

Example 1 with MemoryPipeKey

use of com.alibaba.otter.node.etl.common.pipe.impl.memory.MemoryPipeKey in project otter by alibaba.

the class MemoryPipeTest method test_timeout.

@Test
public void test_timeout() {
    RowDataMemoryPipe pipe = new RowDataMemoryPipe();
    // 1s后超时
    pipe.setTimeout(1 * 1000L);
    pipe.setDownloadDir(tmp);
    try {
        pipe.afterPropertiesSet();
    } catch (Exception e) {
        want.fail();
    }
    DbBatch source = new DbBatch();
    RowBatch rowBatch = new RowBatch();
    Identity identity = new Identity();
    identity.setChannelId(100L);
    identity.setPipelineId(100L);
    identity.setProcessId(100L);
    rowBatch.setIdentity(identity);
    source.setRowBatch(rowBatch);
    MemoryPipeKey key = pipe.put(source);
    try {
        Thread.sleep(1500L);
    } catch (InterruptedException e) {
        want.fail();
    }
    DbBatch target = pipe.get(key);
    // 返回结果为空
    want.bool(target == null).is(true);
}

Example 2 with MemoryPipeKey

use of com.alibaba.otter.node.etl.common.pipe.impl.memory.MemoryPipeKey in project otter by alibaba.

the class RowDataPipeDelegate method get.

public DbBatch get(List<PipeKey> keys) {
    Assert.notNull(keys);
    DbBatch dbBatch = new DbBatch();
    Future<File> future = null;
    for (final PipeKey key : keys) {
        if (key == null) {
            // 忽略空的key
            continue;
        }
        if (key instanceof MemoryPipeKey) {
            dbBatch = rowDataMemoryPipe.get((MemoryPipeKey) key);
            // 直接返回
            return dbBatch;
        } else if (key instanceof HttpPipeKey) {
            if (key.getDataType().isDbBatch()) {
                // 区分一下数据下载
                dbBatch = rowDataHttpPipe.get((HttpPipeKey) key);
            } else {
                future = executorService.submit(new Callable<File>() {

                    public File call() throws Exception {
                        try {
                            HttpPipeKey pipeKey = (HttpPipeKey) key;
                            MDC.put(OtterConstants.splitPipelineLogFileKey, String.valueOf(pipeKey.getIdentity().getPipelineId()));
                            return attachmentHttpPipe.get(pipeKey);
                        } finally {
                            MDC.remove(OtterConstants.splitPipelineLogFileKey);
                        }
                    }
                });
            }
        } else if (key instanceof RpcPipeKey) {
            dbBatch = rowDataRpcPipe.get((RpcPipeKey) key);
        } else {
            throw new PipeException("unknow_PipeKey", key.toString());
        }
    }
    if (future != null && dbBatch != null) {
        try {
            dbBatch.setRoot(future.get());
        } catch (Exception e) {
            throw new PipeException(e);
        }
    }
    return dbBatch;
}

Example 3 with MemoryPipeKey

use of com.alibaba.otter.node.etl.common.pipe.impl.memory.MemoryPipeKey in project otter by alibaba.

the class MemoryPipeTest method test_ok.

@Test
public void test_ok() {
    RowDataMemoryPipe pipe = new RowDataMemoryPipe();
    pipe.setDownloadDir(tmp);
    try {
        pipe.afterPropertiesSet();
    } catch (Exception e) {
        want.fail();
    }
    DbBatch source = new DbBatch();
    RowBatch rowBatch = new RowBatch();
    Identity identity = new Identity();
    identity.setChannelId(100L);
    identity.setPipelineId(100L);
    identity.setProcessId(100L);
    rowBatch.setIdentity(identity);
    source.setRowBatch(rowBatch);
    MemoryPipeKey key = pipe.put(source);
    DbBatch target = pipe.get(key);
    // 引用为同一个
    want.bool(source == target).is(true);
}