Search in sources :

Example 16 with Identity

use of com.alibaba.otter.shared.etl.model.Identity in project otter by alibaba.

the class HttpPipeIntegration method test_attachment.

@Test
public void test_attachment() {
    final Node currentNode = new Node();
    currentNode.setId(1L);
    currentNode.setIp("127.0.0.1");
    currentNode.setParameters(new NodeParameter());
    final Pipeline pipeline = new Pipeline();
    pipeline.getParameters().setRetriever(RetrieverType.ARIA2C);
    // mock一下
    new NonStrictExpectations() {

        {
            configClientService.currentNode();
            returns(currentNode);
            configClientService.findPipeline(anyLong);
            returns(pipeline);
        }
    };
    Identity identity = new Identity();
    identity.setChannelId(100L);
    identity.setPipelineId(100L);
    identity.setProcessId(100L);
    FileBatch fileBatch = new FileBatch();
    fileBatch.setIdentity(identity);
    File localFile = new File(tmp, "httpPipeTest.jpg");
    FileData localFileData = new FileData();
    localFileData.setEventType(EventType.INSERT);
    localFileData.setPath(localFile.getPath());
    fileBatch.getFiles().add(localFileData);
    try {
        byte[] data = getBlock(10 * 1024);
        NioUtils.write(data, localFile);
        HttpPipeKey key = attachmentHttpPipe.put(fileBatch);
        File target = attachmentHttpPipe.get(key);
        byte[] getbytes = NioUtils.read(new File(target, localFile.getPath()));
        check(data, getbytes);
    } catch (IOException e) {
        want.fail();
    } finally {
        NioUtils.delete(localFile);
    }
}
Also used : HttpPipeKey(com.alibaba.otter.node.etl.common.pipe.impl.http.HttpPipeKey) FileBatch(com.alibaba.otter.shared.etl.model.FileBatch) NodeParameter(com.alibaba.otter.shared.common.model.config.node.NodeParameter) Node(com.alibaba.otter.shared.common.model.config.node.Node) IOException(java.io.IOException) Identity(com.alibaba.otter.shared.etl.model.Identity) File(java.io.File) FileData(com.alibaba.otter.shared.etl.model.FileData) Pipeline(com.alibaba.otter.shared.common.model.config.pipeline.Pipeline) Test(org.testng.annotations.Test) BaseOtterTest(com.alibaba.otter.node.etl.BaseOtterTest)

Example 17 with Identity

use of com.alibaba.otter.shared.etl.model.Identity in project otter by alibaba.

the class HttpPipeIntegration method test_rowData.

@Test
public void test_rowData() {
    final Node currentNode = new Node();
    currentNode.setId(1L);
    currentNode.setIp("127.0.0.1");
    currentNode.setParameters(new NodeParameter());
    final Pipeline pipeline = new Pipeline();
    pipeline.getParameters().setRetriever(RetrieverType.ARIA2C);
    // mock一下
    new NonStrictExpectations() {

        {
            configClientService.currentNode();
            returns(currentNode);
            configClientService.findPipeline(anyLong);
            returns(pipeline);
        }
    };
    Identity identity = new Identity();
    identity.setChannelId(100L);
    identity.setPipelineId(100L);
    identity.setProcessId(100L);
    FileBatch fileBatch = new FileBatch();
    fileBatch.setIdentity(identity);
    File localFile = new File(tmp, "httpPipeTest.jpg");
    FileData localFileData = new FileData();
    localFileData.setPath(localFile.getPath());
    localFileData.setEventType(EventType.INSERT);
    localFileData.setLastModifiedTime(new Date().getTime());
    localFileData.setSize(100L);
    localFileData.setTableId(1L);
    fileBatch.getFiles().add(localFileData);
    RowBatch rowBatch = new RowBatch();
    rowBatch.setIdentity(identity);
    EventData eventData = new EventData();
    eventData.setTableId(1L);
    eventData.setSchemaName("otter");
    eventData.setTableName("test");
    eventData.setEventType(EventType.INSERT);
    eventData.setExecuteTime(100L);
    EventColumn primaryKey = new EventColumn();
    primaryKey.setColumnName("id");
    primaryKey.setColumnType(1);
    primaryKey.setColumnValue("1");
    primaryKey.setKey(true);
    primaryKey.setNull(false);
    eventData.getKeys().add(primaryKey);
    EventColumn column = new EventColumn();
    column.setColumnName("name");
    column.setColumnType(1);
    column.setColumnValue("test");
    column.setKey(false);
    column.setNull(false);
    eventData.getColumns().add(column);
    rowBatch.merge(eventData);
    DbBatch dbBatch = new DbBatch();
    dbBatch.setRowBatch(rowBatch);
    dbBatch.setFileBatch(fileBatch);
    HttpPipeKey key = rowDataHttpPipe.put(dbBatch);
    DbBatch target = rowDataHttpPipe.get(key);
    want.bool(target.getRowBatch().getIdentity().equals(identity));
    want.object(target).notNull();
}
Also used : FileBatch(com.alibaba.otter.shared.etl.model.FileBatch) EventColumn(com.alibaba.otter.shared.etl.model.EventColumn) Node(com.alibaba.otter.shared.common.model.config.node.Node) Date(java.util.Date) EventData(com.alibaba.otter.shared.etl.model.EventData) DbBatch(com.alibaba.otter.shared.etl.model.DbBatch) Pipeline(com.alibaba.otter.shared.common.model.config.pipeline.Pipeline) HttpPipeKey(com.alibaba.otter.node.etl.common.pipe.impl.http.HttpPipeKey) RowBatch(com.alibaba.otter.shared.etl.model.RowBatch) NodeParameter(com.alibaba.otter.shared.common.model.config.node.NodeParameter) Identity(com.alibaba.otter.shared.etl.model.Identity) File(java.io.File) FileData(com.alibaba.otter.shared.etl.model.FileData) Test(org.testng.annotations.Test) BaseOtterTest(com.alibaba.otter.node.etl.BaseOtterTest)

Example 18 with Identity

use of com.alibaba.otter.shared.etl.model.Identity in project otter by alibaba.

the class MemoryPipeTest method test_ok.

@Test
public void test_ok() {
    RowDataMemoryPipe pipe = new RowDataMemoryPipe();
    pipe.setDownloadDir(tmp);
    try {
        pipe.afterPropertiesSet();
    } catch (Exception e) {
        want.fail();
    }
    DbBatch source = new DbBatch();
    RowBatch rowBatch = new RowBatch();
    Identity identity = new Identity();
    identity.setChannelId(100L);
    identity.setPipelineId(100L);
    identity.setProcessId(100L);
    rowBatch.setIdentity(identity);
    source.setRowBatch(rowBatch);
    MemoryPipeKey key = pipe.put(source);
    DbBatch target = pipe.get(key);
    // 引用为同一个
    want.bool(source == target).is(true);
}
Also used : RowBatch(com.alibaba.otter.shared.etl.model.RowBatch) MemoryPipeKey(com.alibaba.otter.node.etl.common.pipe.impl.memory.MemoryPipeKey) Identity(com.alibaba.otter.shared.etl.model.Identity) RowDataMemoryPipe(com.alibaba.otter.node.etl.common.pipe.impl.memory.RowDataMemoryPipe) DbBatch(com.alibaba.otter.shared.etl.model.DbBatch) Test(org.testng.annotations.Test) BaseOtterTest(com.alibaba.otter.node.etl.BaseOtterTest)

Example 19 with Identity

use of com.alibaba.otter.shared.etl.model.Identity in project otter by alibaba.

the class RpcPipeTest method test_ok.

@Test
public void test_ok() {
    final DbBatch source = new DbBatch();
    RowBatch rowBatch = new RowBatch();
    Identity identity = new Identity();
    identity.setChannelId(100L);
    identity.setPipelineId(100L);
    identity.setProcessId(100L);
    rowBatch.setIdentity(identity);
    source.setRowBatch(rowBatch);
    final RowDataRpcPipe pipe = new RowDataRpcPipe();
    try {
        pipe.afterPropertiesSet();
    } catch (Exception e) {
        want.fail();
    }
    Mockit.setUpMock(NodeCommmunicationClient.class, new Object() {

        @Mock
        public Object call(Long nid, final Event event) {
            try {
                return TestUtils.invokeMethod(pipe, "onGet", event);
            } catch (Exception e) {
                want.fail();
            }
            return null;
        }
    });
    Mockit.setUpMock(RowDataRpcPipe.class, new Object() {

        @Mock
        private Long getNid() {
            return 1L;
        }
    });
    pipe.setNodeCommmunicationClient(new NodeCommmunicationClient());
    RpcPipeKey key = pipe.put(source);
    DbBatch target = pipe.get(key);
    // identify相等
    want.bool(source.getRowBatch().getIdentity().equals(target.getRowBatch().getIdentity())).is(true);
}
Also used : RowDataRpcPipe(com.alibaba.otter.node.etl.common.pipe.impl.rpc.RowDataRpcPipe) RowBatch(com.alibaba.otter.shared.etl.model.RowBatch) Event(com.alibaba.otter.shared.communication.core.model.Event) NodeCommmunicationClient(com.alibaba.otter.node.common.communication.NodeCommmunicationClient) Identity(com.alibaba.otter.shared.etl.model.Identity) DbBatch(com.alibaba.otter.shared.etl.model.DbBatch) Mock(mockit.Mock) RpcPipeKey(com.alibaba.otter.node.etl.common.pipe.impl.rpc.RpcPipeKey) Test(org.testng.annotations.Test) BaseOtterTest(com.alibaba.otter.node.etl.BaseOtterTest)

Example 20 with Identity

use of com.alibaba.otter.shared.etl.model.Identity in project otter by alibaba.

the class RpcPipeTest method test_timeout.

@Test
public void test_timeout() {
    final DbBatch source = new DbBatch();
    RowBatch rowBatch = new RowBatch();
    Identity identity = new Identity();
    identity.setChannelId(100L);
    identity.setPipelineId(100L);
    identity.setProcessId(100L);
    rowBatch.setIdentity(identity);
    source.setRowBatch(rowBatch);
    final RowDataRpcPipe pipe = new RowDataRpcPipe();
    // 1s后超时
    pipe.setTimeout(1 * 1000L);
    try {
        pipe.afterPropertiesSet();
    } catch (Exception e) {
        want.fail();
    }
    Mockit.setUpMock(NodeCommmunicationClient.class, new Object() {

        @Mock
        public Object call(Long nid, final Event event) {
            try {
                return TestUtils.invokeMethod(pipe, "onGet", event);
            } catch (Exception e) {
                want.fail();
            }
            return null;
        }
    });
    Mockit.setUpMock(RowDataRpcPipe.class, new Object() {

        @Mock
        private Long getNid() {
            return 1L;
        }
    });
    pipe.setNodeCommmunicationClient(new NodeCommmunicationClient());
    RpcPipeKey key = pipe.put(source);
    try {
        Thread.sleep(1500L);
    } catch (InterruptedException e) {
        want.fail();
    }
    DbBatch target = pipe.get(key);
    // 返回结果为空
    want.bool(target == null).is(true);
}
Also used : RowDataRpcPipe(com.alibaba.otter.node.etl.common.pipe.impl.rpc.RowDataRpcPipe) RowBatch(com.alibaba.otter.shared.etl.model.RowBatch) Event(com.alibaba.otter.shared.communication.core.model.Event) NodeCommmunicationClient(com.alibaba.otter.node.common.communication.NodeCommmunicationClient) Identity(com.alibaba.otter.shared.etl.model.Identity) DbBatch(com.alibaba.otter.shared.etl.model.DbBatch) Mock(mockit.Mock) RpcPipeKey(com.alibaba.otter.node.etl.common.pipe.impl.rpc.RpcPipeKey) Test(org.testng.annotations.Test) BaseOtterTest(com.alibaba.otter.node.etl.BaseOtterTest)

Aggregations

Identity (com.alibaba.otter.shared.etl.model.Identity)28 Test (org.testng.annotations.Test)16 Pipeline (com.alibaba.otter.shared.common.model.config.pipeline.Pipeline)12 RowBatch (com.alibaba.otter.shared.etl.model.RowBatch)12 BaseDbTest (com.alibaba.otter.node.etl.BaseDbTest)9 EventData (com.alibaba.otter.shared.etl.model.EventData)9 FileBatch (com.alibaba.otter.shared.etl.model.FileBatch)9 DataMediaPair (com.alibaba.otter.shared.common.model.config.data.DataMediaPair)8 BaseOtterTest (com.alibaba.otter.node.etl.BaseOtterTest)7 DbBatch (com.alibaba.otter.shared.etl.model.DbBatch)7 File (java.io.File)6 Channel (com.alibaba.otter.shared.common.model.config.channel.Channel)5 Node (com.alibaba.otter.shared.common.model.config.node.Node)5 BatchObject (com.alibaba.otter.shared.etl.model.BatchObject)5 FileData (com.alibaba.otter.shared.etl.model.FileData)5 WeightController (com.alibaba.otter.node.etl.load.loader.weight.WeightController)4 DbDataMedia (com.alibaba.otter.shared.common.model.config.data.db.DbDataMedia)3 ArrayList (java.util.ArrayList)3 NodeCommmunicationClient (com.alibaba.otter.node.common.communication.NodeCommmunicationClient)2 HttpPipeKey (com.alibaba.otter.node.etl.common.pipe.impl.http.HttpPipeKey)2