Search in sources :

Example 16 with FileBatch

use of com.alibaba.otter.shared.etl.model.FileBatch in project otter by alibaba.

the class HttpPipeIntegration method test_rowData.

@Test
public void test_rowData() {
    final Node currentNode = new Node();
    currentNode.setId(1L);
    currentNode.setIp("127.0.0.1");
    currentNode.setParameters(new NodeParameter());
    final Pipeline pipeline = new Pipeline();
    pipeline.getParameters().setRetriever(RetrieverType.ARIA2C);
    // mock一下
    new NonStrictExpectations() {

        {
            configClientService.currentNode();
            returns(currentNode);
            configClientService.findPipeline(anyLong);
            returns(pipeline);
        }
    };
    Identity identity = new Identity();
    identity.setChannelId(100L);
    identity.setPipelineId(100L);
    identity.setProcessId(100L);
    FileBatch fileBatch = new FileBatch();
    fileBatch.setIdentity(identity);
    File localFile = new File(tmp, "httpPipeTest.jpg");
    FileData localFileData = new FileData();
    localFileData.setPath(localFile.getPath());
    localFileData.setEventType(EventType.INSERT);
    localFileData.setLastModifiedTime(new Date().getTime());
    localFileData.setSize(100L);
    localFileData.setTableId(1L);
    fileBatch.getFiles().add(localFileData);
    RowBatch rowBatch = new RowBatch();
    rowBatch.setIdentity(identity);
    EventData eventData = new EventData();
    eventData.setTableId(1L);
    eventData.setSchemaName("otter");
    eventData.setTableName("test");
    eventData.setEventType(EventType.INSERT);
    eventData.setExecuteTime(100L);
    EventColumn primaryKey = new EventColumn();
    primaryKey.setColumnName("id");
    primaryKey.setColumnType(1);
    primaryKey.setColumnValue("1");
    primaryKey.setKey(true);
    primaryKey.setNull(false);
    eventData.getKeys().add(primaryKey);
    EventColumn column = new EventColumn();
    column.setColumnName("name");
    column.setColumnType(1);
    column.setColumnValue("test");
    column.setKey(false);
    column.setNull(false);
    eventData.getColumns().add(column);
    rowBatch.merge(eventData);
    DbBatch dbBatch = new DbBatch();
    dbBatch.setRowBatch(rowBatch);
    dbBatch.setFileBatch(fileBatch);
    HttpPipeKey key = rowDataHttpPipe.put(dbBatch);
    DbBatch target = rowDataHttpPipe.get(key);
    want.bool(target.getRowBatch().getIdentity().equals(identity));
    want.object(target).notNull();
}
Also used : FileBatch(com.alibaba.otter.shared.etl.model.FileBatch) EventColumn(com.alibaba.otter.shared.etl.model.EventColumn) Node(com.alibaba.otter.shared.common.model.config.node.Node) Date(java.util.Date) EventData(com.alibaba.otter.shared.etl.model.EventData) DbBatch(com.alibaba.otter.shared.etl.model.DbBatch) Pipeline(com.alibaba.otter.shared.common.model.config.pipeline.Pipeline) HttpPipeKey(com.alibaba.otter.node.etl.common.pipe.impl.http.HttpPipeKey) RowBatch(com.alibaba.otter.shared.etl.model.RowBatch) NodeParameter(com.alibaba.otter.shared.common.model.config.node.NodeParameter) Identity(com.alibaba.otter.shared.etl.model.Identity) File(java.io.File) FileData(com.alibaba.otter.shared.etl.model.FileData) Test(org.testng.annotations.Test) BaseOtterTest(com.alibaba.otter.node.etl.BaseOtterTest)

Example 17 with FileBatch

use of com.alibaba.otter.shared.etl.model.FileBatch in project otter by alibaba.

the class FileBatchConflictDetectServiceIntegration method test_localFile.

@Test
public void test_localFile() {
    final Pipeline pipeline = new Pipeline();
    pipeline.setId(100L);
    final Node currentNode = new Node();
    currentNode.setId(1L);
    new NonStrictExpectations() {

        {
            configClientService.currentNode();
            returns(currentNode);
            configClientService.findPipeline(anyLong);
            returns(pipeline);
        }
    };
    Identity identity = new Identity();
    identity.setChannelId(100L);
    identity.setPipelineId(100L);
    identity.setProcessId(100L);
    FileBatch fileBatch = new FileBatch();
    fileBatch.setIdentity(identity);
    fileBatch.getFiles().addAll(generatorLocalFileData("fileLoad", 10));
    FileBatch result = fileBatchConflictDetectService.detect(fileBatch, 1L);
    want.number(result.getFiles().size()).isEqualTo(0);
    NioUtils.delete(new File(tmp + File.separator + OTTERLOAD));
}
Also used : FileBatch(com.alibaba.otter.shared.etl.model.FileBatch) Node(com.alibaba.otter.shared.common.model.config.node.Node) Identity(com.alibaba.otter.shared.etl.model.Identity) File(java.io.File) Pipeline(com.alibaba.otter.shared.common.model.config.pipeline.Pipeline) Test(org.testng.annotations.Test) BaseOtterTest(com.alibaba.otter.node.etl.BaseOtterTest)

Example 18 with FileBatch

use of com.alibaba.otter.shared.etl.model.FileBatch in project otter by alibaba.

the class OtterTransformerFactory method merge.

// =============================== helper method
// ============================
// 将生成的item对象合并到结果对象中
private synchronized void merge(Identity identity, Map<Class, BatchObject> data, Object item) {
    Class clazz = item.getClass();
    BatchObject batchObject = data.get(clazz);
    // 初始化一下对象
    if (batchObject == null) {
        batchObject = initBatchObject(identity, clazz);
        data.put(clazz, batchObject);
    }
    // 进行merge处理
    if (batchObject instanceof RowBatch) {
        ((RowBatch) batchObject).merge((EventData) item);
    } else if (batchObject instanceof FileBatch) {
        ((FileBatch) batchObject).getFiles().add((FileData) item);
    } else {
        throw new TransformException("no support Data[" + clazz.getName() + "]");
    }
}
Also used : FileBatch(com.alibaba.otter.shared.etl.model.FileBatch) RowBatch(com.alibaba.otter.shared.etl.model.RowBatch) BatchObject(com.alibaba.otter.shared.etl.model.BatchObject) TransformException(com.alibaba.otter.node.etl.transform.exception.TransformException) FileData(com.alibaba.otter.shared.etl.model.FileData)

Aggregations

FileBatch (com.alibaba.otter.shared.etl.model.FileBatch)18 Pipeline (com.alibaba.otter.shared.common.model.config.pipeline.Pipeline)9 FileData (com.alibaba.otter.shared.etl.model.FileData)9 Identity (com.alibaba.otter.shared.etl.model.Identity)9 File (java.io.File)8 Test (org.testng.annotations.Test)8 RowBatch (com.alibaba.otter.shared.etl.model.RowBatch)7 BaseDbTest (com.alibaba.otter.node.etl.BaseDbTest)5 DbBatch (com.alibaba.otter.shared.etl.model.DbBatch)4 IOException (java.io.IOException)4 BaseOtterTest (com.alibaba.otter.node.etl.BaseOtterTest)3 WeightController (com.alibaba.otter.node.etl.load.loader.weight.WeightController)3 Node (com.alibaba.otter.shared.common.model.config.node.Node)3 EventColumn (com.alibaba.otter.shared.etl.model.EventColumn)3 EventData (com.alibaba.otter.shared.etl.model.EventData)3 ArrayList (java.util.ArrayList)3 PipeException (com.alibaba.otter.node.etl.common.pipe.exception.PipeException)2 HttpPipeKey (com.alibaba.otter.node.etl.common.pipe.impl.http.HttpPipeKey)2 LoadException (com.alibaba.otter.node.etl.load.exception.LoadException)2 FileLoadContext (com.alibaba.otter.node.etl.load.loader.db.context.FileLoadContext)2