Search in sources :

Example 1 with FileBatch

use of com.alibaba.otter.shared.etl.model.FileBatch in project otter by alibaba.

the class OtterTransformerTest method test_fileData.

@Test
public void test_fileData() {
    final Pipeline pipeline = new Pipeline();
    pipeline.setId(100L);
    List<DataMediaPair> pairs = new ArrayList<DataMediaPair>();
    DataMediaPair pair1 = new DataMediaPair();
    pair1.setId(1L);
    pair1.setPipelineId(pipeline.getId());
    pair1.setPullWeight(1L);
    pair1.setPushWeight(1L);
    DbDataMedia oracleMedia = getOracleMedia();
    oracleMedia.setId(1L);
    pair1.setSource(oracleMedia);
    DbDataMedia mysqlMedia = getMysqlMedia();
    pair1.setTarget(mysqlMedia);
    pairs.add(pair1);
    pipeline.setPairs(pairs);
    new NonStrictExpectations() {

        {
            configClientService.findPipeline(anyLong);
            returns(pipeline);
        }
    };
    Identity identity = new Identity();
    identity.setChannelId(100L);
    identity.setPipelineId(100L);
    identity.setProcessId(100L);
    FileBatch fileBatch = new FileBatch();
    fileBatch.setIdentity(identity);
    File localFile = new File("/tmp", "httpPipeTest.jpg");
    FileData localFileData = new FileData();
    localFileData.setTableId(1L);
    localFileData.setPairId(1L);
    localFileData.setPath(localFile.getPath());
    fileBatch.getFiles().add(localFileData);
    Map<Class, BatchObject> batchs = otterTransformFactory.transform(fileBatch);
    FileBatch result = (FileBatch) batchs.get(FileData.class);
    want.number(result.getFiles().size()).isEqualTo(1);
}
Also used : FileBatch(com.alibaba.otter.shared.etl.model.FileBatch) DataMediaPair(com.alibaba.otter.shared.common.model.config.data.DataMediaPair) ArrayList(java.util.ArrayList) Pipeline(com.alibaba.otter.shared.common.model.config.pipeline.Pipeline) BatchObject(com.alibaba.otter.shared.etl.model.BatchObject) Identity(com.alibaba.otter.shared.etl.model.Identity) DbDataMedia(com.alibaba.otter.shared.common.model.config.data.db.DbDataMedia) File(java.io.File) FileData(com.alibaba.otter.shared.etl.model.FileData) Test(org.testng.annotations.Test) BaseDbTest(com.alibaba.otter.node.etl.BaseDbTest)

Example 2 with FileBatch

use of com.alibaba.otter.shared.etl.model.FileBatch in project otter by alibaba.

the class OtterTransformerFactory method initBatchObject.

// 根据对应的类型初始化batchObject对象
private BatchObject initBatchObject(Identity identity, Class clazz) {
    if (EventData.class.equals(clazz)) {
        RowBatch rowbatch = new RowBatch();
        rowbatch.setIdentity(identity);
        return rowbatch;
    } else if (FileData.class.equals(clazz)) {
        FileBatch fileBatch = new FileBatch();
        fileBatch.setIdentity(identity);
        return fileBatch;
    } else {
        throw new TransformException("no support Data[" + clazz.getName() + "]");
    }
}
Also used : FileBatch(com.alibaba.otter.shared.etl.model.FileBatch) RowBatch(com.alibaba.otter.shared.etl.model.RowBatch) TransformException(com.alibaba.otter.node.etl.transform.exception.TransformException) FileData(com.alibaba.otter.shared.etl.model.FileData)

Example 3 with FileBatch

use of com.alibaba.otter.shared.etl.model.FileBatch in project otter by alibaba.

the class FileLoadActionTest method testLoadWithLocal.

@Test
public void testLoadWithLocal() throws IOException {
    // 构造fileData使用的参数,fileDataStartIndex 决定着 pipeline 与 fileData 对应的关系(通过
    // dataMediaPair 的 id),
    // 以及 dataMediaPair 的 pushWeight
    final int fileDataStartIndex = 0;
    final int fileDataCount = 50;
    final Pipeline pipeline = buildPipeline(fileDataStartIndex, fileDataCount);
    final Channel channel = new Channel();
    new NonStrictExpectations() {

        {
            configClientService.findChannel(anyLong);
            returns(channel);
            configClientService.findPipeline(anyLong);
            returns(pipeline);
        }
    };
    Identity id = buildIdentity(1L, 2L, 3L);
    FileBatch fileBatch = buildFileBatch(id);
    fileBatch.getFiles().addAll(buildFileDatas(null, EventType.INSERT, fileDataStartIndex, fileDataCount, true));
    WeightController controller = new WeightController(1);
    FileLoadContext context = fileLoadAction.load(fileBatch, ROOT_DIR, controller);
    want.object(context.getChannel()).isEqualTo(channel);
    want.object(context.getPipeline()).isEqualTo(pipeline);
    want.object(context.getPrepareDatas()).isEqualTo(fileBatch.getFiles());
    want.number(context.getProcessedDatas().size()).isEqualTo(fileBatch.getFiles().size());
}
Also used : FileBatch(com.alibaba.otter.shared.etl.model.FileBatch) Channel(com.alibaba.otter.shared.common.model.config.channel.Channel) FileLoadContext(com.alibaba.otter.node.etl.load.loader.db.context.FileLoadContext) WeightController(com.alibaba.otter.node.etl.load.loader.weight.WeightController) Identity(com.alibaba.otter.shared.etl.model.Identity) Pipeline(com.alibaba.otter.shared.common.model.config.pipeline.Pipeline) Test(org.testng.annotations.Test) BaseDbTest(com.alibaba.otter.node.etl.BaseDbTest)

Example 4 with FileBatch

use of com.alibaba.otter.shared.etl.model.FileBatch in project otter by alibaba.

the class FileLoadActionTest method buildFileBatch.

protected FileBatch buildFileBatch(Identity identity) {
    FileBatch fileBatch = new FileBatch();
    fileBatch.setIdentity(identity);
    return fileBatch;
}
Also used : FileBatch(com.alibaba.otter.shared.etl.model.FileBatch)

Example 5 with FileBatch

use of com.alibaba.otter.shared.etl.model.FileBatch in project otter by alibaba.

the class OtterLoaderFactoryIntegration method test_simple.

@Test
public void test_simple() {
    Identity identity = new Identity();
    identity.setChannelId(100L);
    identity.setPipelineId(100L);
    identity.setProcessId(100L);
    RowBatch rowBatch = new RowBatch();
    rowBatch.setIdentity(identity);
    FileBatch fileBatch = new FileBatch();
    fileBatch.setIdentity(identity);
    final DbBatch dbBatch = new DbBatch();
    dbBatch.setRowBatch(rowBatch);
    dbBatch.setFileBatch(fileBatch);
    final CountDownLatch latch = new CountDownLatch(1);
    executorService.submit(new Runnable() {

        public void run() {
            System.out.println("first run!!!!!!");
            otterLoaderFactory.load(dbBatch);
            latch.countDown();
        }
    });
    try {
        latch.await();
    } catch (InterruptedException e) {
    }
}
Also used : FileBatch(com.alibaba.otter.shared.etl.model.FileBatch) RowBatch(com.alibaba.otter.shared.etl.model.RowBatch) Identity(com.alibaba.otter.shared.etl.model.Identity) CountDownLatch(java.util.concurrent.CountDownLatch) DbBatch(com.alibaba.otter.shared.etl.model.DbBatch) Test(org.testng.annotations.Test) BaseDbTest(com.alibaba.otter.node.etl.BaseDbTest)

Aggregations

FileBatch (com.alibaba.otter.shared.etl.model.FileBatch)18 Pipeline (com.alibaba.otter.shared.common.model.config.pipeline.Pipeline)9 FileData (com.alibaba.otter.shared.etl.model.FileData)9 Identity (com.alibaba.otter.shared.etl.model.Identity)9 File (java.io.File)8 Test (org.testng.annotations.Test)8 RowBatch (com.alibaba.otter.shared.etl.model.RowBatch)7 BaseDbTest (com.alibaba.otter.node.etl.BaseDbTest)5 DbBatch (com.alibaba.otter.shared.etl.model.DbBatch)4 IOException (java.io.IOException)4 BaseOtterTest (com.alibaba.otter.node.etl.BaseOtterTest)3 WeightController (com.alibaba.otter.node.etl.load.loader.weight.WeightController)3 Node (com.alibaba.otter.shared.common.model.config.node.Node)3 EventColumn (com.alibaba.otter.shared.etl.model.EventColumn)3 EventData (com.alibaba.otter.shared.etl.model.EventData)3 ArrayList (java.util.ArrayList)3 PipeException (com.alibaba.otter.node.etl.common.pipe.exception.PipeException)2 HttpPipeKey (com.alibaba.otter.node.etl.common.pipe.impl.http.HttpPipeKey)2 LoadException (com.alibaba.otter.node.etl.load.exception.LoadException)2 FileLoadContext (com.alibaba.otter.node.etl.load.loader.db.context.FileLoadContext)2