Examples with FileData com.alibaba.otter.shared.etl.model.FileData used on opensource projects

Example 1 with FileData

use of com.alibaba.otter.shared.etl.model.FileData in project otter by alibaba.

the class OtterTransformerTest method test_fileData.

@Test
public void test_fileData() {
    final Pipeline pipeline = new Pipeline();
    pipeline.setId(100L);
    List<DataMediaPair> pairs = new ArrayList<DataMediaPair>();
    DataMediaPair pair1 = new DataMediaPair();
    pair1.setId(1L);
    pair1.setPipelineId(pipeline.getId());
    pair1.setPullWeight(1L);
    pair1.setPushWeight(1L);
    DbDataMedia oracleMedia = getOracleMedia();
    oracleMedia.setId(1L);
    pair1.setSource(oracleMedia);
    DbDataMedia mysqlMedia = getMysqlMedia();
    pair1.setTarget(mysqlMedia);
    pairs.add(pair1);
    pipeline.setPairs(pairs);
    new NonStrictExpectations() {

        {
            configClientService.findPipeline(anyLong);
            returns(pipeline);
        }
    };
    Identity identity = new Identity();
    identity.setChannelId(100L);
    identity.setPipelineId(100L);
    identity.setProcessId(100L);
    FileBatch fileBatch = new FileBatch();
    fileBatch.setIdentity(identity);
    File localFile = new File("/tmp", "httpPipeTest.jpg");
    FileData localFileData = new FileData();
    localFileData.setTableId(1L);
    localFileData.setPairId(1L);
    localFileData.setPath(localFile.getPath());
    fileBatch.getFiles().add(localFileData);
    Map<Class, BatchObject> batchs = otterTransformFactory.transform(fileBatch);
    FileBatch result = (FileBatch) batchs.get(FileData.class);
    want.number(result.getFiles().size()).isEqualTo(1);
}

Example 2 with FileData

use of com.alibaba.otter.shared.etl.model.FileData in project otter by alibaba.

the class FileBatchConflictDetectServiceIntegration method generatorLocalFileData.

private List<FileData> generatorLocalFileData(String prefix, int count) {
    List<FileData> result = new ArrayList<FileData>();
    for (int i = 0; i < count; i++) {
        String filepath = tmp + File.separator + OTTERLOAD + File.separator;
        File local = new File(filepath, prefix + "_" + i + ".jpg");
        FileData localFileData = new FileData();
        localFileData.setEventType(EventType.UPDATE);
        localFileData.setPairId(i);
        localFileData.setTableId(i);
        localFileData.setNameSpace(null);
        localFileData.setPath(local.getPath());
        try {
            byte[] data = getBlock((i + 1) * 1024);
            localFileData.setSize(data.length);
            NioUtils.write(data, local);
            localFileData.setLastModifiedTime(local.lastModified());
        } catch (IOException e) {
            want.fail();
        }
        result.add(localFileData);
    }
    return result;
}

Example 3 with FileData

use of com.alibaba.otter.shared.etl.model.FileData in project otter by alibaba.

the class FileLoadActionTest method buildFileDatas.

protected List<FileData> buildFileDatas(String namespace, EventType eventType, int start, int count, boolean create) throws IOException {
    List<FileData> files = new ArrayList<FileData>();
    for (int i = start; i < count; i++) {
        FileData fileData = new FileData();
        // namespace is null means file is
        fileData.setNameSpace(namespace);
        // local file
        fileData.setEventType(eventType);
        fileData.setPairId(i % NUMBER_OF_FILE_DATA_COPIES);
        fileData.setPath(ROOT_DIR.getAbsolutePath() + "/target/" + eventType.getValue() + i);
        String parentPath = ROOT_DIR.getPath();
        if (namespace != null) {
            parentPath = parentPath + "/" + namespace;
        }
        File file = new File(parentPath, fileData.getPath());
        if (!file.exists() && create) {
            FileUtils.touch(file);
        }
        fileData.setSize(file.exists() ? file.length() : 0);
        fileData.setLastModifiedTime(file.exists() ? file.lastModified() : Calendar.getInstance().getTimeInMillis());
        fileData.setTableId(TABLE_ID);
        files.add(fileData);
    }
    return files;
}

Example 4 with FileData

use of com.alibaba.otter.shared.etl.model.FileData in project otter by alibaba.

the class ArchiveBeanIntegration method test_pack.

// @Test
public void test_pack() {
    ArchiveBean archiveBean = new ArchiveBean();
    try {
        archiveBean.afterPropertiesSet();
        archiveBean.setUseLocalFileMutliThread(false);
    } catch (Exception e1) {
        want.fail();
    }
    File file = new File("/tmp/otter/test");
    Collection<File> allFiles = FileUtils.listFiles(file, new String[] { "jpg" }, true);
    List<FileData> fileDatas = new ArrayList<FileData>();
    for (File files : allFiles) {
        FileData data = new FileData();
        // data.setPath("wsproduct_repository/product_sku/76/84/32/84/768432847_10.summ.jpg");
        data.setPath(StringUtils.substringAfter(files.getAbsolutePath(), "/tmp/otter/test"));
        fileDatas.add(data);
    }
    File archiveFile = new File("/tmp/otter/test.gzip");
    if (archiveFile.exists()) {
        archiveFile.delete();
    }
    boolean result = archiveBean.pack(archiveFile, fileDatas, new ArchiveRetriverCallback<FileData>() {

        public InputStream retrive(FileData source) {
            return new LazyFileInputStream(new File("/tmp/otter/test", source.getPath()));
        }
    });
    if (!result) {
        want.fail();
    }
}

Example 5 with FileData

use of com.alibaba.otter.shared.etl.model.FileData in project otter by alibaba.

the class RowDataHttpPipe method saveDbBatch.

// ======================== help method ===================
// 保存对应的dbBatch
private HttpPipeKey saveDbBatch(DbBatch dbBatch) {
    RowBatch rowBatch = dbBatch.getRowBatch();
    // 转化为proto对象
    BatchProto.RowBatch.Builder rowBatchBuilder = BatchProto.RowBatch.newBuilder();
    rowBatchBuilder.setIdentity(build(rowBatch.getIdentity()));
    // 处理具体的字段rowData
    for (EventData eventData : rowBatch.getDatas()) {
        BatchProto.RowData.Builder rowDataBuilder = BatchProto.RowData.newBuilder();
        rowDataBuilder.setPairId(eventData.getPairId());
        rowDataBuilder.setTableId(eventData.getTableId());
        if (eventData.getSchemaName() != null) {
            rowDataBuilder.setSchemaName(eventData.getSchemaName());
        }
        rowDataBuilder.setTableName(eventData.getTableName());
        rowDataBuilder.setEventType(eventData.getEventType().getValue());
        rowDataBuilder.setExecuteTime(eventData.getExecuteTime());
        // add by ljh at 2012-10-31
        if (eventData.getSyncMode() != null) {
            rowDataBuilder.setSyncMode(eventData.getSyncMode().getValue());
        }
        if (eventData.getSyncConsistency() != null) {
            rowDataBuilder.setSyncConsistency(eventData.getSyncConsistency().getValue());
        }
        // 构造key column
        for (EventColumn keyColumn : eventData.getKeys()) {
            rowDataBuilder.addKeys(buildColumn(keyColumn));
        }
        // 构造old key column
        if (CollectionUtils.isEmpty(eventData.getOldKeys()) == false) {
            for (EventColumn keyColumn : eventData.getOldKeys()) {
                rowDataBuilder.addOldKeys(buildColumn(keyColumn));
            }
        }
        // 构造其他 column
        for (EventColumn column : eventData.getColumns()) {
            rowDataBuilder.addColumns(buildColumn(column));
        }
        rowDataBuilder.setRemedy(eventData.isRemedy());
        rowDataBuilder.setSize(eventData.getSize());
        if (StringUtils.isNotEmpty(eventData.getSql())) {
            rowDataBuilder.setSql(eventData.getSql());
        }
        if (StringUtils.isNotEmpty(eventData.getDdlSchemaName())) {
            rowDataBuilder.setDdlSchemaName(eventData.getDdlSchemaName());
        }
        if (StringUtils.isNotEmpty(eventData.getHint())) {
            rowDataBuilder.setHint(eventData.getHint());
        }
        rowDataBuilder.setWithoutSchema(eventData.isWithoutSchema());
        // 添加一条rowData记录
        rowBatchBuilder.addRows(rowDataBuilder.build());
    }
    // 处理下FileBatch
    FileBatch fileBatch = dbBatch.getFileBatch();
    BatchProto.FileBatch.Builder fileBatchBuilder = null;
    fileBatchBuilder = BatchProto.FileBatch.newBuilder();
    fileBatchBuilder.setIdentity(build(fileBatch.getIdentity()));
    // 构造对应的proto对象
    for (FileData fileData : fileBatch.getFiles()) {
        BatchProto.FileData.Builder fileDataBuilder = BatchProto.FileData.newBuilder();
        fileDataBuilder.setPairId(fileData.getPairId());
        fileDataBuilder.setTableId(fileData.getTableId());
        if (fileData.getNameSpace() != null) {
            fileDataBuilder.setNamespace(fileData.getNameSpace());
        }
        if (fileData.getPath() != null) {
            fileDataBuilder.setPath(fileData.getPath());
        }
        fileDataBuilder.setEventType(fileData.getEventType().getValue());
        fileDataBuilder.setSize(fileData.getSize());
        fileDataBuilder.setLastModifiedTime(fileData.getLastModifiedTime());
        // 添加一条fileData记录
        fileBatchBuilder.addFiles(fileDataBuilder.build());
    }
    // 处理构造对应的文件url
    String filename = buildFileName(rowBatch.getIdentity(), ClassUtils.getShortClassName(dbBatch.getClass()));
    // 写入数据
    File file = new File(htdocsDir, filename);
    OutputStream output = null;
    try {
        output = new BufferedOutputStream(new FileOutputStream(file));
        com.alibaba.otter.node.etl.model.protobuf.BatchProto.RowBatch rowBatchProto = rowBatchBuilder.build();
        // 输出大小
        output.write(ByteUtils.int2bytes(rowBatchProto.getSerializedSize()));
        // 输出row batch
        rowBatchProto.writeTo(output);
        com.alibaba.otter.node.etl.model.protobuf.BatchProto.FileBatch fileBatchProto = fileBatchBuilder.build();
        // 输出大小
        output.write(ByteUtils.int2bytes(fileBatchProto.getSerializedSize()));
        // 输出file batch
        fileBatchProto.writeTo(output);
        output.flush();
    } catch (IOException e) {
        throw new PipeException("write_byte_error", e);
    } finally {
        IOUtils.closeQuietly(output);
    }
    HttpPipeKey key = new HttpPipeKey();
    key.setUrl(remoteUrlBuilder.getUrl(rowBatch.getIdentity().getPipelineId(), filename));
    key.setDataType(PipeDataType.DB_BATCH);
    key.setIdentity(rowBatch.getIdentity());
    Pipeline pipeline = configClientService.findPipeline(rowBatch.getIdentity().getPipelineId());
    if (pipeline.getParameters().getUseFileEncrypt()) {
        // 加密处理
        EncryptedData encryptedData = encryptFile(file);
        key.setKey(encryptedData.getKey());
        key.setCrc(encryptedData.getCrc());
    }
    return key;
}