Examples with FileInfo com.alibaba.otter.shared.etl.extend.fileresolver.FileInfo used on opensource projects

Example 1 with FileInfo

use of com.alibaba.otter.shared.etl.extend.fileresolver.FileInfo in project otter by alibaba.

the class TestFileResolver method getFileInfo.

public FileInfo[] getFileInfo(Map<String, String> rowMap) {
    // 基本步骤：
    // 1. 获取binlog中的变更字段，比如组成文件有多个字段组成version+path
    // 2. 基于字段内容，构造一个文件路径，目前开源版本只支持本地文件的同步.(如果是网络文件，建议进行NFS mount到ndde机器).
    // 3. 返回FileInfo数组，(目前不支持目录同步，如果是目录需要展开为多个FileInfo的子文件)，如果不需要同步，则返回null.
    //注意为大写
    String path = rowMap.get("FIELD");
    FileInfo fileInfo = null;
    if (StringUtils.isNotEmpty(path)) {
        fileInfo = new FileInfo(path);
        return new FileInfo[] { fileInfo };
    } else {
        return null;
    }
}

Example 2 with FileInfo

use of com.alibaba.otter.shared.etl.extend.fileresolver.FileInfo in project otter by alibaba.

the class FileExtractor method getSingleRowFileInfos.

private List<FileData> getSingleRowFileInfos(long pairId, FileResolver fileResolver, EventData eventData) {
    if (eventData.getEventType() == EventType.DELETE && fileResolver.isDeleteRequired() == false) {
        return new ArrayList<FileData>();
    }
    Map<String, String> rowMap = new HashMap<String, String>();
    List<EventColumn> keyColumns = eventData.getKeys();
    List<EventColumn> eventColumns = eventData.getUpdatedColumns();
    for (EventColumn eventColumn : keyColumns) {
        rowMap.put(eventColumn.getColumnName().toUpperCase(), eventColumn.getColumnValue());
    }
    for (EventColumn eventColumn : eventColumns) {
        rowMap.put(eventColumn.getColumnName().toUpperCase(), eventColumn.getColumnValue());
    }
    FileInfo[] fileInfos = fileResolver.getFileInfo(rowMap);
    if (fileInfos == null || fileInfos.length == 0) {
        return new ArrayList<FileData>();
    } else {
        List<FileData> fileDatas = new ArrayList<FileData>();
        for (FileInfo fileInfo : fileInfos) {
            FileData fileData = new FileData();
            // 记录一下具体映射规则的id
            fileData.setPairId(pairId);
            fileData.setTableId(eventData.getTableId());
            fileData.setEventType(eventData.getEventType());
            fileData.setLastModifiedTime(fileInfo.getLastModifiedTime());
            fileData.setNameSpace(fileInfo.getNamespace());
            fileData.setPath(fileInfo.getPath());
            fileData.setSize(fileInfo.getSize());
            fileDatas.add(fileData);
        }
        return fileDatas;
    }
}