Examples with FileData com.alibaba.otter.shared.etl.model.FileData used on opensource projects

Example 16 with FileData

use of com.alibaba.otter.shared.etl.model.FileData in project otter by alibaba.

the class RowDataHttpPipe method getDbBatch.

// 处理对应的dbBatch
private DbBatch getDbBatch(HttpPipeKey key) {
    String dataUrl = key.getUrl();
    Pipeline pipeline = configClientService.findPipeline(key.getIdentity().getPipelineId());
    DataRetriever dataRetriever = dataRetrieverFactory.createRetriever(pipeline.getParameters().getRetriever(), dataUrl, downloadDir);
    File archiveFile = null;
    try {
        dataRetriever.connect();
        dataRetriever.doRetrieve();
        archiveFile = dataRetriever.getDataAsFile();
    } catch (Exception e) {
        dataRetriever.abort();
        throw new PipeException("download_error", e);
    } finally {
        dataRetriever.disconnect();
    }
    // 处理下有加密的数据
    if (StringUtils.isNotEmpty(key.getKey()) && StringUtils.isNotEmpty(key.getCrc())) {
        decodeFile(archiveFile, key.getKey(), key.getCrc());
    }
    InputStream input = null;
    JSONReader reader = null;
    try {
        input = new BufferedInputStream(new FileInputStream(archiveFile));
        DbBatch dbBatch = new DbBatch();
        byte[] lengthBytes = new byte[4];
        input.read(lengthBytes);
        int length = ByteUtils.bytes2int(lengthBytes);
        BatchProto.RowBatch rowbatchProto = BatchProto.RowBatch.parseFrom(new LimitedInputStream(input, length));
        // 构造原始的model对象
        RowBatch rowBatch = new RowBatch();
        rowBatch.setIdentity(build(rowbatchProto.getIdentity()));
        for (BatchProto.RowData rowDataProto : rowbatchProto.getRowsList()) {
            EventData eventData = new EventData();
            eventData.setPairId(rowDataProto.getPairId());
            eventData.setTableId(rowDataProto.getTableId());
            eventData.setTableName(rowDataProto.getTableName());
            eventData.setSchemaName(rowDataProto.getSchemaName());
            eventData.setEventType(EventType.valuesOf(rowDataProto.getEventType()));
            eventData.setExecuteTime(rowDataProto.getExecuteTime());
            // add by ljh at 2012-10-31
            if (StringUtils.isNotEmpty(rowDataProto.getSyncMode())) {
                eventData.setSyncMode(SyncMode.valuesOf(rowDataProto.getSyncMode()));
            }
            if (StringUtils.isNotEmpty(rowDataProto.getSyncConsistency())) {
                eventData.setSyncConsistency(SyncConsistency.valuesOf(rowDataProto.getSyncConsistency()));
            }
            // 处理主键
            List<EventColumn> keys = new ArrayList<EventColumn>();
            for (BatchProto.Column columnProto : rowDataProto.getKeysList()) {
                keys.add(buildColumn(columnProto));
            }
            eventData.setKeys(keys);
            // 处理old主键
            if (CollectionUtils.isEmpty(rowDataProto.getOldKeysList()) == false) {
                List<EventColumn> oldKeys = new ArrayList<EventColumn>();
                for (BatchProto.Column columnProto : rowDataProto.getOldKeysList()) {
                    oldKeys.add(buildColumn(columnProto));
                }
                eventData.setOldKeys(oldKeys);
            }
            // 处理具体的column value
            List<EventColumn> columns = new ArrayList<EventColumn>();
            for (BatchProto.Column columnProto : rowDataProto.getColumnsList()) {
                columns.add(buildColumn(columnProto));
            }
            eventData.setColumns(columns);
            eventData.setRemedy(rowDataProto.getRemedy());
            eventData.setSize(rowDataProto.getSize());
            eventData.setSql(rowDataProto.getSql());
            eventData.setDdlSchemaName(rowDataProto.getDdlSchemaName());
            eventData.setHint(rowDataProto.getHint());
            eventData.setWithoutSchema(rowDataProto.getWithoutSchema());
            // 添加到总记录
            rowBatch.merge(eventData);
        }
        dbBatch.setRowBatch(rowBatch);
        input.read(lengthBytes);
        length = ByteUtils.bytes2int(lengthBytes);
        BatchProto.FileBatch filebatchProto = BatchProto.FileBatch.parseFrom(new LimitedInputStream(input, length));
        // 构造原始的model对象
        FileBatch fileBatch = new FileBatch();
        fileBatch.setIdentity(build(filebatchProto.getIdentity()));
        for (BatchProto.FileData fileDataProto : filebatchProto.getFilesList()) {
            FileData fileData = new FileData();
            fileData.setPairId(fileDataProto.getPairId());
            fileData.setTableId(fileDataProto.getTableId());
            fileData.setEventType(EventType.valuesOf(fileDataProto.getEventType()));
            fileData.setLastModifiedTime(fileDataProto.getLastModifiedTime());
            fileData.setNameSpace(fileDataProto.getNamespace());
            fileData.setPath(fileDataProto.getPath());
            fileData.setSize(fileDataProto.getSize());
            // 添加到filebatch中
            fileBatch.getFiles().add(fileData);
        }
        dbBatch.setFileBatch(fileBatch);
        return dbBatch;
    } catch (IOException e) {
        throw new PipeException("deserial_error", e);
    } finally {
        IOUtils.closeQuietly(reader);
    }
}

Example 17 with FileData

use of com.alibaba.otter.shared.etl.model.FileData in project otter by alibaba.

the class RowDataMemoryPipe method prepareFile.

// 处理对应的附件
@SuppressWarnings("unused")
private File prepareFile(FileBatch fileBatch) {
    // 处理构造对应的文件url
    String dirname = buildFileName(fileBatch.getIdentity(), ClassUtils.getShortClassName(fileBatch.getClass()));
    File dir = new File(downloadDir, dirname);
    // 创建父目录
    NioUtils.create(dir, false, 3);
    // 压缩对应的文件数据
    List<FileData> fileDatas = fileBatch.getFiles();
    for (FileData fileData : fileDatas) {
        String namespace = fileData.getNameSpace();
        String path = fileData.getPath();
        boolean isLocal = StringUtils.isBlank(namespace);
        String entryName = null;
        if (true == isLocal) {
            entryName = FilenameUtils.getPath(path) + FilenameUtils.getName(path);
        } else {
            entryName = namespace + File.separator + path;
        }
        InputStream input = retrive(fileBatch.getIdentity(), fileData);
        if (input == null) {
            continue;
        }
        File entry = new File(dir, entryName);
        // 尝试创建父路径
        NioUtils.create(entry.getParentFile(), false, retry);
        FileOutputStream output = null;
        try {
            output = new FileOutputStream(entry);
            // 输出到压缩流中
            NioUtils.copy(input, output);
        } catch (Exception e) {
            throw new PipeException("prepareFile error for file[" + entry.getPath() + "]");
        } finally {
            IOUtils.closeQuietly(output);
        }
    }
    return dir;
}

Example 18 with FileData

use of com.alibaba.otter.shared.etl.model.FileData in project otter by alibaba.

the class FileLoadAction method dryRun.

private void dryRun(FileLoadContext context, List<FileData> fileDatas, File rootDir) {
    for (FileData fileData : fileDatas) {
        boolean isLocal = StringUtils.isBlank(fileData.getNameSpace());
        String entryName = null;
        if (true == isLocal) {
            entryName = FilenameUtils.getPath(fileData.getPath()) + FilenameUtils.getName(fileData.getPath());
        } else {
            entryName = fileData.getNameSpace() + File.separator + fileData.getPath();
        }
        File sourceFile = new File(rootDir, entryName);
        if (true == sourceFile.exists() && false == sourceFile.isDirectory()) {
            if (false == isLocal) {
                throw new LoadException(fileData + " is not support!");
            } else {
                // 记录一下文件的meta信息
                fileData.setSize(sourceFile.length());
                fileData.setLastModifiedTime(sourceFile.lastModified());
                context.getProcessedDatas().add(fileData);
            }
            LoadCounter counter = loadStatsTracker.getStat(context.getIdentity()).getStat(fileData.getPairId());
            counter.getFileCount().incrementAndGet();
            counter.getFileSize().addAndGet(fileData.getSize());
        } else if (fileData.getEventType().isDelete()) {
            // 删除对应的文件
            if (false == isLocal) {
                throw new LoadException(fileData + " is not support!");
            } else {
                context.getProcessedDatas().add(fileData);
            }
        } else {
            // 失败记录
            context.getFailedDatas().add(fileData);
        }
    }
}

Example 19 with FileData

use of com.alibaba.otter.shared.etl.model.FileData in project otter by alibaba.

the class LocalFileLoaderActionTest method generatorLocalFileData.

private List<FileData> generatorLocalFileData(String prefix, int count) {
    List<FileData> result = new ArrayList<FileData>();
    String target = tmp + File.separator + OTTERLOAD + "_loaded/";
    for (int i = 0; i < count; i++) {
        String filepath = tmp + File.separator + OTTERLOAD + target;
        File local = new File(filepath, prefix + "_" + i + ".jpg");
        FileData localFileData = new FileData();
        localFileData.setPairId(i);
        localFileData.setTableId(i);
        localFileData.setPath(target + local.getName());
        localFileData.setLastModifiedTime(new Date().getTime());
        try {
            byte[] data = getBlock((i + 1) * 1024);
            localFileData.setSize(data.length);
            NioUtils.write(data, local);
        } catch (IOException e) {
            want.fail();
        }
        result.add(localFileData);
    }
    return result;
}

Example 20 with FileData

use of com.alibaba.otter.shared.etl.model.FileData in project otter by alibaba.

the class HttpPipeIntegration method test_attachment.

@Test
public void test_attachment() {
    final Node currentNode = new Node();
    currentNode.setId(1L);
    currentNode.setIp("127.0.0.1");
    currentNode.setParameters(new NodeParameter());
    final Pipeline pipeline = new Pipeline();
    pipeline.getParameters().setRetriever(RetrieverType.ARIA2C);
    // mock一下
    new NonStrictExpectations() {

        {
            configClientService.currentNode();
            returns(currentNode);
            configClientService.findPipeline(anyLong);
            returns(pipeline);
        }
    };
    Identity identity = new Identity();
    identity.setChannelId(100L);
    identity.setPipelineId(100L);
    identity.setProcessId(100L);
    FileBatch fileBatch = new FileBatch();
    fileBatch.setIdentity(identity);
    File localFile = new File(tmp, "httpPipeTest.jpg");
    FileData localFileData = new FileData();
    localFileData.setEventType(EventType.INSERT);
    localFileData.setPath(localFile.getPath());
    fileBatch.getFiles().add(localFileData);
    try {
        byte[] data = getBlock(10 * 1024);
        NioUtils.write(data, localFile);
        HttpPipeKey key = attachmentHttpPipe.put(fileBatch);
        File target = attachmentHttpPipe.get(key);
        byte[] getbytes = NioUtils.read(new File(target, localFile.getPath()));
        check(data, getbytes);
    } catch (IOException e) {
        want.fail();
    } finally {
        NioUtils.delete(localFile);
    }
}