Examples with PipeException com.alibaba.otter.node.etl.common.pipe.exception.PipeException used on opensource projects

Example 1 with PipeException

use of com.alibaba.otter.node.etl.common.pipe.exception.PipeException in project otter by alibaba.

the class RowDataPipeDelegate method get.

public DbBatch get(List<PipeKey> keys) {
    Assert.notNull(keys);
    DbBatch dbBatch = new DbBatch();
    Future<File> future = null;
    for (final PipeKey key : keys) {
        if (key == null) {
            // 忽略空的key
            continue;
        }
        if (key instanceof MemoryPipeKey) {
            dbBatch = rowDataMemoryPipe.get((MemoryPipeKey) key);
            // 直接返回
            return dbBatch;
        } else if (key instanceof HttpPipeKey) {
            if (key.getDataType().isDbBatch()) {
                // 区分一下数据下载
                dbBatch = rowDataHttpPipe.get((HttpPipeKey) key);
            } else {
                future = executorService.submit(new Callable<File>() {

                    public File call() throws Exception {
                        try {
                            HttpPipeKey pipeKey = (HttpPipeKey) key;
                            MDC.put(OtterConstants.splitPipelineLogFileKey, String.valueOf(pipeKey.getIdentity().getPipelineId()));
                            return attachmentHttpPipe.get(pipeKey);
                        } finally {
                            MDC.remove(OtterConstants.splitPipelineLogFileKey);
                        }
                    }
                });
            }
        } else if (key instanceof RpcPipeKey) {
            dbBatch = rowDataRpcPipe.get((RpcPipeKey) key);
        } else {
            throw new PipeException("unknow_PipeKey", key.toString());
        }
    }
    if (future != null && dbBatch != null) {
        try {
            dbBatch.setRoot(future.get());
        } catch (Exception e) {
            throw new PipeException(e);
        }
    }
    return dbBatch;
}

Example 2 with PipeException

use of com.alibaba.otter.node.etl.common.pipe.exception.PipeException in project otter by alibaba.

the class RowDataPipeDelegate method put.

/**
     * 将对应的数据传递到指定的Node id节点上
     */
public List<PipeKey> put(final DbBatch data, Long nid) throws PipeException {
    List<PipeKey> keys = new ArrayList<PipeKey>();
    if (isLocal(nid)) {
        keys.add(rowDataMemoryPipe.put(data));
    } else {
        Future<PipeKey> future = null;
        Pipeline pipeline = configClientService.findPipeline(data.getRowBatch().getIdentity().getPipelineId());
        if (data.getFileBatch() != null && !CollectionUtils.isEmpty(data.getFileBatch().getFiles())) {
            future = executorService.submit(new Callable<PipeKey>() {

                public PipeKey call() throws Exception {
                    try {
                        MDC.put(OtterConstants.splitPipelineLogFileKey, String.valueOf(data.getFileBatch().getIdentity().getPipelineId()));
                        return attachmentHttpPipe.put(data.getFileBatch());
                    } finally {
                        MDC.remove(OtterConstants.splitPipelineLogFileKey);
                    }
                }
            });
        }
        try {
            PipeChooseMode pipeChooseMode = pipeline.getParameters().getPipeChooseType();
            if (pipeChooseMode.isAutomatic()) {
                if (calculateSize(data) <= sizeThresold) {
                    keys.add(rowDataRpcPipe.put(data));
                } else {
                    keys.add(rowDataHttpPipe.put(data));
                }
            } else if (pipeChooseMode.isRpc()) {
                keys.add(rowDataRpcPipe.put(data));
            } else if (pipeChooseMode.isHttp()) {
                keys.add(rowDataHttpPipe.put(data));
            } else {
                throw new PipeException("pipeChooseMode is error!" + pipeChooseMode);
            }
            // 等待一下附件处理
            if (future != null) {
                keys.add(future.get());
            }
        } catch (Exception e) {
            throw new PipeException(e);
        }
    }
    return keys;
}

Example 3 with PipeException

use of com.alibaba.otter.node.etl.common.pipe.exception.PipeException in project otter by alibaba.

the class AbstractHttpPipe method decodeFile.

protected void decodeFile(File file, String key, String crc) {
    // 读取校验信息
    RandomAccessFile raf = null;
    try {
        raf = new RandomAccessFile(file, "rw");
        long totallength = file.length();
        int keyLength = ByteUtils.stringToBytes(key).length;
        int crcLength = ByteUtils.stringToBytes(crc).length;
        // 长度字段起始位
        long pos = totallength - keyLength - crcLength;
        // 游标
        raf.seek(pos);
        // 读取key内容
        byte[] keyBytes = new byte[keyLength];
        raf.read(keyBytes, 0, keyLength);
        String keystr = ByteUtils.bytesToString(keyBytes);
        if (!key.equals(keystr)) {
            throw new ChecksumException("unmatch garble key with[" + key + "],[" + keystr + "]");
        }
        // 读取校验码长度
        raf.seek(pos + keyLength);
        byte[] crcBytes = new byte[crcLength];
        raf.read(crcBytes, 0, crcLength);
        String crcStr = ByteUtils.bytesToString(crcBytes);
        if (!crc.equals(crcStr)) {
            throw new ChecksumException("unmatch crc with[" + crc + "],[" + crcStr + "]");
        }
        // 设置文件长度
        raf.setLength(pos);
    } catch (Exception e) {
        throw new PipeException("read_encrypted_error", e);
    } finally {
        IOUtils.closeQuietly(raf);
    }
}

Example 4 with PipeException

use of com.alibaba.otter.node.etl.common.pipe.exception.PipeException in project otter by alibaba.

the class RowDataHttpPipe method saveDbBatch.

// ======================== help method ===================
// 保存对应的dbBatch
private HttpPipeKey saveDbBatch(DbBatch dbBatch) {
    RowBatch rowBatch = dbBatch.getRowBatch();
    // 转化为proto对象
    BatchProto.RowBatch.Builder rowBatchBuilder = BatchProto.RowBatch.newBuilder();
    rowBatchBuilder.setIdentity(build(rowBatch.getIdentity()));
    // 处理具体的字段rowData
    for (EventData eventData : rowBatch.getDatas()) {
        BatchProto.RowData.Builder rowDataBuilder = BatchProto.RowData.newBuilder();
        rowDataBuilder.setPairId(eventData.getPairId());
        rowDataBuilder.setTableId(eventData.getTableId());
        if (eventData.getSchemaName() != null) {
            rowDataBuilder.setSchemaName(eventData.getSchemaName());
        }
        rowDataBuilder.setTableName(eventData.getTableName());
        rowDataBuilder.setEventType(eventData.getEventType().getValue());
        rowDataBuilder.setExecuteTime(eventData.getExecuteTime());
        // add by ljh at 2012-10-31
        if (eventData.getSyncMode() != null) {
            rowDataBuilder.setSyncMode(eventData.getSyncMode().getValue());
        }
        if (eventData.getSyncConsistency() != null) {
            rowDataBuilder.setSyncConsistency(eventData.getSyncConsistency().getValue());
        }
        // 构造key column
        for (EventColumn keyColumn : eventData.getKeys()) {
            rowDataBuilder.addKeys(buildColumn(keyColumn));
        }
        // 构造old key column
        if (CollectionUtils.isEmpty(eventData.getOldKeys()) == false) {
            for (EventColumn keyColumn : eventData.getOldKeys()) {
                rowDataBuilder.addOldKeys(buildColumn(keyColumn));
            }
        }
        // 构造其他 column
        for (EventColumn column : eventData.getColumns()) {
            rowDataBuilder.addColumns(buildColumn(column));
        }
        rowDataBuilder.setRemedy(eventData.isRemedy());
        rowDataBuilder.setSize(eventData.getSize());
        if (StringUtils.isNotEmpty(eventData.getSql())) {
            rowDataBuilder.setSql(eventData.getSql());
        }
        if (StringUtils.isNotEmpty(eventData.getDdlSchemaName())) {
            rowDataBuilder.setDdlSchemaName(eventData.getDdlSchemaName());
        }
        if (StringUtils.isNotEmpty(eventData.getHint())) {
            rowDataBuilder.setHint(eventData.getHint());
        }
        rowDataBuilder.setWithoutSchema(eventData.isWithoutSchema());
        // 添加一条rowData记录
        rowBatchBuilder.addRows(rowDataBuilder.build());
    }
    // 处理下FileBatch
    FileBatch fileBatch = dbBatch.getFileBatch();
    BatchProto.FileBatch.Builder fileBatchBuilder = null;
    fileBatchBuilder = BatchProto.FileBatch.newBuilder();
    fileBatchBuilder.setIdentity(build(fileBatch.getIdentity()));
    // 构造对应的proto对象
    for (FileData fileData : fileBatch.getFiles()) {
        BatchProto.FileData.Builder fileDataBuilder = BatchProto.FileData.newBuilder();
        fileDataBuilder.setPairId(fileData.getPairId());
        fileDataBuilder.setTableId(fileData.getTableId());
        if (fileData.getNameSpace() != null) {
            fileDataBuilder.setNamespace(fileData.getNameSpace());
        }
        if (fileData.getPath() != null) {
            fileDataBuilder.setPath(fileData.getPath());
        }
        fileDataBuilder.setEventType(fileData.getEventType().getValue());
        fileDataBuilder.setSize(fileData.getSize());
        fileDataBuilder.setLastModifiedTime(fileData.getLastModifiedTime());
        // 添加一条fileData记录
        fileBatchBuilder.addFiles(fileDataBuilder.build());
    }
    // 处理构造对应的文件url
    String filename = buildFileName(rowBatch.getIdentity(), ClassUtils.getShortClassName(dbBatch.getClass()));
    // 写入数据
    File file = new File(htdocsDir, filename);
    OutputStream output = null;
    try {
        output = new BufferedOutputStream(new FileOutputStream(file));
        com.alibaba.otter.node.etl.model.protobuf.BatchProto.RowBatch rowBatchProto = rowBatchBuilder.build();
        // 输出大小
        output.write(ByteUtils.int2bytes(rowBatchProto.getSerializedSize()));
        // 输出row batch
        rowBatchProto.writeTo(output);
        com.alibaba.otter.node.etl.model.protobuf.BatchProto.FileBatch fileBatchProto = fileBatchBuilder.build();
        // 输出大小
        output.write(ByteUtils.int2bytes(fileBatchProto.getSerializedSize()));
        // 输出file batch
        fileBatchProto.writeTo(output);
        output.flush();
    } catch (IOException e) {
        throw new PipeException("write_byte_error", e);
    } finally {
        IOUtils.closeQuietly(output);
    }
    HttpPipeKey key = new HttpPipeKey();
    key.setUrl(remoteUrlBuilder.getUrl(rowBatch.getIdentity().getPipelineId(), filename));
    key.setDataType(PipeDataType.DB_BATCH);
    key.setIdentity(rowBatch.getIdentity());
    Pipeline pipeline = configClientService.findPipeline(rowBatch.getIdentity().getPipelineId());
    if (pipeline.getParameters().getUseFileEncrypt()) {
        // 加密处理
        EncryptedData encryptedData = encryptFile(file);
        key.setKey(encryptedData.getKey());
        key.setCrc(encryptedData.getCrc());
    }
    return key;
}

Example 5 with PipeException

use of com.alibaba.otter.node.etl.common.pipe.exception.PipeException in project otter by alibaba.

the class AbstractHttpPipe method encryptFile.

protected EncryptedData encryptFile(File file) {
    // 构造校验对象，这里考虑性能只将file path做为加密源
    EncryptedData encryptedData = null;
    try {
        encryptedData = EncryptUtils.encrypt(file.getPath().getBytes("UTF-8"));
    } catch (UnsupportedEncodingException e) {
    // ignore
    }
    // 写入校验信息
    RandomAccessFile raf = null;
    try {
        raf = new RandomAccessFile(file, "rw");
        long origLength = file.length();
        int keyLength = ByteUtils.stringToBytes(encryptedData.getKey()).length;
        int crcLength = ByteUtils.stringToBytes(encryptedData.getCrc()).length;
        long totalLength = origLength + crcLength + keyLength;
        raf.setLength(totalLength);
        raf.seek(origLength);
        raf.write(ByteUtils.stringToBytes(encryptedData.getKey()), 0, keyLength);
        raf.seek(origLength + keyLength);
        raf.write(ByteUtils.stringToBytes(encryptedData.getCrc()), 0, crcLength);
    } catch (Exception e) {
        throw new PipeException("write_encrypted_error", e);
    } finally {
        IOUtils.closeQuietly(raf);
    }
    return encryptedData;
}