Examples with EventColumn com.alibaba.otter.shared.etl.model.EventColumn used on opensource projects

Example 16 with EventColumn

use of com.alibaba.otter.shared.etl.model.EventColumn in project otter by alibaba.

the class DatabaseExtractorTest method getPrimary.

private List<EventColumn> getPrimary(int value) {
    EventColumn pk = new EventColumn();
    pk.setColumnName("id");
    pk.setColumnType(java.sql.Types.INTEGER);
    pk.setColumnValue(String.valueOf(value));
    pk.setIndex(1);
    pk.setNull(false);
    return Arrays.asList(pk);
}

Example 17 with EventColumn

use of com.alibaba.otter.shared.etl.model.EventColumn in project otter by alibaba.

the class FreedomExtractorTest method getPrimary.

private List<EventColumn> getPrimary(int value) {
    EventColumn pk = new EventColumn();
    pk.setColumnName("id");
    pk.setColumnType(java.sql.Types.INTEGER);
    pk.setColumnValue(String.valueOf(value));
    pk.setIndex(1);
    pk.setNull(false);
    return Arrays.asList(pk);
}

Example 18 with EventColumn

use of com.alibaba.otter.shared.etl.model.EventColumn in project otter by alibaba.

the class RowDataTransformer method translatePkColumn.

private void translatePkColumn(EventData data, List<EventColumn> pks, List<EventColumn> oldPks, List<EventColumn> columns, DataMediaPair dataMediaPair, Multimap<String, String> translateColumnNames, TableInfoHolder tableHolder) {
    if (CollectionUtils.isEmpty(oldPks)) {
        // 如果不存在主键变更
        List<EventColumn> tpks = new ArrayList<EventColumn>();
        for (EventColumn scolumn : pks) {
            EventColumn tcolumn = translateColumn(data, scolumn, tableHolder, dataMediaPair, translateColumnNames);
            if (tcolumn != null) {
                tpks.add(tcolumn);
            }
        }
        data.setKeys(tpks);
    } else {
        // 存在主键变更
        // modify by ljh at 2012-11-07 , 只做view视图映射的转化，不再做update table xxx
        // set pk = newPK where pk = oldPk的处理
        List<EventColumn> tnewPks = new ArrayList<EventColumn>();
        List<EventColumn> toldPks = new ArrayList<EventColumn>();
        for (int i = 0; i < pks.size(); i++) {
            EventColumn newPk = pks.get(i);
            EventColumn oldPk = oldPks.get(i);
            // 转化new pk
            EventColumn tnewPk = translateColumn(data, newPk, tableHolder, dataMediaPair, translateColumnNames);
            if (tnewPk != null) {
                tnewPks.add(tnewPk);
                // 转化old pk，这里不能再用translateColumnNames了，因为转化new
                // pk已经remove过一次view name了
                toldPks.add(translateColumn(tnewPk, oldPk.getColumnValue(), dataMediaPair));
            }
        }
        data.setKeys(tnewPks);
        data.setOldKeys(toldPks);
    // 主键变更构建的sql规则如下：
    // update table xxx set pk = newPK where pk = oldPk;
    // for (int i = 0; i < pks.size(); i++) {
    // EventColumn scolumn = pks.get(i);
    // EventColumn oldPk = oldPks.get(i);
    //
    // EventColumn updatePk = translateColumn(scolumn, tableHolder,
    // dataMediaPair, translateColumnNames);
    // if (scolumn.getColumnValue().equals(oldPk.getColumnValue())) {//
    // 主键内容没变更
    // tcolumns.add(updatePk);
    // } else {
    // columns.add(updatePk);// 添加到变更字段中, 设置set pk = newPK的内容
    // // 设置where pk = oldPk的条件
    // tcolumns.add(translateColumn(updatePk, oldPk.getColumnValue(),
    // dataMediaPair));
    // }
    // }
    }
}

Example 19 with EventColumn

use of com.alibaba.otter.shared.etl.model.EventColumn in project otter by alibaba.

the class RowDataHttpPipe method saveDbBatch.

// ======================== help method ===================
// 保存对应的dbBatch
private HttpPipeKey saveDbBatch(DbBatch dbBatch) {
    RowBatch rowBatch = dbBatch.getRowBatch();
    // 转化为proto对象
    BatchProto.RowBatch.Builder rowBatchBuilder = BatchProto.RowBatch.newBuilder();
    rowBatchBuilder.setIdentity(build(rowBatch.getIdentity()));
    // 处理具体的字段rowData
    for (EventData eventData : rowBatch.getDatas()) {
        BatchProto.RowData.Builder rowDataBuilder = BatchProto.RowData.newBuilder();
        rowDataBuilder.setPairId(eventData.getPairId());
        rowDataBuilder.setTableId(eventData.getTableId());
        if (eventData.getSchemaName() != null) {
            rowDataBuilder.setSchemaName(eventData.getSchemaName());
        }
        rowDataBuilder.setTableName(eventData.getTableName());
        rowDataBuilder.setEventType(eventData.getEventType().getValue());
        rowDataBuilder.setExecuteTime(eventData.getExecuteTime());
        // add by ljh at 2012-10-31
        if (eventData.getSyncMode() != null) {
            rowDataBuilder.setSyncMode(eventData.getSyncMode().getValue());
        }
        if (eventData.getSyncConsistency() != null) {
            rowDataBuilder.setSyncConsistency(eventData.getSyncConsistency().getValue());
        }
        // 构造key column
        for (EventColumn keyColumn : eventData.getKeys()) {
            rowDataBuilder.addKeys(buildColumn(keyColumn));
        }
        // 构造old key column
        if (CollectionUtils.isEmpty(eventData.getOldKeys()) == false) {
            for (EventColumn keyColumn : eventData.getOldKeys()) {
                rowDataBuilder.addOldKeys(buildColumn(keyColumn));
            }
        }
        // 构造其他 column
        for (EventColumn column : eventData.getColumns()) {
            rowDataBuilder.addColumns(buildColumn(column));
        }
        rowDataBuilder.setRemedy(eventData.isRemedy());
        rowDataBuilder.setSize(eventData.getSize());
        if (StringUtils.isNotEmpty(eventData.getSql())) {
            rowDataBuilder.setSql(eventData.getSql());
        }
        if (StringUtils.isNotEmpty(eventData.getDdlSchemaName())) {
            rowDataBuilder.setDdlSchemaName(eventData.getDdlSchemaName());
        }
        if (StringUtils.isNotEmpty(eventData.getHint())) {
            rowDataBuilder.setHint(eventData.getHint());
        }
        rowDataBuilder.setWithoutSchema(eventData.isWithoutSchema());
        // 添加一条rowData记录
        rowBatchBuilder.addRows(rowDataBuilder.build());
    }
    // 处理下FileBatch
    FileBatch fileBatch = dbBatch.getFileBatch();
    BatchProto.FileBatch.Builder fileBatchBuilder = null;
    fileBatchBuilder = BatchProto.FileBatch.newBuilder();
    fileBatchBuilder.setIdentity(build(fileBatch.getIdentity()));
    // 构造对应的proto对象
    for (FileData fileData : fileBatch.getFiles()) {
        BatchProto.FileData.Builder fileDataBuilder = BatchProto.FileData.newBuilder();
        fileDataBuilder.setPairId(fileData.getPairId());
        fileDataBuilder.setTableId(fileData.getTableId());
        if (fileData.getNameSpace() != null) {
            fileDataBuilder.setNamespace(fileData.getNameSpace());
        }
        if (fileData.getPath() != null) {
            fileDataBuilder.setPath(fileData.getPath());
        }
        fileDataBuilder.setEventType(fileData.getEventType().getValue());
        fileDataBuilder.setSize(fileData.getSize());
        fileDataBuilder.setLastModifiedTime(fileData.getLastModifiedTime());
        // 添加一条fileData记录
        fileBatchBuilder.addFiles(fileDataBuilder.build());
    }
    // 处理构造对应的文件url
    String filename = buildFileName(rowBatch.getIdentity(), ClassUtils.getShortClassName(dbBatch.getClass()));
    // 写入数据
    File file = new File(htdocsDir, filename);
    OutputStream output = null;
    try {
        output = new BufferedOutputStream(new FileOutputStream(file));
        com.alibaba.otter.node.etl.model.protobuf.BatchProto.RowBatch rowBatchProto = rowBatchBuilder.build();
        // 输出大小
        output.write(ByteUtils.int2bytes(rowBatchProto.getSerializedSize()));
        // 输出row batch
        rowBatchProto.writeTo(output);
        com.alibaba.otter.node.etl.model.protobuf.BatchProto.FileBatch fileBatchProto = fileBatchBuilder.build();
        // 输出大小
        output.write(ByteUtils.int2bytes(fileBatchProto.getSerializedSize()));
        // 输出file batch
        fileBatchProto.writeTo(output);
        output.flush();
    } catch (IOException e) {
        throw new PipeException("write_byte_error", e);
    } finally {
        IOUtils.closeQuietly(output);
    }
    HttpPipeKey key = new HttpPipeKey();
    key.setUrl(remoteUrlBuilder.getUrl(rowBatch.getIdentity().getPipelineId(), filename));
    key.setDataType(PipeDataType.DB_BATCH);
    key.setIdentity(rowBatch.getIdentity());
    Pipeline pipeline = configClientService.findPipeline(rowBatch.getIdentity().getPipelineId());
    if (pipeline.getParameters().getUseFileEncrypt()) {
        // 加密处理
        EncryptedData encryptedData = encryptFile(file);
        key.setKey(encryptedData.getKey());
        key.setCrc(encryptedData.getCrc());
    }
    return key;
}

Example 20 with EventColumn

use of com.alibaba.otter.shared.etl.model.EventColumn in project otter by alibaba.

the class ViewExtractor method columnFilter.

private List<EventColumn> columnFilter(List<EventColumn> eventColumns, List<ColumnPair> columnPairs, ColumnPairMode mode) {
    if (mode == null) {
        mode = ColumnPairMode.INCLUDE;
    }
    List<EventColumn> tempColumns = new ArrayList<EventColumn>();
    Map<String, ColumnPair> viewNames = new HashMap<String, ColumnPair>();
    for (ColumnPair columnPair : columnPairs) {
        viewNames.put(StringUtils.lowerCase(columnPair.getSourceColumn().getName()), columnPair);
    }
    for (EventColumn eventColumn : eventColumns) {
        if (mode.isInclude() && viewNames.containsKey(StringUtils.lowerCase(eventColumn.getColumnName()))) {
            // 正向匹配
            tempColumns.add(eventColumn);
        } else if (mode.isExclude() && !viewNames.containsKey(StringUtils.lowerCase(eventColumn.getColumnName()))) {
            // 逆向匹配
            tempColumns.add(eventColumn);
        }
    }
    return tempColumns;
}