Examples with DataMediaPair com.alibaba.otter.shared.common.model.config.data.DataMediaPair used on opensource projects

Example 6 with DataMediaPair

use of com.alibaba.otter.shared.common.model.config.data.DataMediaPair in project otter by alibaba.

the class PipelineServiceImpl method doToModel.

/**
     * 用于DO对象转化为Model对象
     * 
     * @param pipelineDO
     * @return Pipeline
     */
private Pipeline doToModel(PipelineDO pipelineDo) {
    Pipeline pipeline = new Pipeline();
    try {
        pipeline.setId(pipelineDo.getId());
        pipeline.setName(pipelineDo.getName());
        pipeline.setParameters(pipelineDo.getParameters());
        pipeline.setDescription(pipelineDo.getDescription());
        pipeline.setGmtCreate(pipelineDo.getGmtCreate());
        pipeline.setGmtModified(pipelineDo.getGmtModified());
        pipeline.setChannelId(pipelineDo.getChannelId());
        pipeline.getParameters().setMainstemClientId(pipeline.getId().shortValue());
        // 组装DatamediaPair
        List<DataMediaPair> pairs = dataMediaPairService.listByPipelineId(pipelineDo.getId());
        Collections.sort(pairs, new DataMediaPairComparable());
        pipeline.setPairs(pairs);
        // 组装Node
        List<PipelineNodeRelationDO> relations = pipelineNodeRelationDao.listByPipelineIds(pipelineDo.getId());
        List<Long> totalNodeIds = new ArrayList<Long>();
        for (PipelineNodeRelationDO relation : relations) {
            Long nodeId = relation.getNodeId();
            if (!totalNodeIds.contains(nodeId)) {
                totalNodeIds.add(nodeId);
            }
        }
        List<Node> totalNodes = nodeService.listByIds(totalNodeIds.toArray(new Long[totalNodeIds.size()]));
        List<Node> selectNodes = new ArrayList<Node>();
        List<Node> extractNodes = new ArrayList<Node>();
        List<Node> loadNodes = new ArrayList<Node>();
        for (Node node : totalNodes) {
            for (PipelineNodeRelationDO relation : relations) {
                if (node.getId().equals(relation.getNodeId())) {
                    if (relation.getLocation().isSelect()) {
                        selectNodes.add(node);
                    } else if (relation.getLocation().isExtract()) {
                        extractNodes.add(node);
                    } else if (relation.getLocation().isLoad()) {
                        loadNodes.add(node);
                    }
                }
            }
        }
        pipeline.setSelectNodes(selectNodes);
        pipeline.setExtractNodes(extractNodes);
        pipeline.setLoadNodes(loadNodes);
    } catch (Exception e) {
        logger.error("ERROR ## change the pipeline Do to Model has an exception");
        throw new ManagerException(e);
    }
    return pipeline;
}

Example 7 with DataMediaPair

use of com.alibaba.otter.shared.common.model.config.data.DataMediaPair in project otter by alibaba.

the class FileLoadActionTest method buildPipeline.

protected Pipeline buildPipeline(final int fileDataStartIndex, int fileDataCount) {
    final Pipeline pipeline = new Pipeline();
    pipeline.setParameters(new PipelineParameter());
    int dataMediaPairCount = fileDataCount / NUMBER_OF_FILE_DATA_COPIES;
    pipeline.setPairs(new ArrayList<DataMediaPair>(dataMediaPairCount));
    for (int i = fileDataStartIndex; i < dataMediaPairCount; i++) {
        DataMediaPair dataMediaPair = buildDataMediaPair(i, i);
        pipeline.getPairs().add(dataMediaPair);
    }
    return pipeline;
}

Example 8 with DataMediaPair

use of com.alibaba.otter.shared.common.model.config.data.DataMediaPair in project otter by alibaba.

the class FileLoadActionTest method buildDataMediaPair.

protected DataMediaPair buildDataMediaPair(long id, long pushWeight) {
    DataMediaPair result = new DataMediaPair();
    result.setId(id);
    result.setPushWeight(pushWeight);
    return result;
}

Example 9 with DataMediaPair

use of com.alibaba.otter.shared.common.model.config.data.DataMediaPair in project otter by alibaba.

the class LocalFileLoaderActionTest method generatorDataMediaPair.

private List<DataMediaPair> generatorDataMediaPair(int count) {
    List<DataMediaPair> pairs = new ArrayList<DataMediaPair>();
    for (int i = 0; i < count; i++) {
        DataMediaPair pair = new DataMediaPair();
        pair.setId(Long.valueOf(i));
        pair.setPullWeight(Long.valueOf(i));
        pair.setPushWeight(Long.valueOf(i));
        DbDataMedia mysqlMedia = getMysqlMedia();
        mysqlMedia.setId(Long.valueOf(count + 1));
        pair.setSource(mysqlMedia);
        DbDataMedia oracleMedia = getOracleMedia();
        oracleMedia.setId(Long.valueOf(i));
        pair.setTarget(oracleMedia);
        pairs.add(pair);
    }
    return pairs;
}

Example 10 with DataMediaPair

use of com.alibaba.otter.shared.common.model.config.data.DataMediaPair in project otter by alibaba.

the class OtterTransformerTest method test_rowData_mysql_oracle.

@Test
public void test_rowData_mysql_oracle() {
    final Pipeline pipeline = new Pipeline();
    pipeline.setId(100L);
    List<DataMediaPair> pairs = new ArrayList<DataMediaPair>();
    DataMediaPair pair1 = new DataMediaPair();
    pair1.setId(1L);
    pair1.setPipelineId(pipeline.getId());
    pair1.setPullWeight(1L);
    pair1.setPushWeight(1L);
    DbDataMedia mysqlMedia = getMysqlMedia();
    mysqlMedia.setId(1L);
    pair1.setSource(mysqlMedia);
    DbDataMedia oracleMedia = getOracleMedia();
    pair1.setTarget(oracleMedia);
    pairs.add(pair1);
    pipeline.setPairs(pairs);
    PipelineParameter param = new PipelineParameter();
    param.setSyncMode(SyncMode.ROW);
    pipeline.setParameters(param);
    new NonStrictExpectations() {

        {
            configClientService.findPipeline(anyLong);
            returns(pipeline);
        }
    };
    Identity identity = new Identity();
    identity.setChannelId(100L);
    identity.setPipelineId(100L);
    identity.setProcessId(100L);
    RowBatch rowBatch = new RowBatch();
    rowBatch.setIdentity(identity);
    EventData eventData = new EventData();
    eventData.setTableId(1L);
    eventData.setSchemaName("srf");
    eventData.setTableName("columns");
    eventData.setEventType(EventType.UPDATE);
    eventData.setExecuteTime(100L);
    eventData.getKeys().add(buildColumn("id", Types.INTEGER, "1", true, false));
    eventData.getKeys().add(buildColumn("name", Types.VARCHAR, "ljh", true, false));
    eventData.getColumns().add(buildColumn("alias_name", Types.CHAR, "hello", false, false));
    eventData.getColumns().add(buildColumn("amount", Types.DECIMAL, "100.01", false, false));
    eventData.getColumns().add(buildColumn("text_b", Types.BLOB, "[116,101,120,116,95,98]", false, false));
    eventData.getColumns().add(buildColumn("text_c", Types.CLOB, "text_c", false, false));
    eventData.getColumns().add(buildColumn("curr_date", Types.DATE, "2011-01-01", false, false));
    eventData.getColumns().add(buildColumn("gmt_create", Types.TIMESTAMP, "2011-01-01 11:11:11", false, false));
    eventData.getColumns().add(buildColumn("gmt_modify", Types.TIMESTAMP, "2011-01-01 11:11:11", false, false));
    rowBatch.merge(eventData);
    Map<Class, BatchObject> batchs = otterTransformFactory.transform(rowBatch);
    RowBatch result = (RowBatch) batchs.get(EventData.class);
    want.number(result.getDatas().size()).isEqualTo(1);
}