Examples with DataMediaPair com.alibaba.otter.shared.common.model.config.data.DataMediaPair used on opensource projects

Example 1 with DataMediaPair

use of com.alibaba.otter.shared.common.model.config.data.DataMediaPair in project otter by alibaba.

the class DbLoadAction method buildWeightBuckets.

/**
 * 构建基于weight权重分组的item集合列表
 */
private WeightBuckets<EventData> buildWeightBuckets(DbLoadContext context, List<EventData> datas) {
    WeightBuckets<EventData> buckets = new WeightBuckets<EventData>();
    for (EventData data : datas) {
        // 获取对应的weight
        DataMediaPair pair = ConfigHelper.findDataMediaPair(context.getPipeline(), data.getPairId());
        buckets.addItem(pair.getPushWeight(), data);
    }
    return buckets;
}

Example 2 with DataMediaPair

use of com.alibaba.otter.shared.common.model.config.data.DataMediaPair in project otter by alibaba.

the class FileExtractor method doFileExtract.

/**
 * 返回这批变更数据对应的FileInfo.
 *
 * @param rowBatch
 * @return
 */
private List<FileData> doFileExtract(RowBatch rowBatch) {
    List<FileData> fileDatas = new ArrayList<FileData>();
    // 处理数据
    Pipeline pipeline = getPipeline(rowBatch.getIdentity().getPipelineId());
    List<EventData> eventDatas = rowBatch.getDatas();
    for (EventData eventData : eventDatas) {
        if (eventData.getEventType().isDdl()) {
            continue;
        }
        List<DataMediaPair> dataMediaPairs = ConfigHelper.findDataMediaPairByMediaId(pipeline, eventData.getTableId());
        if (dataMediaPairs == null) {
            throw new ExtractException("ERROR ## the dataMediaId = " + eventData.getTableId() + " dataMediaPair is null,please check");
        }
        for (DataMediaPair dataMediaPair : dataMediaPairs) {
            if (dataMediaPair.getResolverData() == null || dataMediaPair.getResolverData().getExtensionDataType() == null || (dataMediaPair.getResolverData().getExtensionDataType().isClazz() && StringUtils.isBlank(dataMediaPair.getResolverData().getClazzPath())) || (dataMediaPair.getResolverData().getExtensionDataType().isSource() && StringUtils.isBlank(dataMediaPair.getResolverData().getSourceText()))) {
                continue;
            }
            FileResolver fileResolver = null;
            if (dataMediaPair.getResolverData() != null) {
                fileResolver = extensionFactory.getExtension(FileResolver.class, dataMediaPair.getResolverData());
            } else {
                continue;
            }
            if (fileResolver == null) {
                throw new ExtractException("ERROR ## the dataMediaId = " + eventData.getTableId() + " the fileResolver className  = " + dataMediaPair.getResolverData().getClazzPath() + " is null ,please check the class");
            }
            if (fileResolver instanceof RemoteDirectoryFetcherAware) {
                RemoteDirectoryFetcherAware remoteDirectoryFetcherAware = (RemoteDirectoryFetcherAware) fileResolver;
                remoteDirectoryFetcherAware.setRemoteDirectoryFetcher(arandaRemoteDirectoryFetcher);
            }
            List<FileData> singleRowFileDatas = getSingleRowFileInfos(dataMediaPair.getId(), fileResolver, eventData);
            // 做一下去重处理
            for (FileData data : singleRowFileDatas) {
                if (!fileDatas.contains(data)) {
                    fileDatas.add(data);
                }
            }
        }
    }
    // 判断是否需要进行图片重复同步检查
    if (pipeline.getParameters().getFileDetect()) {
        doFileDetectCollector(pipeline, fileDatas);
    }
    return fileDatas;
}

Example 3 with DataMediaPair

use of com.alibaba.otter.shared.common.model.config.data.DataMediaPair in project otter by alibaba.

the class GroupExtractor method extract.

@Override
public void extract(DbBatch dbBatch) throws ExtractException {
    Assert.notNull(dbBatch);
    Assert.notNull(dbBatch.getRowBatch());
    Pipeline pipeline = getPipeline(dbBatch.getRowBatch().getIdentity().getPipelineId());
    List<DataMediaPair> dataMediaPairs = pipeline.getPairs();
    /**
     * Key = TableId<br>
     * Value = a List of this tableId's column need to sync<br>
     */
    Map<Long, List<ColumnGroup>> groupColumns = new HashMap<Long, List<ColumnGroup>>();
    for (DataMediaPair dataMediaPair : dataMediaPairs) {
        List<ColumnGroup> columnGroups = dataMediaPair.getColumnGroups();
        if (!CollectionUtils.isEmpty(columnGroups)) {
            groupColumns.put(dataMediaPair.getSource().getId(), columnGroups);
        }
    }
    List<EventData> eventDatas = dbBatch.getRowBatch().getDatas();
    for (EventData eventData : eventDatas) {
        if (eventData.getEventType().isDdl()) {
            continue;
        }
        List<ColumnGroup> columnGroups = groupColumns.get(eventData.getTableId());
        if (!CollectionUtils.isEmpty(columnGroups)) {
            for (ColumnGroup columnGroup : columnGroups) {
                if (columnGroup != null && !CollectionUtils.isEmpty(columnGroup.getColumnPairs())) {
                    groupFilter(eventData, columnGroup);
                }
            }
        }
    }
}

Example 4 with DataMediaPair

use of com.alibaba.otter.shared.common.model.config.data.DataMediaPair in project otter by alibaba.

the class BehaviorHistoryCurve method execute.

public void execute(@Param("d5221") String startTime, @Param("d5222") String endTime, @Param("dataMediaPairId") Long dataMediaPairId, HttpSession session, Context context) throws Exception {
    Date end = null;
    Date start = null;
    SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm");
    if (StringUtils.isEmpty(startTime) || StringUtils.isEmpty(endTime)) {
        start = new Date(System.currentTimeMillis() / 60000 * 60000 - 24 * 60 * 60 * 1000);
        end = new Date(System.currentTimeMillis() / 60000 * 60000);
    } else {
        // 当前24小时，时间取整分钟
        sdf.setLenient(false);
        if (null != startTime && null != endTime) {
            start = sdf.parse(startTime);
            end = sdf.parse(endTime);
        }
    }
    DataMediaPair dataMediaPair = dataMediaPairService.findById(dataMediaPairId);
    Channel channel = channelService.findByPipelineId(dataMediaPair.getPipelineId());
    Map<Long, BehaviorHistoryInfo> behaviourHistoryInfos = new LinkedHashMap<Long, BehaviorHistoryInfo>();
    TimelineBehaviorHistoryCondition condition = new TimelineBehaviorHistoryCondition();
    if (null != start && null != end) {
        condition.setStart(start);
        condition.setEnd(end);
        condition.setPairId(dataMediaPairId);
        behaviourHistoryInfos = tableStatService.listTimelineBehaviorHistory(condition);
    }
    Long totalInsert = 0L;
    Long totalUpdate = 0L;
    Long totalDelete = 0L;
    Long totalFileCount = 0L;
    Long totalFileSize = 0L;
    for (BehaviorHistoryInfo info : behaviourHistoryInfos.values()) {
        totalInsert += info.getInsertNumber();
        totalUpdate += info.getUpdateNumber();
        totalDelete += info.getDeleteNumber();
        totalFileCount += info.getFileNumber();
        totalFileSize += info.getFileSize();
    }
    context.put("totalInsert", totalInsert);
    context.put("totalUpdate", totalUpdate);
    context.put("totalDelete", totalDelete);
    context.put("totalFileCount", totalFileCount);
    context.put("totalFileSize", totalFileSize);
    context.put("behaviourHistoryInfos", behaviourHistoryInfos);
    context.put("start", sdf.format(start));
    context.put("end", sdf.format(end));
    context.put("dataMediaPair", dataMediaPair);
    context.put("channel", channel);
}

Example 5 with DataMediaPair

use of com.alibaba.otter.shared.common.model.config.data.DataMediaPair in project otter by alibaba.

the class DataMediaPairAction method doBatchAdd.

/**
 * 批量添加DataMediaPair
 *
 * @param dataMediaPairInfo
 * @throws Exception
 */
public void doBatchAdd(@FormGroup("batchDataMediaPairInfo") Group batchDataMediaPairInfo, @Param("pipelineId") Long pipelineId, @FormField(name = "formBatchDataMediaPairError", group = "batchDataMediaPairInfo") CustomErrors err, Navigator nav) throws Exception {
    String batchPairContent = batchDataMediaPairInfo.getField("batchPairContent").getStringValue();
    List<String> StringPairs = Arrays.asList(batchPairContent.split("\r\n"));
    try {
        for (String stringPair : StringPairs) {
            List<String> pairData = Arrays.asList(stringPair.split(","));
            if (pairData.size() < 4) {
                throw new ManagerException("[" + stringPair + "] the line not all parameters");
            }
            // build the pair source
            DataMedia sourceDataMedia = new DataMedia();
            DataMediaSource sourceDataMediaSource = dataMediaSourceService.findById(Long.parseLong(StringUtils.trimToNull(pairData.get(2))));
            sourceDataMedia.setNamespace(StringUtils.trimToNull(pairData.get(0)));
            sourceDataMedia.setName(StringUtils.trimToNull(pairData.get(1)));
            sourceDataMedia.setSource(sourceDataMediaSource);
            Long sourceMediaId = dataMediaService.createReturnId(sourceDataMedia);
            sourceDataMedia.setId(sourceMediaId);
            // build the pair target
            DataMedia targetDataMedia = new DataMedia();
            Long weight = 5L;
            if (StringUtils.isNumeric(pairData.get(3)) && pairData.size() <= 5) {
                // 如果是纯数字，那说明是简化配置模式
                DataMediaSource targetDataMediaSource = dataMediaSourceService.findById(Long.parseLong(StringUtils.trimToNull(pairData.get(3))));
                targetDataMedia.setNamespace(StringUtils.trimToNull(pairData.get(0)));
                targetDataMedia.setName(StringUtils.trimToNull(pairData.get(1)));
                targetDataMedia.setSource(targetDataMediaSource);
                Long targetMediaId = dataMediaService.createReturnId(targetDataMedia);
                targetDataMedia.setId(targetMediaId);
                if (pairData.size() >= 5) {
                    weight = Long.parseLong(StringUtils.trimToNull(pairData.get(4)));
                }
            } else {
                DataMediaSource targetDataMediaSource = dataMediaSourceService.findById(Long.parseLong(StringUtils.trimToNull(pairData.get(5))));
                targetDataMedia.setNamespace(StringUtils.trimToNull(pairData.get(3)));
                targetDataMedia.setName(StringUtils.trimToNull(pairData.get(4)));
                targetDataMedia.setSource(targetDataMediaSource);
                Long targetMediaId = dataMediaService.createReturnId(targetDataMedia);
                targetDataMedia.setId(targetMediaId);
                if (pairData.size() >= 7) {
                    weight = Long.parseLong(StringUtils.trimToNull(pairData.get(6)));
                }
            }
            // build the pair
            DataMediaPair dataMediaPair = new DataMediaPair();
            dataMediaPair.setSource(sourceDataMedia);
            dataMediaPair.setTarget(targetDataMedia);
            dataMediaPair.setPushWeight(weight);
            dataMediaPair.setPipelineId(pipelineId);
            dataMediaPairService.createIfNotExist(dataMediaPair);
        }
    } catch (Exception e) {
        err.setMessage("invalidBatchDataMediaPair");
        return;
    }
    nav.redirectToLocation("dataMediaPairList.htm?pipelineId=" + pipelineId);
}