Examples with Pipeline com.alibaba.otter.shared.common.model.config.pipeline.Pipeline used on opensource projects

Example 71 with Pipeline

use of com.alibaba.otter.shared.common.model.config.pipeline.Pipeline in project otter by alibaba.

the class AttachmentHttpPipe method archiveFile.

// 处理对应的附件
private HttpPipeKey archiveFile(final FileBatch fileBatch) {
    // 处理构造对应的文件url
    String filename = buildFileName(fileBatch.getIdentity(), ClassUtils.getShortClassName(fileBatch.getClass()));
    File file = new File(htdocsDir, filename);
    // 压缩对应的文件数据
    List<FileData> fileDatas = fileBatch.getFiles();
    Pipeline pipeline = configClientService.findPipeline(fileBatch.getIdentity().getPipelineId());
    int poolSize = pipeline.getParameters().getFileLoadPoolSize();
    boolean useLocalFileMutliThread = pipeline.getParameters().getUseLocalFileMutliThread();
    ArchiveBean archiveBean = getArchiveBean();
    // 调整线程池大小
    archiveBean.adjustPoolSize(poolSize);
    // 设置是否启用local多线程同步
    archiveBean.setUseLocalFileMutliThread(useLocalFileMutliThread);
    boolean done = archiveBean.pack(file, fileDatas, new ArchiveRetriverCallback<FileData>() {

        public InputStream retrive(FileData fileData) {
            boolean miss = false;
            try {
                if (StringUtils.isNotEmpty(fileData.getNameSpace())) {
                    throw new RuntimeException(fileData + " is not support!");
                } else {
                    File source = new File(fileData.getPath());
                    if (source.exists() && source.isFile()) {
                        return new LazyFileInputStream(source);
                    } else {
                        miss = true;
                        return null;
                    }
                }
            } finally {
                if (miss && logger.isInfoEnabled()) {
                    MDC.put(OtterConstants.splitPipelineLoadLogFileKey, String.valueOf(fileBatch.getIdentity().getPipelineId()));
                    logger.info(FileloadDumper.dumpMissFileDatas(fileBatch.getIdentity(), fileData));
                }
            }
        }
    });
    if (done == false) {
        // 直接返回
        return null;
    }
    HttpPipeKey key = new HttpPipeKey();
    key.setUrl(remoteUrlBuilder.getUrl(fileBatch.getIdentity().getPipelineId(), filename));
    key.setDataType(PipeDataType.FILE_BATCH);
    key.setIdentity(fileBatch.getIdentity());
    if (encrypt || pipeline.getParameters().getUseFileEncrypt()) {
        // 加密处理
        EncryptedData encryptedData = encryptFile(file);
        key.setKey(encryptedData.getKey());
        key.setCrc(encryptedData.getCrc());
    }
    return key;
}

Example 72 with Pipeline

use of com.alibaba.otter.shared.common.model.config.pipeline.Pipeline in project otter by alibaba.

the class AnalysisStageStat method execute.

public void execute(@Param("pipelineId") Long pipelineId, Context context) throws Exception {
    List<ProcessStat> processStats = new ArrayList<ProcessStat>();
    Pipeline pipeline = pipelineService.findById(pipelineId);
    processStats = processStatService.listRealtimeProcessStat(pipelineId);
    // Map ext = new HashMap<Long, String>();
    // // ext.put(145456451, "asdf");
    // for (Long i = 1L; i <= 3; i++) {
    // List<StageStat> stageStats = new ArrayList<StageStat>();
    // ProcessStat processStat = new ProcessStat();
    // processStat.setPipelineId(1L);
    // processStat.setProcessId(i);
    // StageStat stage = new StageStat();
    // stage.setStage(StageType.SELECT);
    // stage.setStartTime(((new Date()).getTime() + i * 10 * 1000));
    // stage.setEndTime(((new Date()).getTime() + i * 200 * 1000));
    // stage.setNumber(11231230L);
    // stage.setSize(14545645640L);
    // // stage.setExts(ext);
    // stageStats.add(stage);
    // stage = new StageStat();
    // stage.setStage(StageType.EXTRACT);
    // stage.setStartTime(((new Date()).getTime() + i * 2000 * 1000));
    // stage.setEndTime(((new Date()).getTime() + i * 3000 * 1000));
    // stage.setExts(ext);
    // // stage.setNumber(10L);
    // // stage.setSize(10L);
    // stageStats.add(stage);
    // stage = new StageStat();
    // stage.setStage(StageType.TRANSFORM);
    // stage.setStartTime(((new Date()).getTime() + i * 5000 * 1000));
    // stage.setEndTime(((new Date()).getTime() + i * 6000 * 1000));
    // stage.setNumber(154640L);
    // stage.setExts(ext);
    // // stage.setSize(10L);
    // stageStats.add(stage);
    // stage = new StageStat();
    // stage.setStage(StageType.LOAD);
    // stage.setStartTime(((new Date()).getTime() + i * 70000 * 1000));
    // stage.setEndTime(((new Date()).getTime() + i * 80000 * 1000));
    // // stage.setNumber(10L);
    // stage.setSize(101445L);
    // // stage.setExts(ext);
    // stageStats.add(stage);
    // processStat.setStageStats(stageStats);
    // processStats.add(processStat);
    // }
    Long stageStart = 0L;
    // Long stageEnd = new Date().getTime() + 3 * 80000 * 1000;
    Long stageEnd = new Date().getTime();
    Long interval = 0L;
    double offset = 0L;
    // 找出最先开始的process的select阶段的开始时间作为起始时间
    if (processStats.size() > 0) {
        if (processStats.get(0).getStageStats().size() > 0) {
            stageStart = processStats.get(0).getStageStats().get(0).getStartTime();
        }
    }
    // 动态计算每个阶段的长度比例
    if (stageStart > 0) {
        interval = stageEnd - stageStart;
    }
    if (interval > 0) {
        offset = 800.0 / interval;
    }
    // 计算每个process当前任务所做的时间总和
    Map<Long, Long> processTime = new HashMap<Long, Long>();
    for (ProcessStat processStat : processStats) {
        Long timeout = 0L;
        if (processStat.getStageStats().size() > 0) {
            timeout = stageEnd - processStat.getStageStats().get(0).getStartTime();
        }
        processTime.put(processStat.getProcessId(), timeout);
    }
    // 获取下mainstem状态信息
    MainStemEventData mainstemData = arbitrateViewService.mainstemData(pipeline.getChannelId(), pipelineId);
    PositionEventData positionData = arbitrateViewService.getCanalCursor(pipeline.getParameters().getDestinationName(), pipeline.getParameters().getMainstemClientId());
    ChannelStatus status = channelArbitrateEvent.status(pipeline.getChannelId());
    context.put("pipeline", pipeline);
    context.put("pipelineId", pipelineId);
    context.put("processStats", processStats);
    context.put("offset", offset);
    context.put("stageStart", stageStart);
    context.put("stageEnd", stageEnd);
    context.put("processTime", processTime);
    context.put("mainstemData", mainstemData);
    context.put("positionData", positionData);
    context.put("channelStatus", status);
}

Example 73 with Pipeline

use of com.alibaba.otter.shared.common.model.config.pipeline.Pipeline in project otter by alibaba.

the class PipelineAction method doAdd.

public void doAdd(@FormGroup("pipelineInfo") Group pipelineInfo, @FormGroup("pipelineParameterInfo") Group pipelineParameterInfo, @FormField(name = "formPipelineError", group = "pipelineInfo") CustomErrors err, HttpSession session, Navigator nav) throws Exception {
    Pipeline pipeline = new Pipeline();
    PipelineParameter parameters = new PipelineParameter();
    pipelineInfo.setProperties(pipeline);
    pipelineParameterInfo.setProperties(parameters);
    // if (parameters.getLoadPoolSize() < 1) {
    // parameters.setLoadPoolSize(PipelineParameter.DEFAULT_LOAD_POOL_SIZE);
    // }
    List<Long> selectNodeIds = Arrays.asList(ArrayUtils.toObject(pipelineInfo.getField("selectNodeIds").getLongValues()));
    List<Node> selectNodes = new ArrayList<Node>();
    for (Long selectNodeId : selectNodeIds) {
        Node node = new Node();
        node.setId(selectNodeId);
        selectNodes.add(node);
    }
    // select/extract节点普遍配置为同一个节点
    List<Long> extractNodeIds = Arrays.asList(ArrayUtils.toObject(pipelineInfo.getField("selectNodeIds").getLongValues()));
    // List<Long> extractNodeIds =
    // Arrays.asList(ArrayUtils.toObject(pipelineInfo.getField("extractNodeIds").getLongValues()));
    List<Node> extractNodes = new ArrayList<Node>();
    for (Long extractNodeId : extractNodeIds) {
        Node node = new Node();
        node.setId(extractNodeId);
        extractNodes.add(node);
    }
    List<Long> loadNodeIds = Arrays.asList(ArrayUtils.toObject(pipelineInfo.getField("loadNodeIds").getLongValues()));
    List<Node> loadNodes = new ArrayList<Node>();
    for (Long loadNodeId : loadNodeIds) {
        Node node = new Node();
        node.setId(loadNodeId);
        loadNodes.add(node);
    }
    pipeline.setSelectNodes(selectNodes);
    pipeline.setExtractNodes(extractNodes);
    pipeline.setLoadNodes(loadNodes);
    pipeline.setParameters(parameters);
    List<Pipeline> values = pipelineService.listByDestinationWithoutOther(pipeline.getParameters().getDestinationName());
    if (!values.isEmpty()) {
        err.setMessage("invalidDestinationName");
        return;
    }
    try {
        pipelineService.create(pipeline);
    } catch (RepeatConfigureException rce) {
        err.setMessage("invalidPipelineName");
        return;
    }
    nav.redirectToLocation("pipelineList.htm?channelId=" + pipeline.getChannelId());
}

Example 74 with Pipeline

use of com.alibaba.otter.shared.common.model.config.pipeline.Pipeline in project otter by alibaba.

the class ChannelList method execute.

public void execute(@Param("pageIndex") int pageIndex, @Param("searchKey") String searchKey, @Param("channelStatus") String status, @Param("channelId") Long channelId, @Param("errorType") String errorType, Context context) throws Exception {
    @SuppressWarnings("unchecked") Map<String, Object> condition = new HashMap<String, Object>();
    if ("请输入关键字(目前支持Channel的ID、名字搜索)".equals(searchKey)) {
        searchKey = "";
    }
    condition.put("searchKey", searchKey);
    List<Long> theStatusPks = new ArrayList<Long>();
    if (null != status) {
        List<Long> allChannelPks = channelService.listAllChannelId();
        for (Long channelPk : allChannelPks) {
            ChannelStatus channelStatus = arbitrateManageService.channelEvent().status(channelPk);
            if (channelStatus.equals(ChannelStatus.valueOf(status))) {
                theStatusPks.add(channelPk);
            }
        }
    }
    int count = channelService.getCount(condition);
    Paginator paginator = new Paginator();
    paginator.setItems(count);
    paginator.setPage(pageIndex);
    condition.put("offset", paginator.getOffset());
    condition.put("length", paginator.getLength());
    List<Channel> channels = new ArrayList<Channel>();
    if ((null != channelId) && (channelId != 0l)) {
        channels.add(channelService.findById(channelId));
        paginator.setItems(1);
        paginator.setPage(0);
        // 定义为新的searchKey
        searchKey = String.valueOf(channelId);
    } else {
        channels = channelService.listByConditionWithoutColumn(condition);
    }
    List<SeniorChannel> seniorChannels = new ArrayList<SeniorChannel>();
    for (Channel channel : channels) {
        boolean processEmpty = false;
        List<Pipeline> pipelines = channel.getPipelines();
        for (Pipeline pipeline : pipelines) {
            if (processStatService.listRealtimeProcessStat(channel.getId(), pipeline.getId()).isEmpty()) {
                processEmpty = true;
            }
        }
        SeniorChannel seniorChannel = new SeniorChannel();
        seniorChannel.setId(channel.getId());
        seniorChannel.setName(channel.getName());
        seniorChannel.setParameters(channel.getParameters());
        seniorChannel.setPipelines(channel.getPipelines());
        seniorChannel.setStatus(channel.getStatus());
        seniorChannel.setDescription(channel.getDescription());
        seniorChannel.setGmtCreate(channel.getGmtCreate());
        seniorChannel.setGmtModified(channel.getGmtModified());
        seniorChannel.setProcessEmpty(processEmpty);
        seniorChannels.add(seniorChannel);
    }
    context.put("channels", seniorChannels);
    context.put("paginator", paginator);
    context.put("searchKey", searchKey);
    context.put("errorType", errorType);
}

Example 75 with Pipeline

use of com.alibaba.otter.shared.common.model.config.pipeline.Pipeline in project otter by alibaba.

the class PositionAction method doRemove.

public void doRemove(@Param("pipelineId") Long pipelineId, Navigator nav) throws Exception {
    Pipeline pipeline = pipelineService.findById(pipelineId);
    String destination = pipeline.getParameters().getDestinationName();
    short clientId = pipeline.getParameters().getMainstemClientId();
    PositionEventData position = arbitrateViewService.getCanalCursor(destination, clientId);
    // 记录一下日志
    logger.warn("remove pipelineId[{}] position \n {}", pipelineId, position);
    arbitrateViewService.removeCanalCursor(destination, clientId);
    nav.redirectToLocation("analysisStageStat.htm?pipelineId=" + pipelineId);
}