Examples with Identity com.alibaba.otter.shared.etl.model.Identity used on opensource projects

Example 6 with Identity

use of com.alibaba.otter.shared.etl.model.Identity in project otter by alibaba.

the class DbLoadActionTest method test_db_load_mysql.

@Test
public void test_db_load_mysql() {
    ArbitrateConfigRegistry.regist(configClientService);
    dbLoadAction = (DbLoadAction) TestedObject.getSpringBeanFactory().getBean("dbLoadAction");
    final Channel channel = new Channel();
    channel.setId(1L);
    final Pipeline pipeline = new Pipeline();
    pipeline.setId(100L);
    List<DataMediaPair> pairs = generatorDataMediaPairForMysql(20);
    pipeline.setPairs(pairs);
    pipeline.getParameters().merge(new SystemParameter());
    pipeline.getParameters().merge(new ChannelParameter());
    // pipeline.getParameters().setChannelInfo("LJH_DEMO");
    // final Pipeline oppositePipeline = new Pipeline();
    // oppositePipeline.setId(101L);
    channel.setPipelines(Arrays.asList(pipeline));
    final Node currentNode = new Node();
    currentNode.setId(1L);
    new NonStrictExpectations() {

        {
            configClientService.findChannel(anyLong);
            returns(channel);
            configClientService.findPipeline(anyLong);
            returns(pipeline);
            configClientService.currentNode();
            returns(currentNode);
        }
    };
    Identity identity = new Identity();
    identity.setChannelId(100L);
    identity.setPipelineId(100L);
    identity.setProcessId(100L);
    RowBatch rowBatch = new RowBatch();
    rowBatch.setIdentity(identity);
    List<EventData> eventDatas = generatorEventDataForMysql(0, 20, EventType.INSERT);
    for (EventData eventData : eventDatas) {
        rowBatch.merge(eventData);
    }
    eventDatas = generatorEventDataForMysql(10, 10, EventType.INSERT);
    for (EventData eventData : eventDatas) {
        rowBatch.merge(eventData);
    }
    eventDatas = generatorEventDataForMysql(19, 1, EventType.DELETE);
    for (EventData eventData : eventDatas) {
        rowBatch.merge(eventData);
    }
    WeightController controller = new WeightController(1);
    dbLoadAction.load(rowBatch, controller);
}

Example 7 with Identity

use of com.alibaba.otter.shared.etl.model.Identity in project otter by alibaba.

the class DataBatchLoader method split.

/**
     * 将rowBatch中的记录，按找载入的目标数据源进行分类
     */
private List<RowBatch> split(RowBatch rowBatch) {
    final Identity identity = rowBatch.getIdentity();
    Map<DataMediaSource, RowBatch> result = new MapMaker().makeComputingMap(new Function<DataMediaSource, RowBatch>() {

        public RowBatch apply(DataMediaSource input) {
            RowBatch rowBatch = new RowBatch();
            rowBatch.setIdentity(identity);
            return rowBatch;
        }
    });
    for (EventData eventData : rowBatch.getDatas()) {
        // 获取介质信息
        DataMedia media = ConfigHelper.findDataMedia(configClientService.findPipeline(identity.getPipelineId()), eventData.getTableId());
        // 归类
        result.get(media.getSource()).merge(eventData);
    }
    return new ArrayList<RowBatch>(result.values());
}

Example 8 with Identity

use of com.alibaba.otter.shared.etl.model.Identity in project otter by alibaba.

the class DbLoadAction method load.

/**
     * 返回结果为已处理成功的记录
     */
public DbLoadContext load(RowBatch rowBatch, WeightController controller) {
    Assert.notNull(rowBatch);
    Identity identity = rowBatch.getIdentity();
    DbLoadContext context = buildContext(identity);
    try {
        List<EventData> datas = rowBatch.getDatas();
        context.setPrepareDatas(datas);
        // 执行重复录入数据过滤
        datas = context.getPrepareDatas();
        if (datas == null || datas.size() == 0) {
            logger.info("##no eventdata for load, return");
            return context;
        }
        // 因为所有的数据在DbBatchLoader已按照DateMediaSource进行归好类，不同数据源介质会有不同的DbLoadAction进行处理
        // 设置media source时，只需要取第一节点的source即可
        context.setDataMediaSource(ConfigHelper.findDataMedia(context.getPipeline(), datas.get(0).getTableId()).getSource());
        interceptor.prepare(context);
        // 执行重复录入数据过滤
        datas = context.getPrepareDatas();
        // 主要考虑ddl的幂等性问题，尽可能一个ddl一个batch，失败或者回滚都只针对这条sql
        if (isDdlDatas(datas)) {
            doDdl(context, datas);
        } else {
            WeightBuckets<EventData> buckets = buildWeightBuckets(context, datas);
            List<Long> weights = buckets.weights();
            // weights可能为空，也得调用start方法
            controller.start(weights);
            if (CollectionUtils.isEmpty(datas)) {
                logger.info("##no eventdata for load");
            }
            // 根据manager配置调整线程池
            adjustPoolSize(context);
            // 调整一下运行参数
            adjustConfig(context);
            // 处理数据
            for (int i = 0; i < weights.size(); i++) {
                Long weight = weights.get(i);
                controller.await(weight.intValue());
                // 处理同一个weight下的数据
                List<EventData> items = buckets.getItems(weight);
                logger.debug("##start load for weight:" + weight);
                // 预处理下数据
                // 进行一次数据合并，合并相同pk的多次I/U/D操作
                items = DbLoadMerger.merge(items);
                // 按I/U/D进行归并处理
                DbLoadData loadData = new DbLoadData();
                doBefore(items, context, loadData);
                // 执行load操作
                doLoad(context, loadData);
                controller.single(weight.intValue());
                logger.debug("##end load for weight:" + weight);
            }
        }
        interceptor.commit(context);
    } catch (InterruptedException e) {
        Thread.currentThread().interrupt();
        interceptor.error(context);
    } catch (Exception e) {
        interceptor.error(context);
        throw new LoadException(e);
    }
    // 返回处理成功的记录
    return context;
}

Example 9 with Identity

use of com.alibaba.otter.shared.etl.model.Identity in project otter by alibaba.

the class AbstractOperationInterceptor method updateMark.

/**
     * 更新一下事务标记
     */
private void updateMark(DbLoadContext context, DbDialect dialect, int threadId, String sql, boolean needInfo, String hint) {
    Identity identity = context.getIdentity();
    Channel channel = context.getChannel();
    // 获取dbDialect
    String markTableName = context.getPipeline().getParameters().getSystemSchema() + "." + context.getPipeline().getParameters().getSystemMarkTable();
    String markTableColumn = context.getPipeline().getParameters().getSystemMarkTableColumn();
    synchronized (dialect.getJdbcTemplate()) {
        if (tableCheckStatus.contains(dialect.getJdbcTemplate()) == false) {
            init(dialect.getJdbcTemplate(), markTableName, markTableColumn);
            tableCheckStatus.add(dialect.getJdbcTemplate());
        }
    }
    int affectedCount = 0;
    if (needInfo) {
        String infoColumn = context.getPipeline().getParameters().getSystemMarkTableInfo();
        // 记录一下channelInfo
        String info = context.getPipeline().getParameters().getChannelInfo();
        String esql = MessageFormat.format(sql, new Object[] { markTableName, markTableColumn, infoColumn });
        if (hint != null) {
            esql = hint + esql;
        }
        affectedCount = dialect.getJdbcTemplate().update(esql, new Object[] { threadId, channel.getId(), info });
    } else {
        String esql = MessageFormat.format(sql, new Object[] { markTableName, markTableColumn });
        if (hint != null) {
            esql = hint + esql;
        }
        affectedCount = dialect.getJdbcTemplate().update(esql, new Object[] { threadId, channel.getId() });
    }
    if (affectedCount <= 0) {
        logger.warn("## update {} failed by [{}]", markTableName, threadId);
    } else {
        if (logger.isInfoEnabled()) {
            logger.debug("Interceptor For [{}]", identity);
        }
    }
}

Example 10 with Identity

use of com.alibaba.otter.shared.etl.model.Identity in project otter by alibaba.

the class FileExtractor method extract.

public void extract(DbBatch dbBatch) throws ExtractException {
    List<FileData> fileDatas = doFileExtract(dbBatch.getRowBatch());
    FileBatch fileBatch = new FileBatch();
    fileBatch.setFiles(fileDatas);
    Identity identity = new Identity();
    identity.setChannelId(dbBatch.getRowBatch().getIdentity().getChannelId());
    identity.setPipelineId(dbBatch.getRowBatch().getIdentity().getPipelineId());
    identity.setProcessId(dbBatch.getRowBatch().getIdentity().getProcessId());
    fileBatch.setIdentity(identity);
    dbBatch.setFileBatch(fileBatch);
}