Examples with OTSRange com.alibaba.datax.plugin.reader.otsreader.model.OTSRange used on opensource projects

Example 1 with OTSRange

use of com.alibaba.datax.plugin.reader.otsreader.model.OTSRange in project DataX by alibaba.

the class ParamChecker method checkRangeAndGet.

public static OTSRange checkRangeAndGet(TableMeta meta, List<PrimaryKeyValue> begin, List<PrimaryKeyValue> end) {
    OTSRange range = new OTSRange();
    if (begin.size() == 0 && end.size() == 0) {
        RowPrimaryKey beginRow = new RowPrimaryKey();
        RowPrimaryKey endRow = new RowPrimaryKey();
        for (String name : meta.getPrimaryKey().keySet()) {
            beginRow.addPrimaryKeyColumn(name, PrimaryKeyValue.INF_MIN);
            endRow.addPrimaryKeyColumn(name, PrimaryKeyValue.INF_MAX);
        }
        range.setBegin(beginRow);
        range.setEnd(endRow);
    } else {
        RowPrimaryKey beginRow = checkInputPrimaryKeyAndGet(meta, begin);
        RowPrimaryKey endRow = checkInputPrimaryKeyAndGet(meta, end);
        range.setBegin(beginRow);
        range.setEnd(endRow);
    }
    return range;
}

Example 2 with OTSRange

use of com.alibaba.datax.plugin.reader.otsreader.model.OTSRange in project DataX by alibaba.

the class RangeSplit method rangeSplitByCount.

public static List<OTSRange> rangeSplitByCount(TableMeta meta, RowPrimaryKey begin, RowPrimaryKey end, int count) {
    List<OTSRange> results = new ArrayList<OTSRange>();
    OTSPrimaryKeyColumn partitionKey = Common.getPartitionKey(meta);
    PrimaryKeyValue beginPartitionKey = begin.getPrimaryKey().get(partitionKey.getName());
    PrimaryKeyValue endPartitionKey = end.getPrimaryKey().get(partitionKey.getName());
    // 第一，先对PartitionKey列进行拆分
    List<PrimaryKeyValue> ranges = RangeSplit.splitRangeByPrimaryKeyType(partitionKey.getType(), beginPartitionKey, endPartitionKey, count);
    if (ranges.isEmpty()) {
        return results;
    }
    int size = ranges.size();
    for (int i = 0; i < size - 1; i++) {
        RowPrimaryKey bPk = new RowPrimaryKey();
        RowPrimaryKey ePk = new RowPrimaryKey();
        bPk.addPrimaryKeyColumn(partitionKey.getName(), ranges.get(i));
        ePk.addPrimaryKeyColumn(partitionKey.getName(), ranges.get(i + 1));
        results.add(new OTSRange(bPk, ePk));
    }
    // 第二，填充非PartitionKey的ParimaryKey列
    // 注意：在填充过程中，需要使用用户给定的Begin和End来替换切分出来的第一个Range
    // 的Begin和最后一个Range的End
    List<String> keys = new ArrayList<String>(meta.getPrimaryKey().size());
    keys.addAll(meta.getPrimaryKey().keySet());
    for (int i = 0; i < results.size(); i++) {
        for (int j = 1; j < keys.size(); j++) {
            OTSRange c = results.get(i);
            RowPrimaryKey beginPK = c.getBegin();
            RowPrimaryKey endPK = c.getEnd();
            String key = keys.get(j);
            if (i == 0) {
                // 第一行
                beginPK.addPrimaryKeyColumn(key, begin.getPrimaryKey().get(key));
                endPK.addPrimaryKeyColumn(key, PrimaryKeyValue.INF_MIN);
            } else if (i == results.size() - 1) {
                // 最后一行
                beginPK.addPrimaryKeyColumn(key, PrimaryKeyValue.INF_MIN);
                endPK.addPrimaryKeyColumn(key, end.getPrimaryKey().get(key));
            } else {
                beginPK.addPrimaryKeyColumn(key, PrimaryKeyValue.INF_MIN);
                endPK.addPrimaryKeyColumn(key, PrimaryKeyValue.INF_MIN);
            }
        }
    }
    return results;
}

Example 3 with OTSRange

use of com.alibaba.datax.plugin.reader.otsreader.model.OTSRange in project DataX by alibaba.

the class OtsReaderMasterProxy method defaultRangeSplit.

private List<OTSRange> defaultRangeSplit(OTSClient ots, TableMeta meta, OTSRange range, int num) throws Exception {
    if (num == 1) {
        List<OTSRange> ranges = new ArrayList<OTSRange>();
        ranges.add(range);
        return ranges;
    }
    OTSRange reverseRange = new OTSRange();
    reverseRange.setBegin(range.getEnd());
    reverseRange.setEnd(range.getBegin());
    Direction reverseDirection = (direction == Direction.FORWARD ? Direction.BACKWARD : Direction.FORWARD);
    RowPrimaryKey realBegin = getPKOfFirstRow(range, direction);
    RowPrimaryKey realEnd = getPKOfFirstRow(reverseRange, reverseDirection);
    // 所以不再细分，直接使用用户定义的范围
    if (realBegin == null || realEnd == null) {
        List<OTSRange> ranges = new ArrayList<OTSRange>();
        ranges.add(range);
        return ranges;
    }
    // 如果出现realBegin，realEnd的方向和direction不一致的情况，直接返回range
    int cmp = Common.compareRangeBeginAndEnd(meta, realBegin, realEnd);
    Direction realDirection = cmp > 0 ? Direction.BACKWARD : Direction.FORWARD;
    if (realDirection != direction) {
        LOG.warn("Expect '" + direction + "', but direction of realBegin and readlEnd is '" + realDirection + "'");
        List<OTSRange> ranges = new ArrayList<OTSRange>();
        ranges.add(range);
        return ranges;
    }
    List<OTSRange> ranges = RangeSplit.rangeSplitByCount(meta, realBegin, realEnd, num);
    if (ranges.isEmpty()) {
        // 当PartitionKey相等时，工具内部不会切分Range
        ranges.add(range);
    } else {
        // replace first and last
        OTSRange first = ranges.get(0);
        OTSRange last = ranges.get(ranges.size() - 1);
        first.setBegin(range.getBegin());
        last.setEnd(range.getEnd());
    }
    return ranges;
}

Example 4 with OTSRange

use of com.alibaba.datax.plugin.reader.otsreader.model.OTSRange in project DataX by alibaba.

the class OtsReaderSlaveProxy method read.

public void read(RecordSender sender, Configuration configuration) throws Exception {
    LOG.info("read begin.");
    OTSConf conf = GsonParser.jsonToConf(configuration.getString(OTSConst.OTS_CONF));
    OTSRange range = GsonParser.jsonToRange(configuration.getString(OTSConst.OTS_RANGE));
    Direction direction = GsonParser.jsonToDirection(configuration.getString(OTSConst.OTS_DIRECTION));
    OTSServiceConfiguration configure = new OTSServiceConfiguration();
    configure.setRetryStrategy(new DefaultNoRetry());
    OTSClientAsync ots = new OTSClientAsync(conf.getEndpoint(), conf.getAccessId(), conf.getAccesskey(), conf.getInstanceName(), null, configure, null);
    RowPrimaryKey token = range.getBegin();
    List<String> columns = Common.getNormalColumnNameList(conf.getColumns());
    RequestItem request = null;
    do {
        LOG.debug("Next token : {}", GsonParser.rowPrimaryKeyToJson(token));
        if (request == null) {
            request = generateRequestItem(ots, conf, token, range.getEnd(), direction, columns);
        } else {
            RequestItem req = request;
            GetRangeResult result = RetryHelper.executeWithRetry(new GetRangeCallable(ots, req.getCriteria(), req.getFuture()), conf.getRetry(), conf.getSleepInMilliSecond());
            if ((token = result.getNextStartPrimaryKey()) != null) {
                request = generateRequestItem(ots, conf, token, range.getEnd(), direction, columns);
            }
            rowsToSender(result.getRows(), sender, conf.getColumns());
        }
    } while (token != null);
    ots.shutdown();
    LOG.info("read end.");
}

Example 5 with OTSRange

use of com.alibaba.datax.plugin.reader.otsreader.model.OTSRange in project DataX by alibaba.

the class OtsReaderMasterProxy method split.

public List<Configuration> split(int num) throws Exception {
    LOG.info("Expect split num : " + num);
    List<Configuration> configurations = new ArrayList<Configuration>();
    List<OTSRange> ranges = null;
    if (this.conf.getRangeSplit() != null) {
        // 用户显示指定了拆分范围
        LOG.info("Begin userDefinedRangeSplit");
        ranges = userDefinedRangeSplit(meta, range, this.conf.getRangeSplit());
        LOG.info("End userDefinedRangeSplit");
    } else {
        // 采用默认的切分算法 
        LOG.info("Begin defaultRangeSplit");
        ranges = defaultRangeSplit(ots, meta, range, num);
        LOG.info("End defaultRangeSplit");
    }
    // 解决大量的Split Point序列化消耗内存的问题
    // 因为slave中不会使用这个配置，所以置为空
    this.conf.setRangeSplit(null);
    for (OTSRange item : ranges) {
        Configuration configuration = Configuration.newDefault();
        configuration.set(OTSConst.OTS_CONF, GsonParser.confToJson(this.conf));
        configuration.set(OTSConst.OTS_RANGE, GsonParser.rangeToJson(item));
        configuration.set(OTSConst.OTS_DIRECTION, GsonParser.directionToJson(direction));
        configurations.add(configuration);
    }
    LOG.info("Configuration list count : " + configurations.size());
    return configurations;
}