Examples with Direction com.aliyun.openservices.ots.model.Direction used on opensource projects

Example 1 with Direction

use of com.aliyun.openservices.ots.model.Direction in project DataX by alibaba.

the class ParamChecker method checkDirectionAndEnd.

public static Direction checkDirectionAndEnd(TableMeta meta, RowPrimaryKey begin, RowPrimaryKey end) {
    Direction direction = null;
    int cmp = Common.compareRangeBeginAndEnd(meta, begin, end);
    if (cmp > 0) {
        direction = Direction.BACKWARD;
    } else if (cmp < 0) {
        direction = Direction.FORWARD;
    } else {
        throw new IllegalArgumentException("Value of 'range-begin' equal value of 'range-end'.");
    }
    return direction;
}

Example 2 with Direction

use of com.aliyun.openservices.ots.model.Direction in project DataX by alibaba.

the class OtsReaderMasterProxy method defaultRangeSplit.

private List<OTSRange> defaultRangeSplit(OTSClient ots, TableMeta meta, OTSRange range, int num) throws Exception {
    if (num == 1) {
        List<OTSRange> ranges = new ArrayList<OTSRange>();
        ranges.add(range);
        return ranges;
    }
    OTSRange reverseRange = new OTSRange();
    reverseRange.setBegin(range.getEnd());
    reverseRange.setEnd(range.getBegin());
    Direction reverseDirection = (direction == Direction.FORWARD ? Direction.BACKWARD : Direction.FORWARD);
    RowPrimaryKey realBegin = getPKOfFirstRow(range, direction);
    RowPrimaryKey realEnd = getPKOfFirstRow(reverseRange, reverseDirection);
    // 所以不再细分，直接使用用户定义的范围
    if (realBegin == null || realEnd == null) {
        List<OTSRange> ranges = new ArrayList<OTSRange>();
        ranges.add(range);
        return ranges;
    }
    // 如果出现realBegin，realEnd的方向和direction不一致的情况，直接返回range
    int cmp = Common.compareRangeBeginAndEnd(meta, realBegin, realEnd);
    Direction realDirection = cmp > 0 ? Direction.BACKWARD : Direction.FORWARD;
    if (realDirection != direction) {
        LOG.warn("Expect '" + direction + "', but direction of realBegin and readlEnd is '" + realDirection + "'");
        List<OTSRange> ranges = new ArrayList<OTSRange>();
        ranges.add(range);
        return ranges;
    }
    List<OTSRange> ranges = RangeSplit.rangeSplitByCount(meta, realBegin, realEnd, num);
    if (ranges.isEmpty()) {
        // 当PartitionKey相等时，工具内部不会切分Range
        ranges.add(range);
    } else {
        // replace first and last
        OTSRange first = ranges.get(0);
        OTSRange last = ranges.get(ranges.size() - 1);
        first.setBegin(range.getBegin());
        last.setEnd(range.getEnd());
    }
    return ranges;
}

Example 3 with Direction

use of com.aliyun.openservices.ots.model.Direction in project DataX by alibaba.

the class OtsReaderSlaveProxy method read.

public void read(RecordSender sender, Configuration configuration) throws Exception {
    LOG.info("read begin.");
    OTSConf conf = GsonParser.jsonToConf(configuration.getString(OTSConst.OTS_CONF));
    OTSRange range = GsonParser.jsonToRange(configuration.getString(OTSConst.OTS_RANGE));
    Direction direction = GsonParser.jsonToDirection(configuration.getString(OTSConst.OTS_DIRECTION));
    OTSServiceConfiguration configure = new OTSServiceConfiguration();
    configure.setRetryStrategy(new DefaultNoRetry());
    OTSClientAsync ots = new OTSClientAsync(conf.getEndpoint(), conf.getAccessId(), conf.getAccesskey(), conf.getInstanceName(), null, configure, null);
    RowPrimaryKey token = range.getBegin();
    List<String> columns = Common.getNormalColumnNameList(conf.getColumns());
    RequestItem request = null;
    do {
        LOG.debug("Next token : {}", GsonParser.rowPrimaryKeyToJson(token));
        if (request == null) {
            request = generateRequestItem(ots, conf, token, range.getEnd(), direction, columns);
        } else {
            RequestItem req = request;
            GetRangeResult result = RetryHelper.executeWithRetry(new GetRangeCallable(ots, req.getCriteria(), req.getFuture()), conf.getRetry(), conf.getSleepInMilliSecond());
            if ((token = result.getNextStartPrimaryKey()) != null) {
                request = generateRequestItem(ots, conf, token, range.getEnd(), direction, columns);
            }
            rowsToSender(result.getRows(), sender, conf.getColumns());
        }
    } while (token != null);
    ots.shutdown();
    LOG.info("read end.");
}