Examples with RowPrimaryKey com.aliyun.openservices.ots.model.RowPrimaryKey used on opensource projects

Example 1 with RowPrimaryKey

use of com.aliyun.openservices.ots.model.RowPrimaryKey in project DataX by alibaba.

the class ParamChecker method checkRangeAndGet.

public static OTSRange checkRangeAndGet(TableMeta meta, List<PrimaryKeyValue> begin, List<PrimaryKeyValue> end) {
    OTSRange range = new OTSRange();
    if (begin.size() == 0 && end.size() == 0) {
        RowPrimaryKey beginRow = new RowPrimaryKey();
        RowPrimaryKey endRow = new RowPrimaryKey();
        for (String name : meta.getPrimaryKey().keySet()) {
            beginRow.addPrimaryKeyColumn(name, PrimaryKeyValue.INF_MIN);
            endRow.addPrimaryKeyColumn(name, PrimaryKeyValue.INF_MAX);
        }
        range.setBegin(beginRow);
        range.setEnd(endRow);
    } else {
        RowPrimaryKey beginRow = checkInputPrimaryKeyAndGet(meta, begin);
        RowPrimaryKey endRow = checkInputPrimaryKeyAndGet(meta, end);
        range.setBegin(beginRow);
        range.setEnd(endRow);
    }
    return range;
}

Example 2 with RowPrimaryKey

use of com.aliyun.openservices.ots.model.RowPrimaryKey in project DataX by alibaba.

the class ParamChecker method checkInputPrimaryKeyAndGet.

public static RowPrimaryKey checkInputPrimaryKeyAndGet(TableMeta meta, List<PrimaryKeyValue> range) {
    if (meta.getPrimaryKey().size() != range.size()) {
        throw new IllegalArgumentException(String.format("Input size of values not equal size of primary key. input size:%d, primary key size:%d .", range.size(), meta.getPrimaryKey().size()));
    }
    RowPrimaryKey pk = new RowPrimaryKey();
    int i = 0;
    for (Entry<String, PrimaryKeyType> e : meta.getPrimaryKey().entrySet()) {
        PrimaryKeyValue value = range.get(i);
        if (e.getValue() != value.getType() && value != PrimaryKeyValue.INF_MIN && value != PrimaryKeyValue.INF_MAX) {
            throw new IllegalArgumentException("Input range type not match primary key. Input type:" + value.getType() + ", Primary Key Type:" + e.getValue() + ", Index:" + i);
        } else {
            pk.addPrimaryKeyColumn(e.getKey(), value);
        }
        i++;
    }
    return pk;
}

Example 3 with RowPrimaryKey

use of com.aliyun.openservices.ots.model.RowPrimaryKey in project DataX by alibaba.

the class RangeSplit method rangeSplitByCount.

public static List<OTSRange> rangeSplitByCount(TableMeta meta, RowPrimaryKey begin, RowPrimaryKey end, int count) {
    List<OTSRange> results = new ArrayList<OTSRange>();
    OTSPrimaryKeyColumn partitionKey = Common.getPartitionKey(meta);
    PrimaryKeyValue beginPartitionKey = begin.getPrimaryKey().get(partitionKey.getName());
    PrimaryKeyValue endPartitionKey = end.getPrimaryKey().get(partitionKey.getName());
    // 第一，先对PartitionKey列进行拆分
    List<PrimaryKeyValue> ranges = RangeSplit.splitRangeByPrimaryKeyType(partitionKey.getType(), beginPartitionKey, endPartitionKey, count);
    if (ranges.isEmpty()) {
        return results;
    }
    int size = ranges.size();
    for (int i = 0; i < size - 1; i++) {
        RowPrimaryKey bPk = new RowPrimaryKey();
        RowPrimaryKey ePk = new RowPrimaryKey();
        bPk.addPrimaryKeyColumn(partitionKey.getName(), ranges.get(i));
        ePk.addPrimaryKeyColumn(partitionKey.getName(), ranges.get(i + 1));
        results.add(new OTSRange(bPk, ePk));
    }
    // 第二，填充非PartitionKey的ParimaryKey列
    // 注意：在填充过程中，需要使用用户给定的Begin和End来替换切分出来的第一个Range
    // 的Begin和最后一个Range的End
    List<String> keys = new ArrayList<String>(meta.getPrimaryKey().size());
    keys.addAll(meta.getPrimaryKey().keySet());
    for (int i = 0; i < results.size(); i++) {
        for (int j = 1; j < keys.size(); j++) {
            OTSRange c = results.get(i);
            RowPrimaryKey beginPK = c.getBegin();
            RowPrimaryKey endPK = c.getEnd();
            String key = keys.get(j);
            if (i == 0) {
                // 第一行
                beginPK.addPrimaryKeyColumn(key, begin.getPrimaryKey().get(key));
                endPK.addPrimaryKeyColumn(key, PrimaryKeyValue.INF_MIN);
            } else if (i == results.size() - 1) {
                // 最后一行
                beginPK.addPrimaryKeyColumn(key, PrimaryKeyValue.INF_MIN);
                endPK.addPrimaryKeyColumn(key, end.getPrimaryKey().get(key));
            } else {
                beginPK.addPrimaryKeyColumn(key, PrimaryKeyValue.INF_MIN);
                endPK.addPrimaryKeyColumn(key, PrimaryKeyValue.INF_MIN);
            }
        }
    }
    return results;
}

Example 4 with RowPrimaryKey

use of com.aliyun.openservices.ots.model.RowPrimaryKey in project DataX by alibaba.

the class OtsReaderMasterProxy method defaultRangeSplit.

private List<OTSRange> defaultRangeSplit(OTSClient ots, TableMeta meta, OTSRange range, int num) throws Exception {
    if (num == 1) {
        List<OTSRange> ranges = new ArrayList<OTSRange>();
        ranges.add(range);
        return ranges;
    }
    OTSRange reverseRange = new OTSRange();
    reverseRange.setBegin(range.getEnd());
    reverseRange.setEnd(range.getBegin());
    Direction reverseDirection = (direction == Direction.FORWARD ? Direction.BACKWARD : Direction.FORWARD);
    RowPrimaryKey realBegin = getPKOfFirstRow(range, direction);
    RowPrimaryKey realEnd = getPKOfFirstRow(reverseRange, reverseDirection);
    // 所以不再细分，直接使用用户定义的范围
    if (realBegin == null || realEnd == null) {
        List<OTSRange> ranges = new ArrayList<OTSRange>();
        ranges.add(range);
        return ranges;
    }
    // 如果出现realBegin，realEnd的方向和direction不一致的情况，直接返回range
    int cmp = Common.compareRangeBeginAndEnd(meta, realBegin, realEnd);
    Direction realDirection = cmp > 0 ? Direction.BACKWARD : Direction.FORWARD;
    if (realDirection != direction) {
        LOG.warn("Expect '" + direction + "', but direction of realBegin and readlEnd is '" + realDirection + "'");
        List<OTSRange> ranges = new ArrayList<OTSRange>();
        ranges.add(range);
        return ranges;
    }
    List<OTSRange> ranges = RangeSplit.rangeSplitByCount(meta, realBegin, realEnd, num);
    if (ranges.isEmpty()) {
        // 当PartitionKey相等时，工具内部不会切分Range
        ranges.add(range);
    } else {
        // replace first and last
        OTSRange first = ranges.get(0);
        OTSRange last = ranges.get(ranges.size() - 1);
        first.setBegin(range.getBegin());
        last.setEnd(range.getEnd());
    }
    return ranges;
}

Example 5 with RowPrimaryKey

use of com.aliyun.openservices.ots.model.RowPrimaryKey in project DataX by alibaba.

the class OtsReaderSlaveProxy method read.

public void read(RecordSender sender, Configuration configuration) throws Exception {
    LOG.info("read begin.");
    OTSConf conf = GsonParser.jsonToConf(configuration.getString(OTSConst.OTS_CONF));
    OTSRange range = GsonParser.jsonToRange(configuration.getString(OTSConst.OTS_RANGE));
    Direction direction = GsonParser.jsonToDirection(configuration.getString(OTSConst.OTS_DIRECTION));
    OTSServiceConfiguration configure = new OTSServiceConfiguration();
    configure.setRetryStrategy(new DefaultNoRetry());
    OTSClientAsync ots = new OTSClientAsync(conf.getEndpoint(), conf.getAccessId(), conf.getAccesskey(), conf.getInstanceName(), null, configure, null);
    RowPrimaryKey token = range.getBegin();
    List<String> columns = Common.getNormalColumnNameList(conf.getColumns());
    RequestItem request = null;
    do {
        LOG.debug("Next token : {}", GsonParser.rowPrimaryKeyToJson(token));
        if (request == null) {
            request = generateRequestItem(ots, conf, token, range.getEnd(), direction, columns);
        } else {
            RequestItem req = request;
            GetRangeResult result = RetryHelper.executeWithRetry(new GetRangeCallable(ots, req.getCriteria(), req.getFuture()), conf.getRetry(), conf.getSleepInMilliSecond());
            if ((token = result.getNextStartPrimaryKey()) != null) {
                request = generateRequestItem(ots, conf, token, range.getEnd(), direction, columns);
            }
            rowsToSender(result.getRows(), sender, conf.getColumns());
        }
    } while (token != null);
    ots.shutdown();
    LOG.info("read end.");
}