Examples with PrimaryKeyValue com.aliyun.openservices.ots.model.PrimaryKeyValue used on opensource projects

Example 6 with PrimaryKeyValue

use of com.aliyun.openservices.ots.model.PrimaryKeyValue in project DataX by alibaba.

the class OtsReaderMasterProxy method init.

/**
     * 1.检查参数是否为
     *     null，endpoint,accessid,accesskey,instance-name,table,column,range-begin,range-end,range-split
     * 2.检查参数是否为空字符串 
     *     endpoint,accessid,accesskey,instance-name,table
     * 3.检查是否为空数组
     *     column
     * 4.检查Range的类型个个数是否和PrimaryKey匹配
     *     column,range-begin,range-end
     * 5.检查Range Split 顺序和类型是否Range一致，类型是否于PartitionKey一致
     *     column-split
     * @param param
     * @throws Exception
     */
public void init(Configuration param) throws Exception {
    // 默认参数
    // 每次重试的时间都是上一次的一倍，当sleep时间大于30秒时，Sleep重试时间不在增长。18次能覆盖OTS的Failover时间5分钟
    conf.setRetry(param.getInt(OTSConst.RETRY, 18));
    conf.setSleepInMilliSecond(param.getInt(OTSConst.SLEEP_IN_MILLI_SECOND, 100));
    // 必选参数
    conf.setEndpoint(ParamChecker.checkStringAndGet(param, Key.OTS_ENDPOINT));
    conf.setAccessId(ParamChecker.checkStringAndGet(param, Key.OTS_ACCESSID));
    conf.setAccesskey(ParamChecker.checkStringAndGet(param, Key.OTS_ACCESSKEY));
    conf.setInstanceName(ParamChecker.checkStringAndGet(param, Key.OTS_INSTANCE_NAME));
    conf.setTableName(ParamChecker.checkStringAndGet(param, Key.TABLE_NAME));
    ots = new OTSClient(this.conf.getEndpoint(), this.conf.getAccessId(), this.conf.getAccesskey(), this.conf.getInstanceName());
    meta = getTableMeta(ots, conf.getTableName());
    LOG.info("Table Meta : {}", GsonParser.metaToJson(meta));
    conf.setColumns(ReaderModelParser.parseOTSColumnList(ParamChecker.checkListAndGet(param, Key.COLUMN, true)));
    Map<String, Object> rangeMap = ParamChecker.checkMapAndGet(param, Key.RANGE, true);
    conf.setRangeBegin(ReaderModelParser.parsePrimaryKey(ParamChecker.checkListAndGet(rangeMap, Key.RANGE_BEGIN, false)));
    conf.setRangeEnd(ReaderModelParser.parsePrimaryKey(ParamChecker.checkListAndGet(rangeMap, Key.RANGE_END, false)));
    range = ParamChecker.checkRangeAndGet(meta, this.conf.getRangeBegin(), this.conf.getRangeEnd());
    direction = ParamChecker.checkDirectionAndEnd(meta, range.getBegin(), range.getEnd());
    LOG.info("Direction : {}", direction);
    List<PrimaryKeyValue> points = ReaderModelParser.parsePrimaryKey(ParamChecker.checkListAndGet(rangeMap, Key.RANGE_SPLIT));
    ParamChecker.checkInputSplitPoints(meta, range, direction, points);
    conf.setRangeSplit(points);
}

Example 7 with PrimaryKeyValue

use of com.aliyun.openservices.ots.model.PrimaryKeyValue in project DataX by alibaba.

the class Common method compareRangeBeginAndEnd.

public static int compareRangeBeginAndEnd(TableMeta meta, RowPrimaryKey begin, RowPrimaryKey end) {
    if (begin.getPrimaryKey().size() != end.getPrimaryKey().size()) {
        throw new IllegalArgumentException("Input size of begin not equal size of end, begin size : " + begin.getPrimaryKey().size() + ", end size : " + end.getPrimaryKey().size() + ".");
    }
    for (String key : meta.getPrimaryKey().keySet()) {
        PrimaryKeyValue v1 = begin.getPrimaryKey().get(key);
        PrimaryKeyValue v2 = end.getPrimaryKey().get(key);
        int cmp = primaryKeyValueCmp(v1, v2);
        if (cmp != 0) {
            return cmp;
        }
    }
    return 0;
}

Example 8 with PrimaryKeyValue

use of com.aliyun.openservices.ots.model.PrimaryKeyValue in project DataX by alibaba.

the class RangeSplit method getSplitPoint.

/**
     * 根据输入的范围begin和end，从target中取得对应的point
     * @param begin
     * @param end
     * @param target
     * @return
     */
public static List<PrimaryKeyValue> getSplitPoint(PrimaryKeyValue begin, PrimaryKeyValue end, List<PrimaryKeyValue> target) {
    List<PrimaryKeyValue> result = new ArrayList<PrimaryKeyValue>();
    int cmp = Common.primaryKeyValueCmp(begin, end);
    if (cmp == 0) {
        return result;
    }
    result.add(begin);
    Comparator<PrimaryKeyValue> comparator = new Comparator<PrimaryKeyValue>() {

        public int compare(PrimaryKeyValue arg0, PrimaryKeyValue arg1) {
            return Common.primaryKeyValueCmp(arg0, arg1);
        }
    };
    if (cmp > 0) {
        // 如果是逆序，则 reverse Comparator
        comparator = Collections.reverseOrder(comparator);
    }
    Collections.sort(target, comparator);
    for (PrimaryKeyValue value : target) {
        if (comparator.compare(value, begin) > 0 && comparator.compare(value, end) < 0) {
            result.add(value);
        }
    }
    result.add(end);
    return result;
}

Example 9 with PrimaryKeyValue

use of com.aliyun.openservices.ots.model.PrimaryKeyValue in project DataX by alibaba.

the class RangeSplit method rangeSplitByPoint.

public static List<OTSRange> rangeSplitByPoint(TableMeta meta, RowPrimaryKey beginPK, RowPrimaryKey endPK, List<PrimaryKeyValue> splits) {
    List<OTSRange> results = new ArrayList<OTSRange>();
    int pkCount = meta.getPrimaryKey().size();
    String partName = Common.getPartitionKey(meta).getName();
    PrimaryKeyValue begin = beginPK.getPrimaryKey().get(partName);
    PrimaryKeyValue end = endPK.getPrimaryKey().get(partName);
    List<PrimaryKeyValue> newSplits = getSplitPoint(begin, end, splits);
    if (newSplits.isEmpty()) {
        return results;
    }
    for (int i = 0; i < newSplits.size() - 1; i++) {
        OTSRange item = new OTSRange(ParamChecker.checkInputPrimaryKeyAndGet(meta, getCompletePK(pkCount, newSplits.get(i))), ParamChecker.checkInputPrimaryKeyAndGet(meta, getCompletePK(pkCount, newSplits.get(i + 1))));
        results.add(item);
    }
    // replace first and last
    OTSRange first = results.get(0);
    OTSRange last = results.get(results.size() - 1);
    first.setBegin(beginPK);
    last.setEnd(endPK);
    return results;
}