Examples with OTSClient com.aliyun.openservices.ots.OTSClient used on opensource projects

Example 1 with OTSClient

use of com.aliyun.openservices.ots.OTSClient in project DataX by alibaba.

the class OtsWriterMasterProxy method init.

/**
     * @param param
     * @throws Exception
     */
public void init(Configuration param) throws Exception {
    // 默认参数
    conf.setRetry(param.getInt(OTSConst.RETRY, 18));
    conf.setSleepInMillisecond(param.getInt(OTSConst.SLEEP_IN_MILLISECOND, 100));
    conf.setBatchWriteCount(param.getInt(OTSConst.BATCH_WRITE_COUNT, 100));
    conf.setConcurrencyWrite(param.getInt(OTSConst.CONCURRENCY_WRITE, 5));
    conf.setIoThreadCount(param.getInt(OTSConst.IO_THREAD_COUNT, 1));
    conf.setSocketTimeout(param.getInt(OTSConst.SOCKET_TIMEOUT, 20000));
    conf.setConnectTimeout(param.getInt(OTSConst.CONNECT_TIMEOUT, 10000));
    conf.setBufferSize(param.getInt(OTSConst.BUFFER_SIZE, 1024));
    RestrictConf restrictConf = conf.new RestrictConf();
    restrictConf.setRequestTotalSizeLimition(param.getInt(OTSConst.REQUEST_TOTAL_SIZE_LIMITATION, 1024 * 1024));
    restrictConf.setAttributeColumnSize(param.getInt(OTSConst.ATTRIBUTE_COLUMN_SIZE_LIMITATION, 2 * 1024 * 1024));
    restrictConf.setPrimaryKeyColumnSize(param.getInt(OTSConst.PRIMARY_KEY_COLUMN_SIZE_LIMITATION, 1024));
    restrictConf.setMaxColumnsCount(param.getInt(OTSConst.ATTRIBUTE_COLUMN_MAX_COUNT, 1024));
    conf.setRestrictConf(restrictConf);
    // 必选参数
    conf.setEndpoint(ParamChecker.checkStringAndGet(param, Key.OTS_ENDPOINT));
    conf.setAccessId(ParamChecker.checkStringAndGet(param, Key.OTS_ACCESSID));
    conf.setAccessKey(ParamChecker.checkStringAndGet(param, Key.OTS_ACCESSKEY));
    conf.setInstanceName(ParamChecker.checkStringAndGet(param, Key.OTS_INSTANCE_NAME));
    conf.setTableName(ParamChecker.checkStringAndGet(param, Key.TABLE_NAME));
    conf.setOperation(WriterModelParser.parseOTSOpType(ParamChecker.checkStringAndGet(param, Key.WRITE_MODE)));
    ots = new OTSClient(this.conf.getEndpoint(), this.conf.getAccessId(), this.conf.getAccessKey(), this.conf.getInstanceName());
    meta = getTableMeta(ots, conf.getTableName());
    LOG.info("Table Meta : {}", GsonParser.metaToJson(meta));
    conf.setPrimaryKeyColumn(WriterModelParser.parseOTSPKColumnList(ParamChecker.checkListAndGet(param, Key.PRIMARY_KEY, true)));
    ParamChecker.checkPrimaryKey(meta, conf.getPrimaryKeyColumn());
    conf.setAttributeColumn(WriterModelParser.parseOTSAttrColumnList(ParamChecker.checkListAndGet(param, Key.COLUMN, conf.getOperation() == OTSOpType.UPDATE_ROW ? true : false)));
    ParamChecker.checkAttribute(conf.getAttributeColumn());
}

Example 2 with OTSClient

use of com.aliyun.openservices.ots.OTSClient in project DataX by alibaba.

the class OtsReaderMasterProxy method init.

/**
     * 1.检查参数是否为
     *     null，endpoint,accessid,accesskey,instance-name,table,column,range-begin,range-end,range-split
     * 2.检查参数是否为空字符串 
     *     endpoint,accessid,accesskey,instance-name,table
     * 3.检查是否为空数组
     *     column
     * 4.检查Range的类型个个数是否和PrimaryKey匹配
     *     column,range-begin,range-end
     * 5.检查Range Split 顺序和类型是否Range一致，类型是否于PartitionKey一致
     *     column-split
     * @param param
     * @throws Exception
     */
public void init(Configuration param) throws Exception {
    // 默认参数
    // 每次重试的时间都是上一次的一倍，当sleep时间大于30秒时，Sleep重试时间不在增长。18次能覆盖OTS的Failover时间5分钟
    conf.setRetry(param.getInt(OTSConst.RETRY, 18));
    conf.setSleepInMilliSecond(param.getInt(OTSConst.SLEEP_IN_MILLI_SECOND, 100));
    // 必选参数
    conf.setEndpoint(ParamChecker.checkStringAndGet(param, Key.OTS_ENDPOINT));
    conf.setAccessId(ParamChecker.checkStringAndGet(param, Key.OTS_ACCESSID));
    conf.setAccesskey(ParamChecker.checkStringAndGet(param, Key.OTS_ACCESSKEY));
    conf.setInstanceName(ParamChecker.checkStringAndGet(param, Key.OTS_INSTANCE_NAME));
    conf.setTableName(ParamChecker.checkStringAndGet(param, Key.TABLE_NAME));
    ots = new OTSClient(this.conf.getEndpoint(), this.conf.getAccessId(), this.conf.getAccesskey(), this.conf.getInstanceName());
    meta = getTableMeta(ots, conf.getTableName());
    LOG.info("Table Meta : {}", GsonParser.metaToJson(meta));
    conf.setColumns(ReaderModelParser.parseOTSColumnList(ParamChecker.checkListAndGet(param, Key.COLUMN, true)));
    Map<String, Object> rangeMap = ParamChecker.checkMapAndGet(param, Key.RANGE, true);
    conf.setRangeBegin(ReaderModelParser.parsePrimaryKey(ParamChecker.checkListAndGet(rangeMap, Key.RANGE_BEGIN, false)));
    conf.setRangeEnd(ReaderModelParser.parsePrimaryKey(ParamChecker.checkListAndGet(rangeMap, Key.RANGE_END, false)));
    range = ParamChecker.checkRangeAndGet(meta, this.conf.getRangeBegin(), this.conf.getRangeEnd());
    direction = ParamChecker.checkDirectionAndEnd(meta, range.getBegin(), range.getEnd());
    LOG.info("Direction : {}", direction);
    List<PrimaryKeyValue> points = ReaderModelParser.parsePrimaryKey(ParamChecker.checkListAndGet(rangeMap, Key.RANGE_SPLIT));
    ParamChecker.checkInputSplitPoints(meta, range, direction, points);
    conf.setRangeSplit(points);
}