Examples with DataStoreConnection com.creditease.uav.datastore.api.DataStoreConnection used on opensource projects

Example 6 with DataStoreConnection

use of com.creditease.uav.datastore.api.DataStoreConnection in project uavstack by uavorg.

the class HBaseDataStore method query.

@SuppressWarnings({ "unchecked", "rawtypes" })
@Override
protected List query(DataStoreMsg msg) {
    log.info(this, "DataStore Query Log data start");
    msg = (DataStoreMsg) adaptor.prepareQueryObj(msg, datasource.getDataStoreConnection());
    // 根据TABLE名取得table实例
    String tableName = (String) msg.get(DataStoreProtocol.HBASE_TABLE_NAME);
    // 根据family名取得scaner实例
    String cfName = (String) msg.get(DataStoreProtocol.HBASE_FAMILY_NAME);
    Scan scan = null;
    List<byte[]> kv = null;
    try (Table table = datasource.getSourceConnect().getTable(TableName.valueOf(tableName))) {
        scan = new Scan();
        DataStoreConnection con = datasource.getDataStoreConnection();
        scan.setCaching(Integer.parseInt((String) con.getContext(DataStoreProtocol.HBASE_QUERY_CACHING)));
        scan.setMaxResultSize(Long.parseLong((String) con.getContext(DataStoreProtocol.HBASE_QUERY_MAXRESULTSIZE)));
        scan.setReversed((boolean) msg.get(DataStoreProtocol.HBASE_QUERY_REVERSE));
        scan.addFamily(cfName.getBytes("UTF-8"));
        // 根据查询信息构建实例,目前支持rowkey过滤
        FilterList flist = new FilterList(FilterList.Operator.MUST_PASS_ALL);
        if (msg.containsKey(DataStoreProtocol.HBASE_QUERY_STARTROW)) {
            scan.setStartRow((byte[]) msg.get(DataStoreProtocol.HBASE_QUERY_STARTROW));
        }
        if (msg.containsKey(DataStoreProtocol.HBASE_QUERY_ENDROW)) {
            scan.setStopRow((byte[]) msg.get(DataStoreProtocol.HBASE_QUERY_ENDROW));
        }
        if (msg.containsKey(DataStoreProtocol.HBASE_QUERY_ROW_KEYVALUE)) {
            kv = (List<byte[]>) msg.get(DataStoreProtocol.HBASE_QUERY_ROW_KEYVALUE);
            for (byte[] b : kv) {
                flist.addFilter(new RowFilter(CompareOp.EQUAL, new SubstringComparator(new String(b))));
            }
        }
        flist.addFilter(new PageFilter((long) msg.get(DataStoreProtocol.HBASE_QUERY_PAGESIZE)));
        scan.setFilter(flist);
        log.info(this, "DataStore Query Log data: getFilter String:" + scan.getFilter().toString());
        try (ResultScanner result = table.getScanner(scan)) {
            List<NavigableMap<byte[], byte[]>> resultList = Lists.newArrayList();
            for (Result r : result) {
                NavigableMap<byte[], byte[]> map = r.getFamilyMap(cfName.getBytes());
                map.put("_timestamp".getBytes(), String.valueOf(r.rawCells()[0].getTimestamp()).getBytes());
                resultList.add(map);
            }
            return adaptor.handleQueryResult(resultList, msg, datasource.getDataStoreConnection());
        }
    } catch (IOException e) {
        log.err(this, "QUERY HBASE TABLE[" + tableName + "] FAMILY[" + cfName + "] FAIL:" + msg.toJSONString(), e);
        return null;
    }
}

Example 7 with DataStoreConnection

use of com.creditease.uav.datastore.api.DataStoreConnection in project uavstack by uavorg.

the class DoTestLogData method testQueryHBase.

/**
 * { "starttime": 145629382438, "endtime": 145629382438, //optional "ip": "127.0.0.1", "svrid":
 * "D:/UAV/apache-tomcat-6.0.41::D:/eclipseProject/.metadata/.plugins/org.eclipse.wst.server.core/tmp0", "appid":
 * "sms" }
 */
@SuppressWarnings("unchecked")
public static void testQueryHBase() {
    DataStoreMsg msg = new DataStoreMsg();
    msg.put(DataStoreProtocol.HBASE_QUERY_JSON_KEY, queryJson);
    List<String> servers = DataConvertHelper.toList(zklist, ",");
    DataStoreConnection obj = new DataStoreConnection(null, null, null, servers, DataStoreType.HBASE);
    obj.putContext(DataStoreProtocol.HBASE_ZK_QUORUM, zklist);
    obj.putContext(DataStoreProtocol.HBASE_QUERY_CACHING, caching);
    obj.putContext(DataStoreProtocol.HBASE_QUERY_MAXRESULTSIZE, maxResultSize);
    AbstractDataStore<Connection> store = DataStoreFactory.getInstance().build(HealthManagerConstants.DataStore_Log, obj, new LogDataAdapter(), "");
    store.start();
    List<String> rst = store.doQuery(msg);
    store.stop();
    DataStoreUnitTest.printTestResult("testqueryHBase", rst, queryJson);
}

Example 8 with DataStoreConnection

use of com.creditease.uav.datastore.api.DataStoreConnection in project uavstack by uavorg.

the class DoTestLogData method testInsertHBase.

// {"MT_Log":"[{time:1462329007990,host:\"09-201211070016\",ip:\"127.0.0.1\",svrid:\"F:/testenv/apache-tomcat-7.0.65::E:/eclipse/workspace3/.metadata/.plugins/org.eclipse.wst.server.core/tmp1\",tag:\"L\",frames:{\"com.creditease.uav.monitorframework.buildFat,F:/temp/log/FAT_log1.txt\":
/**
 * [ { "time": 1456293824385, "host": "09-201509070105", "ip": "127.0.0.1", "svrid":
 * "D:/UAV/apache-tomcat-6.0.41::D:/eclipseProject/.metadata/.plugins/org.eclipse.wst.server.core/tmp0", "tag": "L",
 * "frames": { "WebTest": [ { "content": "[CE] aaaaa" } ] } } ]
 */
@SuppressWarnings("unchecked")
public static void testInsertHBase() {
    // MongoDBHandler
    DataStoreMsg msg = new DataStoreMsg();
    String rawData = DataStoreUnitTest.getData(insertJson);
    msg.put(MonitorDataFrame.MessageType.Log.toString(), rawData);
    msg.put(DataStoreProtocol.HBASE_TABLE_NAME, HealthManagerConstants.HBASE_TABLE_LOGDATA);
    List<String> servers = DataConvertHelper.toList(zklist, ",");
    DataStoreConnection obj = new DataStoreConnection(null, null, null, servers, DataStoreType.HBASE);
    obj.putContext(DataStoreProtocol.HBASE_ZK_QUORUM, zklist);
    obj.putContext(DataStoreProtocol.HBASE_QUERY_CACHING, caching);
    obj.putContext(DataStoreProtocol.HBASE_QUERY_MAXRESULTSIZE, maxResultSize);
    obj.putContext(DataStoreProtocol.HBASE_QUERY_REVERSE, true);
    obj.putContext(DataStoreProtocol.HBASE_QUERY_PAGESIZE, 3000);
    AbstractDataStore<Connection> store = DataStoreFactory.getInstance().build(HealthManagerConstants.DataStore_Log, obj, new LogDataAdapter(), "");
    store.start();
    boolean rst = store.doInsert(msg);
    store.stop();
    DataStoreUnitTest.printTestResult("testInsertHBase", rst);
}

Example 9 with DataStoreConnection

use of com.creditease.uav.datastore.api.DataStoreConnection in project uavstack by uavorg.

the class AppHubManager method buildDBService.

/**
 * 构建db服务
 *
 * @param dataStoreName
 * @param adapter
 */
private void buildDBService(String dataStoreName, DataStoreAdapter adapter) {
    // 验证是否可用
    Boolean check = Boolean.parseBoolean(this.getConfigManager().getFeatureConfiguration(this.feature, "ds.enable"));
    if (!check) {
        return;
    }
    String serverlist = this.getConfigManager().getFeatureConfiguration(this.feature, "ds.servers");
    String dbName = this.getConfigManager().getFeatureConfiguration(this.feature, "ds.db");
    String dbPower = this.getConfigManager().getFeatureConfiguration(this.feature, "ds.power");
    String userName = this.getConfigManager().getFeatureConfiguration(this.feature, "ds.usr");
    String password = this.getConfigManager().getFeatureConfiguration(this.feature, "ds.pwd");
    List<String> servers = DataConvertHelper.toList(serverlist, ",");
    DataStoreConnection obj_mongo = new DataStoreConnection(userName, password, dbName, dbPower, servers, DataStoreFactory.DataStoreType.MONGODB);
    DataStoreFactory.getInstance().build(dataStoreName, obj_mongo, adapter, this.feature);
}

Example 10 with DataStoreConnection

use of com.creditease.uav.datastore.api.DataStoreConnection in project uavstack by uavorg.

the class DoTestProfileData method testupdateMongoDB.

@SuppressWarnings({ "rawtypes", "unused" })
public static void testupdateMongoDB() {
    for (int i = 0; i < updatefileName.length; i++) {
        System.out.println("test update : " + updatefileName[i]);
        String update = DataStoreUnitTest.getData(queryJsonDir + updatefileName[i]);
        DataStoreMsg request = new DataStoreMsg();
        request.put(DataStoreProtocol.MONGO_QUERY_SQL, update.toString());
        request.put(DataStoreProtocol.MONGO_COLLECTION_NAME, HealthManagerConstants.MONGO_COLLECTION_PROFILE);
        DataStoreConnection obj = new DataStoreConnection(userName, password, dbName, serverlist, DataStoreType.MONGODB);
        AbstractDataStore store = DataStoreFactory.getInstance().build(HealthManagerConstants.DataStore_Profile, obj, new ProfileDataAdpater(), "");
        store.start();
        long start = System.currentTimeMillis();
        boolean rst = store.doUpdate(request);
        long stop = System.currentTimeMillis();
        StringBuffer sb = new StringBuffer();
        sb.append("update " + updatefileName[i] + ": " + (stop - start));
        try {
            fileWrite(sb);
        } catch (IOException e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
        }
        store.stop();
    // DataStoreUnitTest.printTestResult("MongoDB:"+updatefileName[i], rst);
    }
}