Search in sources :

Example 1 with ServerJoinGroup

use of com.qlangtech.tis.manage.servlet.ServerJoinGroup in project tis by qlangtech.

the class CollectionAction method doGetIndexTopology.

/**
 * @param context
 * @throws Exception
 */
public void doGetIndexTopology(Context context) throws Exception {
    this.getIndexWithPost();
    JSONObject biz = new JSONObject();
    CollectionTopology topology = CoreAction.getCollectionTopology(this);
    JSONArray shards = new JSONArray();
    JSONObject shard = null;
    JSONArray replics = null;
    JSONObject replic = null;
    for (CollectionTopology.Shared s : topology.getShareds()) {
        shard = new JSONObject();
        replics = new JSONArray();
        shard.put(KEY_SHARD_NAME, s.getName());
        for (Replica r : s.getReplics()) {
            replic = new JSONObject();
            replic.put(KEY_CORE_URL, r.getCoreUrl());
            replic.put(KEY_IS_ACTIVE, r.getState() == Replica.State.ACTIVE);
            replics.add(replic);
        }
        shard.put(KEY_REPLICS, replics);
        shards.add(shard);
    }
    biz.put(SqlTaskNodeMeta.KEY_PROFILE_TOPOLOGY, shards);
    SnapshotDomain snapshot = ViewPojo.getSnapshotDoamin(this, this.getAppDomain());
    ParseResult parseResult = SolrFieldsParser.parse(() -> {
        return snapshot.getSolrSchema().getContent();
    }).getSchemaParseResult();
    biz.put(KEY_PK, parseResult.getUniqueKey());
    QueryResutStrategy queryStrategy = QueryIndexServlet.createQueryResutStrategy(this.getAppDomain(), new IndexQuery.QueryRequestWrapper(getRequest(), context), getResponse(), getDaoContext());
    List<ServerJoinGroup> nodes = queryStrategy.queryProcess();
    List<PSchemaField> sfields = IndexQuery.getSfields(this.getRequest(), queryStrategy, nodes);
    JSONArray colsMeta = new JSONArray();
    JSONObject colmeta = null;
    for (PSchemaField field : sfields) {
        colmeta = new JSONObject();
        colmeta.put("name", field.getName());
        colmeta.put("typeName", field.getType().getJavaType().getSimpleName());
        colmeta.put("typeCode", field.getType().getJavaType().getTypeCode());
        colmeta.put("docval", field.isDocValue());
        colmeta.put("indexd", field.isIndexed());
        colmeta.put("stored", field.isStored());
        colmeta.put("dynamic", field.isDynamic());
        colsMeta.add(colmeta);
    }
    biz.put(KEY_COLS_META, colsMeta);
    this.setBizResult(context, biz);
}
Also used : QueryResutStrategy(com.qlangtech.tis.manage.servlet.QueryResutStrategy) IndexQuery(com.qlangtech.tis.runtime.module.screen.IndexQuery) ParseResult(com.qlangtech.tis.solrdao.impl.ParseResult) PSchemaField(com.qlangtech.tis.solrdao.pojo.PSchemaField) JSONArray(com.alibaba.fastjson.JSONArray) ServerJoinGroup(com.qlangtech.tis.manage.servlet.ServerJoinGroup) Replica(org.apache.solr.common.cloud.Replica) JSONObject(com.alibaba.fastjson.JSONObject)

Example 2 with ServerJoinGroup

use of com.qlangtech.tis.manage.servlet.ServerJoinGroup in project tis by qlangtech.

the class CollectionAction method doQuery.

/**
 * 利用solr的disMax QP进行查询
 *
 * @param context
 * @throws Exception
 */
public void doQuery(Context context) throws Exception {
    // this.parseJsonPost();
    JSONObject post = getIndexWithPost();
    // if (StringUtils.isEmpty(post.getString(KEY_INDEX_NAME))) {
    // throw new IllegalArgumentException("indexName can not be null");
    // }
    // this.indexName = TISCollectionUtils.NAME_PREFIX + post.getString(KEY_INDEX_NAME);
    JSONArray searchFields = post.getJSONArray(KEY_QUERY_SEARCH_FIELDS);
    Objects.requireNonNull(searchFields, "param " + KEY_QUERY_SEARCH_FIELDS + " can not be null ");
    if (searchFields.size() < 1) {
        throw new IllegalArgumentException(KEY_QUERY_SEARCH_FIELDS + " relevant field can not be empty");
    }
    final List<SubCriteria> andQueryCriteria = Lists.newArrayList();
    searchFields.forEach((f) -> {
        SubCriteria subCriteria = new SubCriteria();
        JSONObject o = (JSONObject) f;
        Option opt = null;
        String word = null;
        for (String key : o.keySet()) {
            word = o.getString(key);
            if (StringUtils.isEmpty(word)) {
                throw new IllegalArgumentException("query field:" + key + ",relevant:" + word + ",val can not be null");
            }
            opt = new Option(key, word);
            subCriteria.addOr(opt);
        }
        andQueryCriteria.add(subCriteria);
    });
    JSONArray storedFields = post.getJSONArray(KEY_QUERY_FIELDS);
    if (storedFields == null) {
        throw new IllegalArgumentException("param 'fields' can not be null");
    }
    storedFields.stream().map((r) -> (String) r).collect(Collectors.toList());
    // final String fields = post.getString(KEY_QUERY_FIELDS);
    // if (StringUtils.isEmpty(fields)) {
    // throw new IllegalArgumentException("param 'fields' can not be null");
    // }
    final Integer limit = post.getInteger(KEY_QUERY_LIMIT);
    if (limit == null) {
        throw new IllegalArgumentException("param limit can not be null");
    }
    // final String queryFields = post.getString(KEY_QUERY_QUERY_FIELDS);
    // if (StringUtils.isEmpty(queryFields)) {
    // throw new IllegalArgumentException("'queryFields' can not be null");
    // }
    final String orderBy = post.getString(KEY_QUERY_ORDER_BY);
    Integer rowsOffset = post.getInteger(KEY_QUERY_ROWS_OFFSET);
    AppDomainInfo app = getAppDomain();
    final QueryResutStrategy queryResutStrategy = QueryIndexServlet.createQueryResutStrategy(app, this.getRequest(), getResponse(), getDaoContext());
    final List<ServerJoinGroup> serverlist = queryResutStrategy.queryProcess();
    for (ServerJoinGroup server : serverlist) {
        // 组装url
        final String url = server.getIpAddress();
        QueryCloudSolrClient solrClient = new QueryCloudSolrClient(url);
        SolrQuery query = new SolrQuery();
        query.set(CommonParams.FL, storedFields.stream().map((r) -> (String) r).collect(Collectors.joining(",")));
        // query.setParam(QUERY_PARSIING_DEF_TYPE, "dismax");
        // query.setParam(DisMaxParams.QF, queryFields);
        query.setQuery(this.createQuery(andQueryCriteria));
        query.setRows(limit);
        if (rowsOffset != null) {
            query.setStart(rowsOffset);
        }
        if (StringUtils.isNotEmpty(orderBy)) {
            query.add(CommonParams.SORT, orderBy);
        }
        QueryResponse result = solrClient.query(indexName.getCollectionName(), query, SolrRequest.METHOD.POST);
        solrClient.close();
        Map<String, Object> biz = Maps.newHashMap();
        long c = result.getResults().getNumFound();
        biz.put(RESULT_KEY_ROWS_COUNT, c);
        List<Map<String, Object>> resultList = Lists.newArrayList();
        Map<String, Object> row = null;
        for (SolrDocument doc : result.getResults()) {
            row = Maps.newHashMap();
            for (Map.Entry<String, Object> f : doc.entrySet()) {
                row.put(f.getKey(), f.getValue());
            }
            resultList.add(row);
        }
        biz.put(RESULT_KEY_ROWS, resultList);
        this.setBizResult(context, biz);
        return;
    }
}
Also used : OfflineDatasourceAction(com.qlangtech.tis.offline.module.action.OfflineDatasourceAction) StringUtils(org.apache.commons.lang.StringUtils) PluginStore(com.qlangtech.tis.plugin.PluginStore) SqlTaskNode(com.qlangtech.tis.sql.parser.SqlTaskNode) PSchemaField(com.qlangtech.tis.solrdao.pojo.PSchemaField) URL(java.net.URL) TIS(com.qlangtech.tis.TIS) LoggerFactory(org.slf4j.LoggerFactory) ReplicasSpec(com.qlangtech.tis.config.k8s.ReplicasSpec) Autowired(org.springframework.beans.factory.annotation.Autowired) LogCollectorClient(com.qlangtech.tis.rpc.grpc.log.LogCollectorClient) CreateIndexConfirmModel(com.qlangtech.tis.runtime.module.action.CreateIndexConfirmModel) SnapshotViewImplDAO(com.qlangtech.tis.manage.biz.dal.dao.impl.SnapshotViewImplDAO) StreamObserver(io.grpc.stub.StreamObserver) StatusRpcClient(com.tis.hadoop.rpc.StatusRpcClient) LogType(com.qlangtech.tis.trigger.socket.LogType) ActionContext(com.opensymphony.xwork2.ActionContext) SelectableServer(com.qlangtech.tis.coredefine.module.control.SelectableServer) IPluginStore(com.qlangtech.tis.plugin.IPluginStore) SchemaAction(com.qlangtech.tis.runtime.module.action.SchemaAction) DependencyNode(com.qlangtech.tis.sql.parser.meta.DependencyNode) IncrStreamFactory(com.qlangtech.tis.plugin.incr.IncrStreamFactory) com.qlangtech.tis.manage.common(com.qlangtech.tis.manage.common) SysInitializeAction(com.qlangtech.tis.runtime.module.action.SysInitializeAction) Descriptor(com.qlangtech.tis.extension.Descriptor) IndexQuery(com.qlangtech.tis.runtime.module.screen.IndexQuery) Collectors(java.util.stream.Collectors) Replica(org.apache.solr.common.cloud.Replica) ViewPojo(com.qlangtech.tis.runtime.module.screen.ViewPojo) PlatformTransactionManager(org.springframework.transaction.PlatformTransactionManager) SolrQuery(org.apache.solr.client.solrj.SolrQuery) QueryCloudSolrClient(com.qlangtech.tis.manage.servlet.QueryCloudSolrClient) RunEnvironment(com.qlangtech.tis.pubhook.common.RunEnvironment) ISchemaPluginContext(com.qlangtech.tis.solrdao.ISchemaPluginContext) TransactionStatus(org.springframework.transaction.TransactionStatus) JSONObject(com.alibaba.fastjson.JSONObject) DBConfigSuit(com.qlangtech.tis.db.parser.DBConfigSuit) com.qlangtech.tis.workflow.pojo(com.qlangtech.tis.workflow.pojo) IMessageHandler(com.qlangtech.tis.runtime.module.misc.IMessageHandler) ILogListener(com.qlangtech.tis.trigger.jst.ILogListener) SolrRequest(org.apache.solr.client.solrj.SolrRequest) ISchemaField(com.qlangtech.tis.solrdao.ISchemaField) ParseResult(com.qlangtech.tis.solrdao.impl.ParseResult) java.util(java.util) SolrFieldsParser(com.qlangtech.tis.solrdao.SolrFieldsParser) QueryIndexServlet(com.qlangtech.tis.manage.servlet.QueryIndexServlet) Context(com.alibaba.citrus.turbine.Context) PMonotorTarget(com.qlangtech.tis.rpc.grpc.log.stream.PMonotorTarget) JSONArray(com.alibaba.fastjson.JSONArray) RpcServiceReference(com.tis.hadoop.rpc.RpcServiceReference) Lists(com.google.common.collect.Lists) QueryResutStrategy(com.qlangtech.tis.manage.servlet.QueryResutStrategy) com.qlangtech.tis.coredefine.module.action(com.qlangtech.tis.coredefine.module.action) PExecuteState(com.qlangtech.tis.rpc.grpc.log.stream.PExecuteState) com.qlangtech.tis.util(com.qlangtech.tis.util) OfflineManager(com.qlangtech.tis.offline.module.manager.impl.OfflineManager) Application(com.qlangtech.tis.manage.biz.dal.pojo.Application) Position(com.qlangtech.tis.sql.parser.meta.Position) Logger(org.slf4j.Logger) QueryResponse(org.apache.solr.client.solrj.response.QueryResponse) NodeType(com.qlangtech.tis.sql.parser.meta.NodeType) com.qlangtech.tis.plugin.ds(com.qlangtech.tis.plugin.ds) FileUtils(org.apache.commons.io.FileUtils) IOException(java.io.IOException) Maps(com.google.common.collect.Maps) File(java.io.File) CommonParams(org.apache.solr.common.params.CommonParams) SchemaResult(com.qlangtech.tis.solrdao.SchemaResult) SolrDocument(org.apache.solr.common.SolrDocument) IParamContext(com.qlangtech.tis.order.center.IParamContext) ERRules(com.qlangtech.tis.sql.parser.er.ERRules) ServerJoinGroup(com.qlangtech.tis.manage.servlet.ServerJoinGroup) SqlTaskNodeMeta(com.qlangtech.tis.sql.parser.SqlTaskNodeMeta) InputStream(java.io.InputStream) QueryResutStrategy(com.qlangtech.tis.manage.servlet.QueryResutStrategy) JSONArray(com.alibaba.fastjson.JSONArray) ServerJoinGroup(com.qlangtech.tis.manage.servlet.ServerJoinGroup) SolrQuery(org.apache.solr.client.solrj.SolrQuery) SolrDocument(org.apache.solr.common.SolrDocument) JSONObject(com.alibaba.fastjson.JSONObject) QueryResponse(org.apache.solr.client.solrj.response.QueryResponse) JSONObject(com.alibaba.fastjson.JSONObject) QueryCloudSolrClient(com.qlangtech.tis.manage.servlet.QueryCloudSolrClient)

Aggregations

JSONArray (com.alibaba.fastjson.JSONArray)2 JSONObject (com.alibaba.fastjson.JSONObject)2 QueryResutStrategy (com.qlangtech.tis.manage.servlet.QueryResutStrategy)2 ServerJoinGroup (com.qlangtech.tis.manage.servlet.ServerJoinGroup)2 IndexQuery (com.qlangtech.tis.runtime.module.screen.IndexQuery)2 ParseResult (com.qlangtech.tis.solrdao.impl.ParseResult)2 PSchemaField (com.qlangtech.tis.solrdao.pojo.PSchemaField)2 Replica (org.apache.solr.common.cloud.Replica)2 Context (com.alibaba.citrus.turbine.Context)1 Lists (com.google.common.collect.Lists)1 Maps (com.google.common.collect.Maps)1 ActionContext (com.opensymphony.xwork2.ActionContext)1 TIS (com.qlangtech.tis.TIS)1 ReplicasSpec (com.qlangtech.tis.config.k8s.ReplicasSpec)1 com.qlangtech.tis.coredefine.module.action (com.qlangtech.tis.coredefine.module.action)1 SelectableServer (com.qlangtech.tis.coredefine.module.control.SelectableServer)1 DBConfigSuit (com.qlangtech.tis.db.parser.DBConfigSuit)1 Descriptor (com.qlangtech.tis.extension.Descriptor)1 SnapshotViewImplDAO (com.qlangtech.tis.manage.biz.dal.dao.impl.SnapshotViewImplDAO)1 Application (com.qlangtech.tis.manage.biz.dal.pojo.Application)1