Examples with JSONList net.yacy.grid.tools.JSONList used on opensource projects

Example 1 with JSONList

use of net.yacy.grid.tools.JSONList in project yacy_grid_mcp by yacy.

the class QueryService method serviceImpl.

@Override
public ServiceResponse serviceImpl(Query call, HttpServletResponse response) {
    // String indexName, String typeName, final String id, JSONObject object
    String indexName = call.get("index", "");
    String typeName = call.get("type", "");
    if (typeName.length() == 0)
        typeName = null;
    String id = call.get("id", "");
    QueryLanguage language = QueryLanguage.valueOf(call.get("language", "yacy"));
    String query = call.get("query", "");
    int maximumRecords = call.get("maximumRecords", call.get("rows", call.get("num", 10)));
    int startRecord = call.get("startRecord", call.get("start", 0));
    JSONObject json = new JSONObject(true);
    if (indexName.length() > 0 && id.length() > 0) {
        try {
            Index index = Data.gridIndex.getElasticIndex();
            String url = index.checkConnection().getConnectionURL();
            JSONObject object = index.query(indexName, typeName, id);
            json.put(ObjectAPIHandler.SUCCESS_KEY, true);
            JSONList list = new JSONList();
            if (object != null)
                list.add(object);
            json.put("count", list.length());
            json.put("list", list.toArray());
            if (url != null)
                json.put(ObjectAPIHandler.SERVICE_KEY, url);
        } catch (IOException e) {
            json.put(ObjectAPIHandler.SUCCESS_KEY, false);
            json.put(ObjectAPIHandler.COMMENT_KEY, e.getMessage());
        }
    } else if (indexName.length() > 0 && query.length() > 0) {
        try {
            Index index = Data.gridIndex.getElasticIndex();
            String url = index.checkConnection().getConnectionURL();
            JSONList list = index.query(indexName, typeName, language, query, startRecord, maximumRecords);
            json.put(ObjectAPIHandler.SUCCESS_KEY, true);
            json.put("count", list.length());
            json.put("list", list.toArray());
            if (url != null)
                json.put(ObjectAPIHandler.SERVICE_KEY, url);
        } catch (IOException e) {
            json.put(ObjectAPIHandler.SUCCESS_KEY, false);
            json.put(ObjectAPIHandler.COMMENT_KEY, e.getMessage());
        }
    } else {
        json.put(ObjectAPIHandler.SUCCESS_KEY, false);
        json.put(ObjectAPIHandler.COMMENT_KEY, "the request must contain an index, type, and either an id or a query");
    }
    return new ServiceResponse(json);
}

Example 2 with JSONList

use of net.yacy.grid.tools.JSONList in project yacy_grid_mcp by yacy.

the class MCPIndexFactory method getIndex.

@Override
public Index getIndex() throws IOException {
    final JSONObject params = new JSONObject(true);
    return new Index() {

        @Override
        public IndexFactory checkConnection() throws IOException {
            String protocolhostportstub = MCPIndexFactory.this.getConnectionURL();
            APIHandler apiHandler = APIServer.getAPI(CheckService.NAME);
            ServiceResponse sr = apiHandler.serviceImpl(protocolhostportstub, params);
            if (!success(sr.getObject()))
                throw new IOException("MCP does not respond properly");
            return MCPIndexFactory.this;
        }

        private JSONObject getResponse(APIHandler handler) throws IOException {
            String protocolhostportstub = MCPIndexFactory.this.getConnectionURL();
            ServiceResponse sr = handler.serviceImpl(protocolhostportstub, params);
            return sr.getObject();
        }

        private boolean success(JSONObject response) {
            return response.has(ObjectAPIHandler.SUCCESS_KEY) && response.getBoolean(ObjectAPIHandler.SUCCESS_KEY);
        }

        private void connectMCP(JSONObject response) {
            if (response.has(ObjectAPIHandler.SERVICE_KEY)) {
                String elastic = response.getString(ObjectAPIHandler.SERVICE_KEY);
                if (MCPIndexFactory.this.index.connectElasticsearch(elastic)) {
                    Data.logger.info("connected MCP index at " + elastic);
                } else {
                    Data.logger.error("failed to connect MCP index at " + elastic);
                }
            }
        }

        private IOException handleError(JSONObject response) {
            if (response.has(ObjectAPIHandler.COMMENT_KEY)) {
                return new IOException("cannot connect to MCP: " + response.getString(ObjectAPIHandler.COMMENT_KEY));
            }
            return new IOException("bad response from MCP: no success and no comment key");
        }

        @Override
        public IndexFactory add(String indexName, String typeName, String id, JSONObject object) throws IOException {
            params.put("index", indexName);
            params.put("type", typeName);
            params.put("id", id);
            params.put("object", object.toString());
            JSONObject response = getResponse(APIServer.getAPI(AddService.NAME));
            // read the broker to store the service definition of the remote queue, if exists
            if (success(response)) {
                connectMCP(response);
                return MCPIndexFactory.this;
            } else {
                throw handleError(response);
            }
        }

        @Override
        public boolean exist(String indexName, String typeName, String id) throws IOException {
            params.put("index", indexName);
            params.put("type", typeName);
            params.put("id", id);
            JSONObject response = getResponse(APIServer.getAPI(ExistService.NAME));
            // read the broker to store the service definition of the remote queue, if exists
            if (success(response)) {
                connectMCP(response);
                return response.has("exists") && response.getBoolean("exists");
            } else {
                throw handleError(response);
            }
        }

        @Override
        public long count(String indexName, String typeName, QueryLanguage language, String query) throws IOException {
            params.put("index", indexName);
            params.put("type", typeName);
            params.put("language", language.name());
            params.put("query", query);
            JSONObject response = getResponse(APIServer.getAPI(CountService.NAME));
            // read the broker to store the service definition of the remote queue, if exists
            if (success(response)) {
                connectMCP(response);
                return response.has("count") ? response.getLong("count") : 0;
            } else {
                throw handleError(response);
            }
        }

        @Override
        public JSONObject query(String indexName, String typeName, String id) throws IOException {
            params.put("index", indexName);
            params.put("type", typeName);
            params.put("id", id);
            JSONObject response = getResponse(APIServer.getAPI(QueryService.NAME));
            // read the broker to store the service definition of the remote queue, if exists
            if (success(response)) {
                connectMCP(response);
                if (!response.has("list"))
                    return null;
                JSONArray list = response.getJSONArray("list");
                if (list.length() == 0)
                    return null;
                return list.getJSONObject(0);
            } else {
                throw handleError(response);
            }
        }

        @Override
        public JSONList query(String indexName, String typeName, QueryLanguage language, String query, int start, int count) throws IOException {
            params.put("index", indexName);
            params.put("type", typeName);
            params.put("language", language.name());
            params.put("query", query);
            JSONObject response = getResponse(APIServer.getAPI(QueryService.NAME));
            // read the broker to store the service definition of the remote queue, if exists
            if (success(response)) {
                connectMCP(response);
                JSONList list = new JSONList();
                if (!response.has("list"))
                    return list;
                JSONArray l = response.getJSONArray("list");
                if (l.length() == 0)
                    return list;
                for (int i = 0; i < l.length(); i++) list.add(l.getJSONObject(i));
                return list;
            } else {
                throw handleError(response);
            }
        }

        @Override
        public boolean delete(String indexName, String typeName, String id) throws IOException {
            params.put("index", indexName);
            params.put("type", typeName);
            params.put("id", id);
            JSONObject response = getResponse(APIServer.getAPI(DeleteService.NAME));
            // read the broker to store the service definition of the remote queue, if exists
            if (success(response)) {
                connectMCP(response);
                return response.has("deleted") && response.getBoolean("deleted");
            } else {
                throw handleError(response);
            }
        }

        @Override
        public long delete(String indexName, String typeName, QueryLanguage language, String query) throws IOException {
            params.put("index", indexName);
            params.put("type", typeName);
            params.put("language", language.name());
            params.put("query", query);
            JSONObject response = getResponse(APIServer.getAPI(DeleteService.NAME));
            // read the broker to store the service definition of the remote queue, if exists
            if (success(response)) {
                connectMCP(response);
                return response.has("count") ? response.getLong("count") : 0;
            } else {
                throw handleError(response);
            }
        }

        @Override
        public void close() {
        }
    };
}

Example 3 with JSONList

use of net.yacy.grid.tools.JSONList in project yacy_grid_mcp by yacy.

the class SusiAction method getJSONListAsset.

public JSONList getJSONListAsset(String name) {
    if (!this.json.has("assets"))
        return null;
    JSONObject assets = this.json.getJSONObject("assets");
    JSONArray jsonlist = assets.getJSONArray(name);
    try {
        return new JSONList(jsonlist);
    } catch (IOException e) {
        Data.logger.warn("error in getJSONListAsset with name " + name, e);
        return null;
    }
}

Example 4 with JSONList

use of net.yacy.grid.tools.JSONList in project yacy_grid_crawler by yacy.

the class CrawlStartService method serviceImpl.

@Override
public ServiceResponse serviceImpl(Query call, HttpServletResponse response) {
    JSONObject crawlstart = CrawlerDefaultValuesService.crawlStartDefaultClone();
    for (String key : crawlstart.keySet()) {
        Object object = crawlstart.get(key);
        if (object instanceof String)
            crawlstart.put(key, call.get(key, crawlstart.getString(key)));
        else if (object instanceof Integer)
            crawlstart.put(key, call.get(key, crawlstart.getInt(key)));
        else if (object instanceof Long)
            crawlstart.put(key, call.get(key, crawlstart.getLong(key)));
        else if (object instanceof JSONArray) {
            JSONArray a = crawlstart.getJSONArray(key);
            Object cv = call.get(key);
            if (cv != null)
                crawlstart.put(key, cv);
        } else {
            System.out.println("unrecognized type: " + object.getClass().toString());
        }
    }
    // set the crawl id
    CrawlstartURLSplitter crawlstartURLs = new CrawlstartURLSplitter(crawlstart.getString("crawlingURL"));
    Date now = new Date();
    // start the crawls; each of the url in a separate crawl to enforce parallel loading from different hosts
    SusiThought allCrawlstarts = new SusiThought();
    int count = 0;
    for (MultiProtocolURL url : crawlstartURLs.getURLs()) {
        JSONObject singlecrawl = new JSONObject();
        // create a clone of crawlstart
        for (String key : crawlstart.keySet()) singlecrawl.put(key, crawlstart.get(key));
        singlecrawl.put("id", Crawler.getCrawlID(url, now, count++));
        try {
            GridQueue queueName = Data.gridBroker.queueName(YaCyServices.crawler, YaCyServices.crawler.getQueues(), ShardingMethod.BALANCE, Crawler.CRAWLER_PRIORITY_DIMENSIONS, singlecrawl.getInt("priority"), url.getHost());
            SusiThought json = new SusiThought();
            json.setData(new JSONArray().put(singlecrawl));
            JSONObject action = new JSONObject().put("type", YaCyServices.crawler.name()).put("queue", queueName.name()).put("id", singlecrawl.getString("id")).put("depth", 0).put("sourcegraph", "rootasset");
            SusiAction crawlAction = new SusiAction(action);
            JSONObject graph = new JSONObject(true).put(WebMapping.canonical_s.getMapping().name(), url.toNormalform(true));
            crawlAction.setJSONListAsset("rootasset", new JSONList().add(graph));
            json.addAction(crawlAction);
            allCrawlstarts.addAction(crawlAction);
            byte[] b = json.toString().getBytes(StandardCharsets.UTF_8);
            Data.gridBroker.send(YaCyServices.crawler, queueName, b);
        } catch (IOException e) {
            Data.logger.warn("error when starting crawl for " + url.toNormalform(true), e);
            allCrawlstarts.put(ObjectAPIHandler.COMMENT_KEY, e.getMessage());
        }
    }
    // construct a crawl start message
    allCrawlstarts.setData(new JSONArray().put(crawlstart));
    allCrawlstarts.put(ObjectAPIHandler.SUCCESS_KEY, allCrawlstarts.getActions().size() > 0);
    // finally add the crawl start on the queue
    return new ServiceResponse(allCrawlstarts);
}