Examples with ServiceResponse net.yacy.grid.http.ServiceResponse used on opensource projects

Example 11 with ServiceResponse

use of net.yacy.grid.http.ServiceResponse in project yacy_grid_mcp by yacy.

the class MCPIndexFactory method getIndex.

@Override
public Index getIndex() throws IOException {
    final JSONObject params = new JSONObject(true);
    return new Index() {

        @Override
        public IndexFactory checkConnection() throws IOException {
            String protocolhostportstub = MCPIndexFactory.this.getConnectionURL();
            APIHandler apiHandler = APIServer.getAPI(CheckService.NAME);
            ServiceResponse sr = apiHandler.serviceImpl(protocolhostportstub, params);
            if (!success(sr.getObject()))
                throw new IOException("MCP does not respond properly");
            return MCPIndexFactory.this;
        }

        private JSONObject getResponse(APIHandler handler) throws IOException {
            String protocolhostportstub = MCPIndexFactory.this.getConnectionURL();
            ServiceResponse sr = handler.serviceImpl(protocolhostportstub, params);
            return sr.getObject();
        }

        private boolean success(JSONObject response) {
            return response.has(ObjectAPIHandler.SUCCESS_KEY) && response.getBoolean(ObjectAPIHandler.SUCCESS_KEY);
        }

        private void connectMCP(JSONObject response) {
            if (response.has(ObjectAPIHandler.SERVICE_KEY)) {
                String elastic = response.getString(ObjectAPIHandler.SERVICE_KEY);
                if (MCPIndexFactory.this.index.connectElasticsearch(elastic)) {
                    Data.logger.info("connected MCP index at " + elastic);
                } else {
                    Data.logger.error("failed to connect MCP index at " + elastic);
                }
            }
        }

        private IOException handleError(JSONObject response) {
            if (response.has(ObjectAPIHandler.COMMENT_KEY)) {
                return new IOException("cannot connect to MCP: " + response.getString(ObjectAPIHandler.COMMENT_KEY));
            }
            return new IOException("bad response from MCP: no success and no comment key");
        }

        @Override
        public IndexFactory add(String indexName, String typeName, String id, JSONObject object) throws IOException {
            params.put("index", indexName);
            params.put("type", typeName);
            params.put("id", id);
            params.put("object", object.toString());
            JSONObject response = getResponse(APIServer.getAPI(AddService.NAME));
            // read the broker to store the service definition of the remote queue, if exists
            if (success(response)) {
                connectMCP(response);
                return MCPIndexFactory.this;
            } else {
                throw handleError(response);
            }
        }

        @Override
        public boolean exist(String indexName, String typeName, String id) throws IOException {
            params.put("index", indexName);
            params.put("type", typeName);
            params.put("id", id);
            JSONObject response = getResponse(APIServer.getAPI(ExistService.NAME));
            // read the broker to store the service definition of the remote queue, if exists
            if (success(response)) {
                connectMCP(response);
                return response.has("exists") && response.getBoolean("exists");
            } else {
                throw handleError(response);
            }
        }

        @Override
        public long count(String indexName, String typeName, QueryLanguage language, String query) throws IOException {
            params.put("index", indexName);
            params.put("type", typeName);
            params.put("language", language.name());
            params.put("query", query);
            JSONObject response = getResponse(APIServer.getAPI(CountService.NAME));
            // read the broker to store the service definition of the remote queue, if exists
            if (success(response)) {
                connectMCP(response);
                return response.has("count") ? response.getLong("count") : 0;
            } else {
                throw handleError(response);
            }
        }

        @Override
        public JSONObject query(String indexName, String typeName, String id) throws IOException {
            params.put("index", indexName);
            params.put("type", typeName);
            params.put("id", id);
            JSONObject response = getResponse(APIServer.getAPI(QueryService.NAME));
            // read the broker to store the service definition of the remote queue, if exists
            if (success(response)) {
                connectMCP(response);
                if (!response.has("list"))
                    return null;
                JSONArray list = response.getJSONArray("list");
                if (list.length() == 0)
                    return null;
                return list.getJSONObject(0);
            } else {
                throw handleError(response);
            }
        }

        @Override
        public JSONList query(String indexName, String typeName, QueryLanguage language, String query, int start, int count) throws IOException {
            params.put("index", indexName);
            params.put("type", typeName);
            params.put("language", language.name());
            params.put("query", query);
            JSONObject response = getResponse(APIServer.getAPI(QueryService.NAME));
            // read the broker to store the service definition of the remote queue, if exists
            if (success(response)) {
                connectMCP(response);
                JSONList list = new JSONList();
                if (!response.has("list"))
                    return list;
                JSONArray l = response.getJSONArray("list");
                if (l.length() == 0)
                    return list;
                for (int i = 0; i < l.length(); i++) list.add(l.getJSONObject(i));
                return list;
            } else {
                throw handleError(response);
            }
        }

        @Override
        public boolean delete(String indexName, String typeName, String id) throws IOException {
            params.put("index", indexName);
            params.put("type", typeName);
            params.put("id", id);
            JSONObject response = getResponse(APIServer.getAPI(DeleteService.NAME));
            // read the broker to store the service definition of the remote queue, if exists
            if (success(response)) {
                connectMCP(response);
                return response.has("deleted") && response.getBoolean("deleted");
            } else {
                throw handleError(response);
            }
        }

        @Override
        public long delete(String indexName, String typeName, QueryLanguage language, String query) throws IOException {
            params.put("index", indexName);
            params.put("type", typeName);
            params.put("language", language.name());
            params.put("query", query);
            JSONObject response = getResponse(APIServer.getAPI(DeleteService.NAME));
            // read the broker to store the service definition of the remote queue, if exists
            if (success(response)) {
                connectMCP(response);
                return response.has("count") ? response.getLong("count") : 0;
            } else {
                throw handleError(response);
            }
        }

        @Override
        public void close() {
        }
    };
}

Example 12 with ServiceResponse

use of net.yacy.grid.http.ServiceResponse in project yacy_grid_mcp by yacy.

the class PeekService method serviceImpl.

@Override
public ServiceResponse serviceImpl(Query call, HttpServletResponse response) {
    String serviceName = call.get("serviceName", "");
    String queueName = call.get("queueName", "");
    JSONObject json = new JSONObject(true);
    if (serviceName.length() > 0 && queueName.length() > 0) {
        try {
            YaCyServices service = YaCyServices.valueOf(serviceName);
            GridQueue queue = new GridQueue(queueName);
            AvailableContainer available = Data.gridBroker.available(service, queue);
            int ac = available.getAvailable();
            String url = available.getFactory().getConnectionURL();
            if (url != null)
                json.put(ObjectAPIHandler.SERVICE_KEY, url);
            if (ac > 0) {
                // load one message and send it right again to prevent that it is lost
                MessageContainer<byte[]> message = Data.gridBroker.receive(service, queue, 3000);
                // message can be null if a timeout occurred
                if (message == null) {
                    json.put(ObjectAPIHandler.SUCCESS_KEY, false);
                    json.put(ObjectAPIHandler.COMMENT_KEY, "timeout");
                } else {
                    // send it again asap!
                    Data.gridBroker.send(service, queue, message.getPayload());
                    // evaluate whats inside
                    String payload = message.getPayload() == null ? null : new String(message.getPayload(), StandardCharsets.UTF_8);
                    JSONObject payloadjson = payload == null ? null : new JSONObject(new JSONTokener(payload));
                    json.put(ObjectAPIHandler.AVAILABLE_KEY, ac);
                    json.put(ObjectAPIHandler.MESSAGE_KEY, payloadjson == null ? new JSONObject() : payloadjson);
                    json.put(ObjectAPIHandler.SUCCESS_KEY, true);
                }
            } else {
                json.put(ObjectAPIHandler.AVAILABLE_KEY, 0);
                json.put(ObjectAPIHandler.SUCCESS_KEY, true);
            }
        } catch (IOException e) {
            json.put(ObjectAPIHandler.SUCCESS_KEY, false);
            json.put(ObjectAPIHandler.COMMENT_KEY, e.getMessage());
        }
    } else {
        json.put(ObjectAPIHandler.SUCCESS_KEY, false);
        json.put(ObjectAPIHandler.COMMENT_KEY, "the request must contain a serviceName and a queueName");
    }
    return new ServiceResponse(json);
}

Example 13 with ServiceResponse

use of net.yacy.grid.http.ServiceResponse in project yacy_grid_mcp by yacy.

the class SendService method serviceImpl.

@Override
public ServiceResponse serviceImpl(Query call, HttpServletResponse response) {
    String serviceName = call.get("serviceName", "");
    String queueName = call.get("queueName", "");
    String message = call.get("message", "");
    JSONObject json = new JSONObject(true);
    if (serviceName.length() > 0 && queueName.length() > 0 && message.length() > 0) {
        try {
            QueueFactory<byte[]> factory = Data.gridBroker.send(YaCyServices.valueOf(serviceName), new GridQueue(queueName), message.getBytes(StandardCharsets.UTF_8));
            String url = factory.getConnectionURL();
            json.put(ObjectAPIHandler.SUCCESS_KEY, true);
            if (url != null)
                json.put(ObjectAPIHandler.SERVICE_KEY, url);
        } catch (IOException e) {
            json.put(ObjectAPIHandler.SUCCESS_KEY, false);
            json.put(ObjectAPIHandler.COMMENT_KEY, e.getMessage());
        }
    } else {
        json.put(ObjectAPIHandler.SUCCESS_KEY, false);
        json.put(ObjectAPIHandler.COMMENT_KEY, "the request must contain a serviceName, a queueName and a message");
    }
    return new ServiceResponse(json);
}

Example 14 with ServiceResponse

use of net.yacy.grid.http.ServiceResponse in project yacy_grid_loader by yacy.

the class ProcessService method serviceImpl.

@Override
public ServiceResponse serviceImpl(Query call, HttpServletResponse response) {
    // construct the same process as if it was submitted on a queue
    SusiThought process = queryToProcess(call);
    // construct a WARC
    String targetasset = process.getObservation("targetasset");
    byte[] b = ContentLoader.eval(process.getActions().get(0), process.getData(), targetasset.endsWith(".gz"), "api call from " + call.getClientHost(), true);
    // store the WARC as asset if wanted
    JSONObject json = new JSONObject(true);
    if (targetasset != null && targetasset.length() > 0) {
        try {
            Data.gridStorage.store(targetasset, b);
            json.put(ObjectAPIHandler.SUCCESS_KEY, true);
            json.put(ObjectAPIHandler.COMMENT_KEY, "asset stored");
        } catch (IOException e) {
            e.printStackTrace();
            json.put(ObjectAPIHandler.SUCCESS_KEY, false);
            json.put(ObjectAPIHandler.COMMENT_KEY, e.getMessage());
        }
    } else {
        json.put(ObjectAPIHandler.SUCCESS_KEY, false);
        json.put(ObjectAPIHandler.COMMENT_KEY, "this process requires a 'targetasset' attribute");
    }
    return new ServiceResponse(json);
}

Example 15 with ServiceResponse

use of net.yacy.grid.http.ServiceResponse in project yacy_grid_crawler by yacy.

the class CrawlStartService method serviceImpl.

@Override
public ServiceResponse serviceImpl(Query call, HttpServletResponse response) {
    JSONObject crawlstart = CrawlerDefaultValuesService.crawlStartDefaultClone();
    for (String key : crawlstart.keySet()) {
        Object object = crawlstart.get(key);
        if (object instanceof String)
            crawlstart.put(key, call.get(key, crawlstart.getString(key)));
        else if (object instanceof Integer)
            crawlstart.put(key, call.get(key, crawlstart.getInt(key)));
        else if (object instanceof Long)
            crawlstart.put(key, call.get(key, crawlstart.getLong(key)));
        else if (object instanceof JSONArray) {
            JSONArray a = crawlstart.getJSONArray(key);
            Object cv = call.get(key);
            if (cv != null)
                crawlstart.put(key, cv);
        } else {
            System.out.println("unrecognized type: " + object.getClass().toString());
        }
    }
    // set the crawl id
    CrawlstartURLSplitter crawlstartURLs = new CrawlstartURLSplitter(crawlstart.getString("crawlingURL"));
    Date now = new Date();
    // start the crawls; each of the url in a separate crawl to enforce parallel loading from different hosts
    SusiThought allCrawlstarts = new SusiThought();
    int count = 0;
    for (MultiProtocolURL url : crawlstartURLs.getURLs()) {
        JSONObject singlecrawl = new JSONObject();
        // create a clone of crawlstart
        for (String key : crawlstart.keySet()) singlecrawl.put(key, crawlstart.get(key));
        singlecrawl.put("id", Crawler.getCrawlID(url, now, count++));
        try {
            GridQueue queueName = Data.gridBroker.queueName(YaCyServices.crawler, YaCyServices.crawler.getQueues(), ShardingMethod.BALANCE, Crawler.CRAWLER_PRIORITY_DIMENSIONS, singlecrawl.getInt("priority"), url.getHost());
            SusiThought json = new SusiThought();
            json.setData(new JSONArray().put(singlecrawl));
            JSONObject action = new JSONObject().put("type", YaCyServices.crawler.name()).put("queue", queueName.name()).put("id", singlecrawl.getString("id")).put("depth", 0).put("sourcegraph", "rootasset");
            SusiAction crawlAction = new SusiAction(action);
            JSONObject graph = new JSONObject(true).put(WebMapping.canonical_s.getMapping().name(), url.toNormalform(true));
            crawlAction.setJSONListAsset("rootasset", new JSONList().add(graph));
            json.addAction(crawlAction);
            allCrawlstarts.addAction(crawlAction);
            byte[] b = json.toString().getBytes(StandardCharsets.UTF_8);
            Data.gridBroker.send(YaCyServices.crawler, queueName, b);
        } catch (IOException e) {
            Data.logger.warn("error when starting crawl for " + url.toNormalform(true), e);
            allCrawlstarts.put(ObjectAPIHandler.COMMENT_KEY, e.getMessage());
        }
    }
    // construct a crawl start message
    allCrawlstarts.setData(new JSONArray().put(crawlstart));
    allCrawlstarts.put(ObjectAPIHandler.SUCCESS_KEY, allCrawlstarts.getActions().size() > 0);
    // finally add the crawl start on the queue
    return new ServiceResponse(allCrawlstarts);
}