Examples with ServiceResponse net.yacy.grid.http.ServiceResponse used on opensource projects

Example 6 with ServiceResponse

use of net.yacy.grid.http.ServiceResponse in project yacy_grid_mcp by yacy.

the class GSASearchService method serviceImpl.

@Override
public ServiceResponse serviceImpl(Query call, HttpServletResponse response) {
    // query Attributes:
    // for original GSA query attributes, see https://www.google.com/support/enterprise/static/gsa/docs/admin/74/gsa_doc_set/xml_reference/request_format.html#1082911
    String q = call.get("q", "");
    // in GSA: the maximum value of this parameter is 1000
    int num = call.get("num", call.get("rows", call.get("maximumRecords", 10)));
    // The index number of the results is 0-based
    int start = call.get("startRecord", call.get("start", 0));
    Classification.ContentDomain contentdom = Classification.ContentDomain.contentdomParser(call.get("contentdom", "all"));
    // important: call arguments may overrule parsed collection values if not empty. This can be used for authentified indexes!
    String site = call.get("site", call.get("collection", "").replace(',', '|'));
    String[] sites = site.length() == 0 ? new String[0] : site.split("\\|");
    int timezoneOffset = call.get("timezoneOffset", 0);
    boolean explain = call.get("explain", false);
    Sort sort = new Sort(call.get("sort", ""));
    String translatedQ = q;
    String daterange = call.get("daterange", "");
    if (daterange.length() > 0)
        translatedQ += " daterange:" + daterange;
    String as_filetype = call.get("as_filetype", "");
    // refers to as_filetype: only 'i' (include) or 'e' (exclude) allowed
    String as_ft = call.get("as_ft", "i");
    if (as_filetype.length() > 0)
        translatedQ += (as_ft.equals("i") ? " " : " -") + "filetype:" + as_filetype;
    String as_sitesearch = call.get("as_sitesearch", "");
    // refers to as_sitesearch: only 'i' (include) or 'e' (exclude) allowed
    String as_dt = call.get("as_dt", "i");
    if (as_sitesearch.length() > 0)
        translatedQ += (as_dt.equals("i") ? " " : " -") + "site:" + as_sitesearch;
    String queryXML = XML.escape(q);
    // prepare a query
    YaCyQuery yq = new YaCyQuery(translatedQ, sites, contentdom, timezoneOffset);
    ElasticsearchClient ec = Data.gridIndex.getElasticClient();
    HighlightBuilder hb = new HighlightBuilder().field(WebMapping.text_t.getMapping().name()).preTags("").postTags("").fragmentSize(140);
    ElasticsearchClient.Query query = ec.query("web", null, yq.queryBuilder, null, sort, hb, timezoneOffset, start, num, 0, explain);
    List<Map<String, Object>> result = query.results;
    List<String> explanations = query.explanations;
    // no xml encoder here on purpose, we will try to not have such things into our software in the future!
    StringBuffer sb = new StringBuffer(2048);
    sb.append("<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n");
    // GSP
    sb.append("<GSP VER=\"3.2\">\n");
    sb.append("<!-- This is a Google Search Appliance API result, provided by YaCy Grid (see: https://github.com/yacy/yacy_grid_mcp). For the GSA protocol, see https://www.google.com/support/enterprise/static/gsa/docs/admin/74/gsa_doc_set/xml_reference/index.html -->\n");
    sb.append("<TM>0</TM>\n");
    sb.append("<Q>").append(queryXML).append("</Q>\n");
    sb.append("<PARAM name=\"output\" value=\"xml_no_dtd\" original_value=\"xml_no_dtd\"/>\n");
    sb.append("<PARAM name=\"ie\" value=\"UTF-8\" original_value=\"UTF-8\"/>\n");
    sb.append("<PARAM name=\"oe\" value=\"UTF-8\" original_value=\"UTF-8\"/>\n");
    sb.append("<PARAM name=\"q\" value=\"").append(queryXML).append("\" original_value=\"").append(queryXML).append("\"/>\n");
    sb.append("<PARAM name=\"start\" value=\"").append(Integer.toString(start)).append("\" original_value=\"").append(Integer.toString(start)).append("\"/>\n");
    sb.append("<PARAM name=\"num\" value=\"").append(Integer.toString(num)).append("\" original_value=\"").append(Integer.toString(num)).append("\"/>\n");
    sb.append("<PARAM name=\"site\" value=\"").append(XML.escape(site)).append("\" original_value=\"").append(XML.escape(site)).append("\"/>\n");
    // RES
    // SN; The index number (1-based) of this search result; EN: Indicates the index (1-based) of the last search result returned in this result set.
    sb.append("<RES SN=\"" + (start + 1) + "\" EN=\"" + (start + result.size()) + "\">\n");
    // this should show the estimated total number of results
    sb.append("<M>").append(Integer.toString(query.hitCount)).append("</M>\n");
    sb.append("<FI/>\n");
    // sb.append("<NB><NU>").append(getAPIPath()).append("?q=\"").append(queryXML).append("\"&amp;site=&amp;lr=&amp;ie=UTF-8&amp;oe=UTF-8&amp;output=xml_no_dtd&amp;client=&amp;access=&amp;sort=&amp;start=").append(Integer.toString(start)).append("&amp;num=").append(Integer.toString(num)).append("&amp;sa=N</NU></NB>\n");
    // List
    final AtomicInteger hit = new AtomicInteger(1);
    for (int hitc = 0; hitc < result.size(); hitc++) {
        WebDocument doc = new WebDocument(result.get(hitc));
        String titleXML = XML.escape(doc.getTitle());
        String link = doc.getLink();
        if (Classification.ContentDomain.IMAGE == contentdom)
            link = doc.pickImage((String) link);
        String linkXML = XML.escape(link.toString());
        String urlhash = Digest.encodeMD5Hex(link);
        String snippet = doc.getSnippet(query.highlights.get(hitc), yq);
        String snippetXML = XML.escape(snippet);
        Date last_modified_date = doc.getDate();
        int size = doc.getSize();
        int sizekb = size / 1024;
        int sizemb = sizekb / 1024;
        String size_string = sizemb > 0 ? (Integer.toString(sizemb) + " mbyte") : sizekb > 0 ? (Integer.toString(sizekb) + " kbyte") : (Integer.toString(size) + " byte");
        // String host = doc.getString(WebMapping.host_s, "");
        sb.append("<R N=\"").append(Integer.toString(hit.getAndIncrement())).append("\" MIME=\"text/html\">\n");
        sb.append("<T>").append(titleXML).append("</T>\n");
        sb.append("<FS NAME=\"date\" VALUE=\"").append(DateParser.formatGSAFS(last_modified_date)).append("\"/>\n");
        sb.append("<CRAWLDATE>").append(DateParser.formatRFC1123(last_modified_date)).append("</CRAWLDATE>\n");
        sb.append("<LANG>en</LANG>\n");
        sb.append("<U>").append(linkXML).append("</U>\n");
        sb.append("<UE>").append(linkXML).append("</UE>\n");
        sb.append("<S>").append(snippetXML).append("</S>\n");
        sb.append("<COLS>dht</COLS>\n");
        sb.append("<HAS><L/><C SZ=\"").append(size_string).append("\" CID=\"").append(urlhash).append("\" ENC=\"UTF-8\"/></HAS>\n");
        // sb.append("<ENT_SOURCE>yacy_v1.921_20170616_9248.tar.gz/amBzuRuUFyt6</ENT_SOURCE>\n");
        if (explain) {
            sb.append("<EXPLANATION><![CDATA[" + explanations.get(hitc) + "]]></EXPLANATION>\n");
        }
        sb.append("</R>\n");
    }
    ;
    // END RES GSP
    sb.append("</RES>\n");
    sb.append("</GSP>\n");
    return new ServiceResponse(sb.toString());
}

Example 7 with ServiceResponse

use of net.yacy.grid.http.ServiceResponse in project yacy_grid_mcp by yacy.

the class QueryService method serviceImpl.

@Override
public ServiceResponse serviceImpl(Query call, HttpServletResponse response) {
    // String indexName, String typeName, final String id, JSONObject object
    String indexName = call.get("index", "");
    String typeName = call.get("type", "");
    if (typeName.length() == 0)
        typeName = null;
    String id = call.get("id", "");
    QueryLanguage language = QueryLanguage.valueOf(call.get("language", "yacy"));
    String query = call.get("query", "");
    int maximumRecords = call.get("maximumRecords", call.get("rows", call.get("num", 10)));
    int startRecord = call.get("startRecord", call.get("start", 0));
    JSONObject json = new JSONObject(true);
    if (indexName.length() > 0 && id.length() > 0) {
        try {
            Index index = Data.gridIndex.getElasticIndex();
            String url = index.checkConnection().getConnectionURL();
            JSONObject object = index.query(indexName, typeName, id);
            json.put(ObjectAPIHandler.SUCCESS_KEY, true);
            JSONList list = new JSONList();
            if (object != null)
                list.add(object);
            json.put("count", list.length());
            json.put("list", list.toArray());
            if (url != null)
                json.put(ObjectAPIHandler.SERVICE_KEY, url);
        } catch (IOException e) {
            json.put(ObjectAPIHandler.SUCCESS_KEY, false);
            json.put(ObjectAPIHandler.COMMENT_KEY, e.getMessage());
        }
    } else if (indexName.length() > 0 && query.length() > 0) {
        try {
            Index index = Data.gridIndex.getElasticIndex();
            String url = index.checkConnection().getConnectionURL();
            JSONList list = index.query(indexName, typeName, language, query, startRecord, maximumRecords);
            json.put(ObjectAPIHandler.SUCCESS_KEY, true);
            json.put("count", list.length());
            json.put("list", list.toArray());
            if (url != null)
                json.put(ObjectAPIHandler.SERVICE_KEY, url);
        } catch (IOException e) {
            json.put(ObjectAPIHandler.SUCCESS_KEY, false);
            json.put(ObjectAPIHandler.COMMENT_KEY, e.getMessage());
        }
    } else {
        json.put(ObjectAPIHandler.SUCCESS_KEY, false);
        json.put(ObjectAPIHandler.COMMENT_KEY, "the request must contain an index, type, and either an id or a query");
    }
    return new ServiceResponse(json);
}

Example 8 with ServiceResponse

use of net.yacy.grid.http.ServiceResponse in project yacy_grid_mcp by yacy.

the class Network method getMCPServiceList.

@SuppressWarnings("unchecked")
public List<Peer> getMCPServiceList() {
    Set<String> failstubs = new HashSet<>();
    JSONArray ja = null;
    loop: for (Set<String> a : new Set[] { mcpActiveAddresses, mcpPassiveAddresses }) {
        for (String hostprotocolstub : a) {
            try {
                ServiceResponse response = APIServer.getAPI(ServicesService.NAME).serviceImpl(hostprotocolstub, new JSONObject());
                ja = response.getArray();
                break loop;
            } catch (IOException e) {
                failstubs.add(hostprotocolstub);
            }
        }
    }
    removeMCP(failstubs);
    List<Peer> list = new ArrayList<>();
    if (ja == null)
        return list;
    ja.forEach(j -> list.add(new Peer((JSONObject) j)));
    return list;
}

Example 9 with ServiceResponse

use of net.yacy.grid.http.ServiceResponse in project yacy_grid_mcp by yacy.

the class StoreService method serviceImpl.

@Override
public ServiceResponse serviceImpl(Query call, HttpServletResponse response) {
    String path = call.get("path", "");
    byte[] asset = call.get("asset", EMPTY_ASSET);
    JSONObject json = new JSONObject(true);
    if (path.length() > 0) {
        try {
            StorageFactory<byte[]> factory = Data.gridStorage.store(path, asset);
            String url = factory.getConnectionURL();
            json.put(ObjectAPIHandler.SUCCESS_KEY, true);
            if (url != null)
                json.put(ObjectAPIHandler.SERVICE_KEY, url);
        } catch (IOException e) {
            json.put(ObjectAPIHandler.SUCCESS_KEY, false);
            json.put(ObjectAPIHandler.COMMENT_KEY, e.getMessage());
        }
    } else {
        json.put(ObjectAPIHandler.SUCCESS_KEY, false);
        json.put(ObjectAPIHandler.COMMENT_KEY, "the request must contain a path and a asset");
    }
    return new ServiceResponse(json);
}

Example 10 with ServiceResponse

use of net.yacy.grid.http.ServiceResponse in project yacy_grid_mcp by yacy.

the class MCPStorageFactory method getStorage.

@Override
public Storage<byte[]> getStorage() throws IOException {
    return new Storage<byte[]>() {

        @Override
        public void checkConnection() throws IOException {
            final Map<String, byte[]> params = new HashMap<>();
            String protocolhostportstub = MCPStorageFactory.this.getConnectionURL();
            ServiceResponse sr = APIServer.getAPI(StatusService.NAME).serviceImpl(protocolhostportstub, params);
            if (!sr.getObject().has("system"))
                throw new IOException("MCP does not respond properly");
        }

        @Override
        public StorageFactory<byte[]> store(String path, byte[] asset) throws IOException {
            final Map<String, byte[]> params = new HashMap<>();
            params.put("path", path.getBytes(StandardCharsets.UTF_8));
            params.put("asset", asset);
            String protocolhostportstub = MCPStorageFactory.this.getConnectionURL();
            ServiceResponse sr = APIServer.getAPI(StoreService.NAME).serviceImpl(protocolhostportstub, params);
            JSONObject response = sr.getObject();
            if (response.has(ObjectAPIHandler.SUCCESS_KEY) && response.getBoolean(ObjectAPIHandler.SUCCESS_KEY)) {
                connectMCP(response);
                return MCPStorageFactory.this;
            } else {
                throw handleError(response);
            }
        }

        @Override
        public Asset<byte[]> load(String path) throws IOException {
            final Map<String, byte[]> params = new HashMap<>();
            params.put("path", path.getBytes(StandardCharsets.UTF_8));
            String protocolhostportstub = MCPStorageFactory.this.getConnectionURL();
            ServiceResponse sr = APIServer.getAPI(LoadService.NAME).serviceImpl(protocolhostportstub, params);
            return new Asset<byte[]>(MCPStorageFactory.this, sr.getByteArray());
        }

        @Override
        public void close() {
        }

        private void connectMCP(JSONObject response) {
            if (response.has(ObjectAPIHandler.SERVICE_KEY)) {
                String server = response.getString(ObjectAPIHandler.SERVICE_KEY);
                int p = server.indexOf("://");
                if (p > 0)
                    MCPStorageFactory.this.remoteSystem = server.substring(0, p);
                if (MCPStorageFactory.this.storage != null) {
                    if (MCPStorageFactory.this.storage.connectFTP(server)) {
                        Data.logger.info("MCPStorageFactory.connectMCP connected MCP storage at " + server);
                    } else {
                        Data.logger.error("MCPStorageFactory.connectMCP failed to connect MCP storage at " + server);
                    }
                }
            }
        }

        private IOException handleError(JSONObject response) {
            if (response.has(ObjectAPIHandler.COMMENT_KEY)) {
                return new IOException("cannot connect to MCP: " + response.getString(ObjectAPIHandler.COMMENT_KEY));
            }
            return new IOException("bad response from MCP: no success and no comment key");
        }
    };
}