Examples with GridQueue net.yacy.grid.io.messages.GridQueue used on opensource projects

Example 6 with GridQueue

use of net.yacy.grid.io.messages.GridQueue in project yacy_grid_crawler by yacy.

the class CrawlStartService method serviceImpl.

@Override
public ServiceResponse serviceImpl(Query call, HttpServletResponse response) {
    JSONObject crawlstart = CrawlerDefaultValuesService.crawlStartDefaultClone();
    for (String key : crawlstart.keySet()) {
        Object object = crawlstart.get(key);
        if (object instanceof String)
            crawlstart.put(key, call.get(key, crawlstart.getString(key)));
        else if (object instanceof Integer)
            crawlstart.put(key, call.get(key, crawlstart.getInt(key)));
        else if (object instanceof Long)
            crawlstart.put(key, call.get(key, crawlstart.getLong(key)));
        else if (object instanceof JSONArray) {
            JSONArray a = crawlstart.getJSONArray(key);
            Object cv = call.get(key);
            if (cv != null)
                crawlstart.put(key, cv);
        } else {
            System.out.println("unrecognized type: " + object.getClass().toString());
        }
    }
    // set the crawl id
    CrawlstartURLSplitter crawlstartURLs = new CrawlstartURLSplitter(crawlstart.getString("crawlingURL"));
    Date now = new Date();
    // start the crawls; each of the url in a separate crawl to enforce parallel loading from different hosts
    SusiThought allCrawlstarts = new SusiThought();
    int count = 0;
    for (MultiProtocolURL url : crawlstartURLs.getURLs()) {
        JSONObject singlecrawl = new JSONObject();
        // create a clone of crawlstart
        for (String key : crawlstart.keySet()) singlecrawl.put(key, crawlstart.get(key));
        singlecrawl.put("id", Crawler.getCrawlID(url, now, count++));
        try {
            GridQueue queueName = Data.gridBroker.queueName(YaCyServices.crawler, YaCyServices.crawler.getQueues(), ShardingMethod.BALANCE, Crawler.CRAWLER_PRIORITY_DIMENSIONS, singlecrawl.getInt("priority"), url.getHost());
            SusiThought json = new SusiThought();
            json.setData(new JSONArray().put(singlecrawl));
            JSONObject action = new JSONObject().put("type", YaCyServices.crawler.name()).put("queue", queueName.name()).put("id", singlecrawl.getString("id")).put("depth", 0).put("sourcegraph", "rootasset");
            SusiAction crawlAction = new SusiAction(action);
            JSONObject graph = new JSONObject(true).put(WebMapping.canonical_s.getMapping().name(), url.toNormalform(true));
            crawlAction.setJSONListAsset("rootasset", new JSONList().add(graph));
            json.addAction(crawlAction);
            allCrawlstarts.addAction(crawlAction);
            byte[] b = json.toString().getBytes(StandardCharsets.UTF_8);
            Data.gridBroker.send(YaCyServices.crawler, queueName, b);
        } catch (IOException e) {
            Data.logger.warn("error when starting crawl for " + url.toNormalform(true), e);
            allCrawlstarts.put(ObjectAPIHandler.COMMENT_KEY, e.getMessage());
        }
    }
    // construct a crawl start message
    allCrawlstarts.setData(new JSONArray().put(crawlstart));
    allCrawlstarts.put(ObjectAPIHandler.SUCCESS_KEY, allCrawlstarts.getActions().size() > 0);
    // finally add the crawl start on the queue
    return new ServiceResponse(allCrawlstarts);
}

Example 7 with GridQueue

use of net.yacy.grid.io.messages.GridQueue in project yacy_grid_mcp by yacy.

the class AvailableService method serviceImpl.

@Override
public ServiceResponse serviceImpl(Query call, HttpServletResponse response) {
    String serviceName = call.get("serviceName", "");
    String queueName = call.get("queueName", "");
    JSONObject json = new JSONObject(true);
    if (serviceName.length() > 0 && queueName.length() > 0) {
        try {
            AvailableContainer available = Data.gridBroker.available(YaCyServices.valueOf(serviceName), new GridQueue(queueName));
            String url = available.getFactory().getConnectionURL();
            json.put(ObjectAPIHandler.AVAILABLE_KEY, available.getAvailable());
            json.put(ObjectAPIHandler.SUCCESS_KEY, true);
            if (url != null)
                json.put(ObjectAPIHandler.SERVICE_KEY, url);
        } catch (IOException e) {
            json.put(ObjectAPIHandler.SUCCESS_KEY, false);
            json.put(ObjectAPIHandler.COMMENT_KEY, e.getMessage());
        }
    } else {
        json.put(ObjectAPIHandler.SUCCESS_KEY, false);
        json.put(ObjectAPIHandler.COMMENT_KEY, "the request must contain a serviceName and a queueName");
    }
    return new ServiceResponse(json);
}

Example 8 with GridQueue

use of net.yacy.grid.io.messages.GridQueue in project yacy_grid_mcp by yacy.

the class ClearService method serviceImpl.

@Override
public ServiceResponse serviceImpl(Query call, HttpServletResponse response) {
    String serviceName = call.get("serviceName", "");
    String queueName = call.get("queueName", "");
    JSONObject json = new JSONObject(true);
    if (serviceName.length() > 0 && queueName.length() > 0) {
        try {
            QueueFactory<byte[]> qf = Data.gridBroker.clear(YaCyServices.valueOf(serviceName), new GridQueue(queueName));
            String url = qf.getConnectionURL();
            json.put(ObjectAPIHandler.SUCCESS_KEY, true);
            if (url != null)
                json.put(ObjectAPIHandler.SERVICE_KEY, url);
        } catch (IOException e) {
            json.put(ObjectAPIHandler.SUCCESS_KEY, false);
            json.put(ObjectAPIHandler.COMMENT_KEY, e.getMessage());
        }
    } else {
        json.put(ObjectAPIHandler.SUCCESS_KEY, false);
        json.put(ObjectAPIHandler.COMMENT_KEY, "the request must contain a serviceName and a queueName");
    }
    return new ServiceResponse(json);
}

Example 9 with GridQueue

use of net.yacy.grid.io.messages.GridQueue in project yacy_grid_mcp by yacy.

the class ReceiveService method serviceImpl.

@Override
public ServiceResponse serviceImpl(Query call, HttpServletResponse response) {
    String serviceName = call.get("serviceName", "");
    String queueName = call.get("queueName", "");
    long timeout = call.get("timeout", -1);
    JSONObject json = new JSONObject(true);
    if (serviceName.length() > 0 && queueName.length() > 0) {
        try {
            MessageContainer<byte[]> message = Data.gridBroker.receive(YaCyServices.valueOf(serviceName), new GridQueue(queueName), timeout);
            // message can be null if a timeout occurred
            if (message == null) {
                json.put(ObjectAPIHandler.SUCCESS_KEY, false);
                json.put(ObjectAPIHandler.COMMENT_KEY, "timeout");
            } else {
                String url = message.getFactory().getConnectionURL();
                byte[] payload = message.getPayload();
                json.put(ObjectAPIHandler.MESSAGE_KEY, payload == null ? "" : new String(payload, StandardCharsets.UTF_8));
                json.put(ObjectAPIHandler.SUCCESS_KEY, true);
                if (url != null)
                    json.put(ObjectAPIHandler.SERVICE_KEY, url);
            }
        } catch (IOException e) {
            json.put(ObjectAPIHandler.SUCCESS_KEY, false);
            json.put(ObjectAPIHandler.COMMENT_KEY, e.getMessage());
        }
    } else {
        json.put(ObjectAPIHandler.SUCCESS_KEY, false);
        json.put(ObjectAPIHandler.COMMENT_KEY, "the request must contain a serviceName and a queueName");
    }
    return new ServiceResponse(json);
}