Examples with SusiThought ai.susi.mind.SusiThought used on opensource projects

Example 1 with SusiThought

use of ai.susi.mind.SusiThought in project yacy_grid_loader by yacy.

the class LoaderService method serviceImpl.

@Override
public ServiceResponse serviceImpl(Query call, HttpServletResponse response) {
    // construct the same process as if it was submitted on a queue
    SusiThought process = ProcessService.queryToProcess(call);
    // extract call parameter here to enhance ability to debug
    SusiAction action = process.getActions().get(0);
    JSONArray data = process.getData();
    // construct a WARC
    byte[] b = ContentLoader.eval(action, data, true, "api call from " + call.getClientHost(), true);
    // store the WARC as asset if wanted
    return new ServiceResponse(b);
}

Example 2 with SusiThought

use of ai.susi.mind.SusiThought in project yacy_grid_mcp by yacy.

the class AbstractBrokerListener method handleMessage.

private void handleMessage(final MessageContainer<byte[]> mc, final String processName, final int processNumber) {
    String payload = new String(mc.getPayload(), StandardCharsets.UTF_8);
    JSONObject json = new JSONObject(new JSONTokener(payload));
    final SusiThought process = new SusiThought(json);
    final JSONArray data = process.getData();
    final List<SusiAction> actions = process.getActions();
    // loop though all actions
    actionloop: for (int ac = 0; ac < actions.size(); ac++) {
        SusiAction action = actions.get(ac);
        String type = action.getStringAttr("type");
        String queue = action.getStringAttr("queue");
        // check if the credentials to execute the queue are valid
        if (type == null || type.length() == 0 || queue == null || queue.length() == 0) {
            Data.logger.info("bad message in queue, continue");
            continue actionloop;
        }
        // check if this is the correct queue
        if (!type.equals(this.service.name())) {
            Data.logger.info("wrong message in queue: " + type + ", continue");
            try {
                // put that into the correct queue
                loadNextAction(action, process.getData());
            } catch (Throwable e) {
                Data.logger.warn("", e);
            }
            continue actionloop;
        }
        // process the action using the previously acquired execution thread
        // this.threadPool.execute(new ActionProcess(action, data));
        // run, not start: we execute this in the current thread
        new ActionProcess(action, data, processName, processNumber).run();
    }
}

Example 3 with SusiThought

use of ai.susi.mind.SusiThought in project yacy_grid_loader by yacy.

the class ProcessService method serviceImpl.

@Override
public ServiceResponse serviceImpl(Query call, HttpServletResponse response) {
    // construct the same process as if it was submitted on a queue
    SusiThought process = queryToProcess(call);
    // construct a WARC
    String targetasset = process.getObservation("targetasset");
    byte[] b = ContentLoader.eval(process.getActions().get(0), process.getData(), targetasset.endsWith(".gz"), "api call from " + call.getClientHost(), true);
    // store the WARC as asset if wanted
    JSONObject json = new JSONObject(true);
    if (targetasset != null && targetasset.length() > 0) {
        try {
            Data.gridStorage.store(targetasset, b);
            json.put(ObjectAPIHandler.SUCCESS_KEY, true);
            json.put(ObjectAPIHandler.COMMENT_KEY, "asset stored");
        } catch (IOException e) {
            e.printStackTrace();
            json.put(ObjectAPIHandler.SUCCESS_KEY, false);
            json.put(ObjectAPIHandler.COMMENT_KEY, e.getMessage());
        }
    } else {
        json.put(ObjectAPIHandler.SUCCESS_KEY, false);
        json.put(ObjectAPIHandler.COMMENT_KEY, "this process requires a 'targetasset' attribute");
    }
    return new ServiceResponse(json);
}

Example 4 with SusiThought

use of ai.susi.mind.SusiThought in project yacy_grid_loader by yacy.

the class ProcessService method queryToProcess.

public static SusiThought queryToProcess(Query call) {
    String url = call.get("url", "");
    int urlCount = call.get("urls", 0);
    String collection = call.get("collection", "");
    String targetasset = call.get("targetasset", "");
    // construct an object that could be taken from the queue server
    SusiThought process = new SusiThought();
    process.setProcess("yacy_grid_loader");
    if (collection.length() > 0)
        process.addObservation("collection", collection);
    // create action
    JSONObject action = new JSONObject();
    JSONArray urls = new JSONArray();
    action.put("type", RenderType.loader.name());
    action.put("queue", "loader");
    action.put("urls", urls);
    if (collection.length() > 0)
        action.put("collection", collection);
    if (targetasset.length() > 0)
        action.put("targetasset", targetasset);
    if (url.length() > 0)
        urls.put(url);
    if (urlCount > 0)
        for (int i = 0; i < urlCount; i++) {
            url = call.get("url_" + i, "");
            if (url.length() > 0)
                urls.put(url);
        }
    process.addAction(new SusiAction(action));
    return process;
}

Example 5 with SusiThought

use of ai.susi.mind.SusiThought in project yacy_grid_crawler by yacy.

the class CrawlStartService method serviceImpl.

@Override
public ServiceResponse serviceImpl(Query call, HttpServletResponse response) {
    JSONObject crawlstart = CrawlerDefaultValuesService.crawlStartDefaultClone();
    for (String key : crawlstart.keySet()) {
        Object object = crawlstart.get(key);
        if (object instanceof String)
            crawlstart.put(key, call.get(key, crawlstart.getString(key)));
        else if (object instanceof Integer)
            crawlstart.put(key, call.get(key, crawlstart.getInt(key)));
        else if (object instanceof Long)
            crawlstart.put(key, call.get(key, crawlstart.getLong(key)));
        else if (object instanceof JSONArray) {
            JSONArray a = crawlstart.getJSONArray(key);
            Object cv = call.get(key);
            if (cv != null)
                crawlstart.put(key, cv);
        } else {
            System.out.println("unrecognized type: " + object.getClass().toString());
        }
    }
    // set the crawl id
    CrawlstartURLSplitter crawlstartURLs = new CrawlstartURLSplitter(crawlstart.getString("crawlingURL"));
    Date now = new Date();
    // start the crawls; each of the url in a separate crawl to enforce parallel loading from different hosts
    SusiThought allCrawlstarts = new SusiThought();
    int count = 0;
    for (MultiProtocolURL url : crawlstartURLs.getURLs()) {
        JSONObject singlecrawl = new JSONObject();
        // create a clone of crawlstart
        for (String key : crawlstart.keySet()) singlecrawl.put(key, crawlstart.get(key));
        singlecrawl.put("id", Crawler.getCrawlID(url, now, count++));
        try {
            GridQueue queueName = Data.gridBroker.queueName(YaCyServices.crawler, YaCyServices.crawler.getQueues(), ShardingMethod.BALANCE, Crawler.CRAWLER_PRIORITY_DIMENSIONS, singlecrawl.getInt("priority"), url.getHost());
            SusiThought json = new SusiThought();
            json.setData(new JSONArray().put(singlecrawl));
            JSONObject action = new JSONObject().put("type", YaCyServices.crawler.name()).put("queue", queueName.name()).put("id", singlecrawl.getString("id")).put("depth", 0).put("sourcegraph", "rootasset");
            SusiAction crawlAction = new SusiAction(action);
            JSONObject graph = new JSONObject(true).put(WebMapping.canonical_s.getMapping().name(), url.toNormalform(true));
            crawlAction.setJSONListAsset("rootasset", new JSONList().add(graph));
            json.addAction(crawlAction);
            allCrawlstarts.addAction(crawlAction);
            byte[] b = json.toString().getBytes(StandardCharsets.UTF_8);
            Data.gridBroker.send(YaCyServices.crawler, queueName, b);
        } catch (IOException e) {
            Data.logger.warn("error when starting crawl for " + url.toNormalform(true), e);
            allCrawlstarts.put(ObjectAPIHandler.COMMENT_KEY, e.getMessage());
        }
    }
    // construct a crawl start message
    allCrawlstarts.setData(new JSONArray().put(crawlstart));
    allCrawlstarts.put(ObjectAPIHandler.SUCCESS_KEY, allCrawlstarts.getActions().size() > 0);
    // finally add the crawl start on the queue
    return new ServiceResponse(allCrawlstarts);
}