Search in sources :

Example 61 with JSONObject

use of net.sf.json.JSONObject in project topcom-cloud by 545314690.

the class EtlController method bbsEtl.

/**
 * 微博id->weiboId
 * md5(id)->微博id
 * @param limit
 * @return
 */
@ApiOperation("bbsEtl")
@RequestMapping(value = "/bbsEtl", method = RequestMethod.GET, produces = "application/json")
@ResponseBody
public Object bbsEtl(@RequestParam Integer limit) {
    long count = 0;
    long count1 = 0;
    SearchResponse searchResponse = scanBbs("", limit);
    while (searchResponse.getHits().getHits().length > 0) {
        SearchHit[] hits = searchResponse.getHits().hits();
        List<JSONObject> commentsList = new ArrayList<>();
        for (int i = 0; i < hits.length; i++) {
            JSONObject jsonObject = JSONObject.fromObject(hits[i].sourceAsMap());
            String type = jsonObject.get("type").toString();
            if (type.equals("Bbs")) {
                Object object = jsonObject.get("floorNum");
                if (object != null && Integer.valueOf(object.toString()) > 0) {
                    commentsList.add(jsonObject);
                }
            }
        }
        // String path = "D:\\data\\bbs\\"+"bbs"+(count1/50000)+".txt";
        String path = "/home/yuqing/data/esFile/" + "bbs" + (count1 / 50000) + ".txt";
        save2File(commentsList, path);
        dleteEsDoc("yuqing", "article", commentsList);
        count1 = count1 + commentsList.size();
        count = count + searchResponse.getHits().getHits().length;
        logger.error("删除BBS条数:" + count1);
        logger.error("处理BBS条数:" + count + "\n 完成度" + (double) count * 100 / searchResponse.getHits().getTotalHits() + "%");
        searchResponse = scanBbs(searchResponse.getScrollId(), limit);
    }
    return "done";
}
Also used : SearchHit(org.elasticsearch.search.SearchHit) JSONObject(net.sf.json.JSONObject) ArrayList(java.util.ArrayList) JSONObject(net.sf.json.JSONObject) SearchResponse(org.elasticsearch.action.search.SearchResponse) ApiOperation(io.swagger.annotations.ApiOperation)

Example 62 with JSONObject

use of net.sf.json.JSONObject in project topcom-cloud by 545314690.

the class EtlController method dleteEsDoc.

private void dleteEsDoc(String yuqing, String article, List<JSONObject> commentsList) {
    Client client = elasticsearchTemplate.getClient();
    BulkRequestBuilder bulkdel = client.prepareBulk();
    for (int i = 0; i < commentsList.size(); i++) {
        JSONObject jsonObject = commentsList.get(i);
        bulkdel.add(client.prepareDelete("yuqing", "article", jsonObject.get("id").toString()).request());
    }
    bulkdel.execute().actionGet();
}
Also used : JSONObject(net.sf.json.JSONObject) BulkRequestBuilder(org.elasticsearch.action.bulk.BulkRequestBuilder) Client(org.elasticsearch.client.Client)

Example 63 with JSONObject

use of net.sf.json.JSONObject in project topcom-cloud by 545314690.

the class DescriptionManagerImpl method specialOutline.

@Override
public Object specialOutline(BoolQueryRequest boolQueryRequest) {
    boolQueryRequest.setPage(new PageRequest(1, 1));
    JSONObject jsonObject = new JSONObject();
    Page page = allESDataService.findByMustShouldDateInType(boolQueryRequest);
    jsonObject.put("monthCount", page.getTotalElements());
    jsonObject.put("maxType", getMaxType(boolQueryRequest, "type"));
    jsonObject.put("label", getMaxType(boolQueryRequest, "nlp.sentiment.label"));
    boolQueryRequest.setType("news");
    jsonObject.put("maxSite", getMaxType(boolQueryRequest, "site"));
    jsonObject.put("maxTitle", getMaxType(boolQueryRequest, "title.raw"));
    return jsonObject;
}
Also used : PageRequest(com.topcom.cms.common.page.PageRequest) JSONObject(net.sf.json.JSONObject) Page(org.springframework.data.domain.Page)

Example 64 with JSONObject

use of net.sf.json.JSONObject in project topcom-cloud by 545314690.

the class DescriptionManagerImpl method monthLyOutline.

@Override
public Object monthLyOutline(BoolQueryRequest boolQueryRequest) {
    boolQueryRequest.setPage(new PageRequest(1, 1));
    JSONObject jsonObject = new JSONObject();
    Page page = allESDataService.findByMustShouldDateInType(boolQueryRequest);
    jsonObject.put("monthCount", page.getTotalElements());
    jsonObject.put("mom", getmom(boolQueryRequest, "month"));
    jsonObject.put("maxSite", getMaxType(boolQueryRequest, "site"));
    jsonObject.put("maxType", getMaxType(boolQueryRequest, "type"));
    jsonObject.put("compare", compareWithLastMonthEverySite(boolQueryRequest, "month"));
    jsonObject.put("label", getMaxType(boolQueryRequest, "nlp.sentiment.label"));
    return jsonObject;
}
Also used : PageRequest(com.topcom.cms.common.page.PageRequest) JSONObject(net.sf.json.JSONObject) Page(org.springframework.data.domain.Page)

Example 65 with JSONObject

use of net.sf.json.JSONObject in project topcom-cloud by 545314690.

the class BriefingCreatorImpl method sendRequestAndCreateBriefing.

/**
 * 根据报告类型和开始结束时间发送请求生成json,并创建附件,保存到数据库
 *
 * @param briefingTask
 * @param startTime
 * @param endTime
 * @return
 * @throws BusinessException
 */
private Future sendRequestAndCreateBriefing(BriefingTask briefingTask, long startTime, long endTime) throws BusinessException {
    FutureTask<JSONObject> futureTask = new FutureTask<>(() -> {
        Briefing.BriefingType type = briefingTask.getBriefingType();
        List<Keywords> keywordList = keywordsManager.findByUserIdAndType(briefingTask.getUserId(), Keywords.Type.BASIC);
        if (keywordList.size() > 0) {
            Keywords keyword = keywordList.get(0);
            String mustWord = keyword.getMustWord();
            String shouldWord = keyword.getShouldWord();
            String mustNotWord = keyword.getMustNotWord();
            try {
                String object = requestBriefingString(type, mustWord, shouldWord, mustNotWord, startTime, endTime, null);
                DBObject dbObject = (BasicDBObject) JSON.parse(object);
                dbObject.put("userId", briefingTask.getUserId());
                dbObject.put("type", type.name());
                dbObject.put("issue", briefingTask.getIssue());
                JSONObject json = JSONObject.fromObject(dbObject);
                json.put("dateCreated", endTime);
                saveBriefingToMongo(json);
                String message = "生成报告成功@" + new Date() + "@" + type.name();
                LogUtil.logger.info(message);
                return json;
            } catch (Exception e) {
                e.printStackTrace();
                String message = "生成报告失败@" + new Date() + "@" + type.name();
                LogUtil.logger.error(message);
                throw new BusinessException(message);
            }
        }
        return null;
    });
    executorService.submit(futureTask);
    return futureTask;
}
Also used : Keywords(com.topcom.cms.yuqing.domain.Keywords) Briefing(com.topcom.cms.yuqing.domain.Briefing) DBObject(com.mongodb.DBObject) BasicDBObject(com.mongodb.BasicDBObject) BusinessException(com.topcom.cms.exception.BusinessException) UnsupportedEncodingException(java.io.UnsupportedEncodingException) BasicDBObject(com.mongodb.BasicDBObject) BusinessException(com.topcom.cms.exception.BusinessException) JSONObject(net.sf.json.JSONObject)

Aggregations

JSONObject (net.sf.json.JSONObject)493 Test (org.junit.Test)99 JSONArray (net.sf.json.JSONArray)94 IOException (java.io.IOException)49 HashMap (java.util.HashMap)48 ArrayList (java.util.ArrayList)36 JSON (net.sf.json.JSON)26 PrintWriter (java.io.PrintWriter)25 Map (java.util.Map)23 File (java.io.File)21 InputStream (java.io.InputStream)18 URISyntaxException (java.net.URISyntaxException)15 UnsupportedEncodingException (java.io.UnsupportedEncodingException)14 JsonConfig (net.sf.json.JsonConfig)14 FreeStyleBuild (hudson.model.FreeStyleBuild)13 URI (java.net.URI)13 URL (java.net.URL)13 JSONException (net.sf.json.JSONException)13 Context (org.zaproxy.zap.model.Context)12 Transactional (org.springframework.transaction.annotation.Transactional)11