use of net.sf.json.JSONObject in project topcom-cloud by 545314690.
the class EtlController method bbsEtl.
/**
* 微博id->weiboId
* md5(id)->微博id
* @param limit
* @return
*/
@ApiOperation("bbsEtl")
@RequestMapping(value = "/bbsEtl", method = RequestMethod.GET, produces = "application/json")
@ResponseBody
public Object bbsEtl(@RequestParam Integer limit) {
long count = 0;
long count1 = 0;
SearchResponse searchResponse = scanBbs("", limit);
while (searchResponse.getHits().getHits().length > 0) {
SearchHit[] hits = searchResponse.getHits().hits();
List<JSONObject> commentsList = new ArrayList<>();
for (int i = 0; i < hits.length; i++) {
JSONObject jsonObject = JSONObject.fromObject(hits[i].sourceAsMap());
String type = jsonObject.get("type").toString();
if (type.equals("Bbs")) {
Object object = jsonObject.get("floorNum");
if (object != null && Integer.valueOf(object.toString()) > 0) {
commentsList.add(jsonObject);
}
}
}
// String path = "D:\\data\\bbs\\"+"bbs"+(count1/50000)+".txt";
String path = "/home/yuqing/data/esFile/" + "bbs" + (count1 / 50000) + ".txt";
save2File(commentsList, path);
dleteEsDoc("yuqing", "article", commentsList);
count1 = count1 + commentsList.size();
count = count + searchResponse.getHits().getHits().length;
logger.error("删除BBS条数:" + count1);
logger.error("处理BBS条数:" + count + "\n 完成度" + (double) count * 100 / searchResponse.getHits().getTotalHits() + "%");
searchResponse = scanBbs(searchResponse.getScrollId(), limit);
}
return "done";
}
use of net.sf.json.JSONObject in project topcom-cloud by 545314690.
the class EtlController method dleteEsDoc.
private void dleteEsDoc(String yuqing, String article, List<JSONObject> commentsList) {
Client client = elasticsearchTemplate.getClient();
BulkRequestBuilder bulkdel = client.prepareBulk();
for (int i = 0; i < commentsList.size(); i++) {
JSONObject jsonObject = commentsList.get(i);
bulkdel.add(client.prepareDelete("yuqing", "article", jsonObject.get("id").toString()).request());
}
bulkdel.execute().actionGet();
}
use of net.sf.json.JSONObject in project topcom-cloud by 545314690.
the class DescriptionManagerImpl method specialOutline.
@Override
public Object specialOutline(BoolQueryRequest boolQueryRequest) {
boolQueryRequest.setPage(new PageRequest(1, 1));
JSONObject jsonObject = new JSONObject();
Page page = allESDataService.findByMustShouldDateInType(boolQueryRequest);
jsonObject.put("monthCount", page.getTotalElements());
jsonObject.put("maxType", getMaxType(boolQueryRequest, "type"));
jsonObject.put("label", getMaxType(boolQueryRequest, "nlp.sentiment.label"));
boolQueryRequest.setType("news");
jsonObject.put("maxSite", getMaxType(boolQueryRequest, "site"));
jsonObject.put("maxTitle", getMaxType(boolQueryRequest, "title.raw"));
return jsonObject;
}
use of net.sf.json.JSONObject in project topcom-cloud by 545314690.
the class DescriptionManagerImpl method monthLyOutline.
@Override
public Object monthLyOutline(BoolQueryRequest boolQueryRequest) {
boolQueryRequest.setPage(new PageRequest(1, 1));
JSONObject jsonObject = new JSONObject();
Page page = allESDataService.findByMustShouldDateInType(boolQueryRequest);
jsonObject.put("monthCount", page.getTotalElements());
jsonObject.put("mom", getmom(boolQueryRequest, "month"));
jsonObject.put("maxSite", getMaxType(boolQueryRequest, "site"));
jsonObject.put("maxType", getMaxType(boolQueryRequest, "type"));
jsonObject.put("compare", compareWithLastMonthEverySite(boolQueryRequest, "month"));
jsonObject.put("label", getMaxType(boolQueryRequest, "nlp.sentiment.label"));
return jsonObject;
}
use of net.sf.json.JSONObject in project topcom-cloud by 545314690.
the class BriefingCreatorImpl method sendRequestAndCreateBriefing.
/**
* 根据报告类型和开始结束时间发送请求生成json,并创建附件,保存到数据库
*
* @param briefingTask
* @param startTime
* @param endTime
* @return
* @throws BusinessException
*/
private Future sendRequestAndCreateBriefing(BriefingTask briefingTask, long startTime, long endTime) throws BusinessException {
FutureTask<JSONObject> futureTask = new FutureTask<>(() -> {
Briefing.BriefingType type = briefingTask.getBriefingType();
List<Keywords> keywordList = keywordsManager.findByUserIdAndType(briefingTask.getUserId(), Keywords.Type.BASIC);
if (keywordList.size() > 0) {
Keywords keyword = keywordList.get(0);
String mustWord = keyword.getMustWord();
String shouldWord = keyword.getShouldWord();
String mustNotWord = keyword.getMustNotWord();
try {
String object = requestBriefingString(type, mustWord, shouldWord, mustNotWord, startTime, endTime, null);
DBObject dbObject = (BasicDBObject) JSON.parse(object);
dbObject.put("userId", briefingTask.getUserId());
dbObject.put("type", type.name());
dbObject.put("issue", briefingTask.getIssue());
JSONObject json = JSONObject.fromObject(dbObject);
json.put("dateCreated", endTime);
saveBriefingToMongo(json);
String message = "生成报告成功@" + new Date() + "@" + type.name();
LogUtil.logger.info(message);
return json;
} catch (Exception e) {
e.printStackTrace();
String message = "生成报告失败@" + new Date() + "@" + type.name();
LogUtil.logger.error(message);
throw new BusinessException(message);
}
}
return null;
});
executorService.submit(futureTask);
return futureTask;
}
Aggregations