Examples with Document de.tblsoft.solr.pipeline.bean.Document used on opensource projects

Example 31 with Document

use of de.tblsoft.solr.pipeline.bean.Document in project solr-cmd-utils by tblsoft.

the class JsonPathReader method read.

public void read() {
    SimpleMapping simpleMapping = new SimpleMapping(getPropertyAsList("mapping", new ArrayList<String>()));
    Map<String, List<String>> mapping = simpleMapping.getMapping();
    try {
        String rootPath = getProperty("rootPath", "$");
        DocumentContext context = loadJsonContext();
        List<Object> jsonHits = context.read(rootPath);
        for (Object obj : jsonHits) {
            Document document = new Document();
            for (Map.Entry<String, List<String>> mappingEntry : mapping.entrySet()) {
                try {
                    Object parsedValue = JsonPath.parse(obj).read(mappingEntry.getKey());
                    for (String target : mappingEntry.getValue()) {
                        document.setField(target, parsedValue);
                    }
                } catch (PathNotFoundException e) {
                // ignore
                }
            }
            executer.document(document);
        }
    } catch (Exception e) {
        throw new RuntimeException(e);
    }
}

Example 32 with Document

use of de.tblsoft.solr.pipeline.bean.Document in project solr-cmd-utils by tblsoft.

the class JsonReader method read.

public void read() {
    try {
        ScriptEngineManager mgr = new ScriptEngineManager();
        engine = mgr.getEngineByName("JavaScript");
        String internalFilename = getProperty("filename", null);
        filename = IOUtils.getAbsoluteFile(getBaseDir(), internalFilename);
        String internalJavaScriptFilename = getProperty("javaScriptFilename", null);
        javaScriptFilename = IOUtils.getAbsoluteFile(getBaseDir(), internalJavaScriptFilename);
        rootPath = getProperty("rootPath", "$");
        script = FileUtils.readFileToString(new File(javaScriptFilename));
        cx = Context.enter();
        JsonSurfer surfer = JsonSurferGson.INSTANCE;
        java.io.Reader sample = new FileReader(filename);
        surfer.configBuilder().bind(rootPath, new JsonPathListener() {

            public void onValue(Object value, ParsingContext context) {
                Scriptable scope = cx.initStandardObjects();
                List<Document> output = new ArrayList<Document>();
                ScriptableObject.putProperty(scope, "documentBuilder", Context.javaToJS(new DocumentBuilder(), scope));
                ScriptableObject.putProperty(scope, "output", Context.javaToJS(output, scope));
                String exec = "var input = " + value.toString() + ";" + script;
                cx.evaluateString(scope, exec, filename, 1, null);
                for (Document out : output) {
                    executer.document(out);
                }
            }
        }).buildAndSurf(sample);
        sample.close();
    } catch (Exception e) {
        throw new RuntimeException(e);
    }
}

Example 33 with Document

use of de.tblsoft.solr.pipeline.bean.Document in project solr-cmd-utils by tblsoft.

the class GrokFilter method processLine.

void processLine(String line) {
    // executer.field("raw", line);
    // System.out.println(line);
    Match gm = grok.match(line);
    gm.captures();
    Map<String, Object> m = gm.toMap();
    Document document = new Document();
    for (Map.Entry<String, Object> entry : m.entrySet()) {
        Object value = entry.getValue();
        document.addField(entry.getKey(), String.valueOf(value));
    }
/*
        if (!m.isEmpty()) {
            document.addField("filename", currentFileName);
            if (keepRaw) {
                document.addField("raw", line);
            }
        }
        executer.document(document);
        */
}

Example 34 with Document

use of de.tblsoft.solr.pipeline.bean.Document in project solr-cmd-utils by tblsoft.

the class HttpFilter method processQueue.

void processQueue() {
    List<Future<Document>> documentFutures = new ArrayList<Future<Document>>();
    for (Document documentFromQueue : documentQueue) {
        HttpWorker worker = new HttpWorker(documentFromQueue, httpclient, urlField, userAgent);
        Future<Document> future = executor.submit(worker);
        documentFutures.add(future);
    }
    for (Future<Document> documentFuture : documentFutures) {
        try {
            Document d = documentFuture.get();
            super.document(d);
        } catch (InterruptedException e) {
            throw new RuntimeException(e);
        } catch (ExecutionException e) {
            throw new RuntimeException(e);
        }
    }
    documentQueue.clear();
}

Example 35 with Document

use of de.tblsoft.solr.pipeline.bean.Document in project solr-cmd-utils by tblsoft.

the class JsonWriter method mapToJsonString.

public static String mapToJsonString(List<Document> documentList) {
    List<Map<String, Object>> documentMap = new ArrayList<Map<String, Object>>();
    for (Document document : documentList) {
        documentMap.add(mapToJson(document));
    }
    Gson gson = new GsonBuilder().setPrettyPrinting().create();
    String json = gson.toJson(documentMap);
    return json;
}