Examples with ResultDocBuilder io.zulia.doc.ResultDocBuilder used on opensource projects

Example 1 with ResultDocBuilder

use of io.zulia.doc.ResultDocBuilder in project zuliasearch by zuliaio.

the class AliasTest method indexRecord.

private void indexRecord(String index, int id, String title, Double rating) throws Exception {
    String uniqueId = "" + id;
    Document mongoDocument = new Document();
    mongoDocument.put("id", uniqueId);
    mongoDocument.put("title", title);
    mongoDocument.put("rating", rating);
    Store s = new Store(uniqueId, index);
    ResultDocBuilder resultDocumentBuilder = ResultDocBuilder.newBuilder().setDocument(mongoDocument);
    s.setResultDocument(resultDocumentBuilder);
    zuliaWorkPool.store(s);
}

Example 2 with ResultDocBuilder

use of io.zulia.doc.ResultDocBuilder in project zuliasearch by zuliaio.

the class ZuliaCmdUtil method index.

public static void index(String inputDir, String recordsFilename, String idField, String index, ZuliaWorkPool workPool, AtomicInteger count, Integer threads, Boolean skipExistingFiles) throws Exception {
    WorkPool threadPool = new WorkPool(threads);
    try (BufferedReader b = new BufferedReader(new FileReader(recordsFilename))) {
        String line;
        while ((line = b.readLine()) != null) {
            final String record = line;
            threadPool.executeAsync((Callable<Void>) () -> {
                try {
                    Document document = Document.parse(record);
                    String id = null;
                    if (idField != null) {
                        id = document.getString(idField);
                    }
                    if (id == null) {
                        // fall through to just "id"
                        id = document.getString("id");
                    }
                    if (id == null) {
                        // if still null, throw exception
                        throw new RuntimeException("No id for record: " + document.toJson());
                    }
                    document.put("indexTime", new Date());
                    Store store = new Store(id, index);
                    store.setResultDocument(new ResultDocBuilder().setDocument(document));
                    workPool.store(store);
                    String fullPathToFile = inputDir + File.separator + id.replaceAll("/", "_") + ".zip";
                    if (Files.exists(Paths.get(fullPathToFile))) {
                        File destDir = new File(inputDir + File.separator + UUID.randomUUID() + "_tempWork");
                        byte[] buffer = new byte[1024];
                        try (ZipArchiveInputStream inputStream = new ZipArchiveInputStream(new FileInputStream(Paths.get(fullPathToFile).toFile()))) {
                            ZipArchiveEntry zipEntry;
                            while ((zipEntry = inputStream.getNextZipEntry()) != null) {
                                decompressZipEntryToDisk(destDir, buffer, inputStream, zipEntry);
                            }
                        }
                        // ensure the file was extractable
                        if (Files.exists(destDir.toPath())) {
                            List<Path> tempFiles = Files.list(destDir.toPath()).collect(Collectors.toList());
                            for (Path path : tempFiles) {
                                if (path.toFile().isDirectory()) {
                                    try {
                                        List<Path> filesPaths = Files.list(path).collect(Collectors.toList());
                                        Document meta = null;
                                        byte[] associatedBytes = new byte[0];
                                        String filename = null;
                                        for (Path filePath : filesPaths) {
                                            try {
                                                if (filePath.toFile().getName().endsWith("_metadata.json")) {
                                                    meta = Document.parse(Files.readString(filePath));
                                                } else {
                                                    associatedBytes = Files.readAllBytes(filePath);
                                                    filename = filePath.toFile().getName();
                                                }
                                            } catch (Throwable t) {
                                                LOG.log(Level.SEVERE, "Could not restore associated file <" + filename + ">", t);
                                            }
                                        }
                                        if (skipExistingFiles) {
                                            if (!fileExists(workPool, id, filename, index)) {
                                                storeAssociatedDoc(index, workPool, id, filename, meta, associatedBytes);
                                            }
                                        } else {
                                            storeAssociatedDoc(index, workPool, id, filename, meta, associatedBytes);
                                        }
                                    } catch (Throwable t) {
                                        LOG.log(Level.SEVERE, "Could not list the individual files for dir <" + path.getFileName() + ">");
                                    }
                                } else {
                                    LOG.log(Level.SEVERE, "Top level file that shouldn't exist: " + path.getFileName());
                                }
                            }
                            // clean up temp work
                            Files.walk(destDir.toPath()).sorted(Comparator.reverseOrder()).map(Path::toFile).forEach(File::delete);
                        } else {
                        // LOG.log(Level.SEVERE, "Could not extract file <" + fullPathToFile + ">");
                        }
                    }
                    int i = count.incrementAndGet();
                    if (i % 10000 == 0) {
                        LOG.info("So far indexed <" + i + "> for index <" + index + ">");
                    }
                    return null;
                } catch (Exception e) {
                    LOG.log(Level.SEVERE, e.getMessage(), e);
                    return null;
                }
            });
        }
    } finally {
        threadPool.shutdown();
    }
}

Example 3 with ResultDocBuilder

use of io.zulia.doc.ResultDocBuilder in project zuliasearch by zuliaio.

the class Mapper method createStore.

public Store createStore(String indexName, T object) throws Exception {
    ResultDocBuilder rd = toResultDocumentBuilder(object);
    Store store = new Store(rd.getUniqueId(), indexName);
    store.setResultDocument(rd);
    return store;
}

Example 4 with ResultDocBuilder

use of io.zulia.doc.ResultDocBuilder in project zuliasearch by zuliaio.

the class FileStorageTest method indexDocument.

private void indexDocument(Document document) throws Exception {
    Store store = new Store(document.getString("id"), TEST_INDEX);
    store.setResultDocument(new ResultDocBuilder().setDocument(document));
    zuliaWorkPool.store(store);
}

Example 5 with ResultDocBuilder

use of io.zulia.doc.ResultDocBuilder in project zuliasearch by zuliaio.

the class StartStopTest method indexRecord.

private void indexRecord(int id, String issn, String eissn, int i) throws Exception {
    boolean half = (i % 2 == 0);
    boolean tenth = (i % 10 == 0);
    String uniqueId = uniqueIdPrefix + id;
    Document mongoDocument = new Document();
    mongoDocument.put("issn", issn);
    mongoDocument.put("eissn", eissn);
    mongoDocument.put("id", id);
    if (id != 0) {
        mongoDocument.put("an", id);
    }
    if (id == 0) {
        mongoDocument.put("testBool", true);
    } else if (id == 1) {
        mongoDocument.put("testBool", 1);
    } else if (id == 2) {
        mongoDocument.put("testBool", "Yes");
    } else if (id == 3) {
        mongoDocument.put("testBool", "true");
    } else if (id == 4) {
        mongoDocument.put("testBool", "Y");
    } else if (id == 5) {
        mongoDocument.put("testBool", "T");
    } else if (id == 6) {
        mongoDocument.put("testBool", false);
    } else if (id == 7) {
        mongoDocument.put("testBool", "False");
    } else if (id == 8) {
        mongoDocument.put("testBool", "F");
    } else if (id == 9) {
        mongoDocument.put("testBool", 0);
    }
    if (!issn.equals("3331-3333")) {
        if (half) {
            mongoDocument.put("title", "Facet Userguide");
        } else {
            mongoDocument.put("title", "Special Userguide");
        }
    }
    if (half) {
        // 1/2 of input
        mongoDocument.put("country", "US");
        mongoDocument.put("testList", Arrays.asList("one", "two"));
    } else {
        // 1/2 of input
        mongoDocument.put("country", "France");
        mongoDocument.put("testList", Arrays.asList("a", "b", "c"));
    }
    if (tenth) {
        // 1/10 of input
        Date d = Date.from(LocalDate.of(2014, Month.OCTOBER, 4).atStartOfDay(ZoneId.of("UTC")).toInstant());
        mongoDocument.put("date", d);
    } else if (half) {
        // 2/5 of input
        Date d = Date.from(LocalDate.of(2013, Month.SEPTEMBER, 4).atStartOfDay(ZoneId.of("UTC")).toInstant());
        mongoDocument.put("date", d);
    } else {
        // 1/2 of input
        Date d = Date.from(LocalDate.of(2012, 8, 4).atStartOfDay(ZoneId.of("UTC")).toInstant());
        mongoDocument.put("date", d);
    }
    Store s = new Store(uniqueId, FACET_TEST_INDEX);
    ResultDocBuilder resultDocumentBuilder = ResultDocBuilder.newBuilder().setDocument(mongoDocument);
    if (half) {
        resultDocumentBuilder.setMetadata(new Document("test", "someValue"));
    }
    s.setResultDocument(resultDocumentBuilder);
    zuliaWorkPool.store(s);
}