Examples with Store io.zulia.client.command.Store used on opensource projects

Example 1 with Store

use of io.zulia.client.command.Store in project zuliasearch by zuliaio.

the class AliasTest method indexRecord.

private void indexRecord(String index, int id, String title, Double rating) throws Exception {
    String uniqueId = "" + id;
    Document mongoDocument = new Document();
    mongoDocument.put("id", uniqueId);
    mongoDocument.put("title", title);
    mongoDocument.put("rating", rating);
    Store s = new Store(uniqueId, index);
    ResultDocBuilder resultDocumentBuilder = ResultDocBuilder.newBuilder().setDocument(mongoDocument);
    s.setResultDocument(resultDocumentBuilder);
    zuliaWorkPool.store(s);
}

Example 2 with Store

use of io.zulia.client.command.Store in project zuliasearch by zuliaio.

the class ZuliaCmdUtil method storeAssociatedDoc.

private static void storeAssociatedDoc(String index, ZuliaWorkPool workPool, String id, String filename, Document meta, byte[] associatedBytes) throws Exception {
    if (associatedBytes.length > 32 * 1024 * 1024) {
        workPool.storeLargeAssociated(new StoreLargeAssociated(id, index, filename, associatedBytes).setMeta(meta));
    } else {
        Store associatedDocStore = new Store(id, index);
        associatedDocStore.addAssociatedDocument(AssociatedBuilder.newBuilder().setDocument(associatedBytes).setMetadata(meta).setFilename(filename));
        workPool.store(associatedDocStore);
    }
}

Example 3 with Store

use of io.zulia.client.command.Store in project zuliasearch by zuliaio.

the class ZuliaCmdUtil method index.

public static void index(String inputDir, String recordsFilename, String idField, String index, ZuliaWorkPool workPool, AtomicInteger count, Integer threads, Boolean skipExistingFiles) throws Exception {
    WorkPool threadPool = new WorkPool(threads);
    try (BufferedReader b = new BufferedReader(new FileReader(recordsFilename))) {
        String line;
        while ((line = b.readLine()) != null) {
            final String record = line;
            threadPool.executeAsync((Callable<Void>) () -> {
                try {
                    Document document = Document.parse(record);
                    String id = null;
                    if (idField != null) {
                        id = document.getString(idField);
                    }
                    if (id == null) {
                        // fall through to just "id"
                        id = document.getString("id");
                    }
                    if (id == null) {
                        // if still null, throw exception
                        throw new RuntimeException("No id for record: " + document.toJson());
                    }
                    document.put("indexTime", new Date());
                    Store store = new Store(id, index);
                    store.setResultDocument(new ResultDocBuilder().setDocument(document));
                    workPool.store(store);
                    String fullPathToFile = inputDir + File.separator + id.replaceAll("/", "_") + ".zip";
                    if (Files.exists(Paths.get(fullPathToFile))) {
                        File destDir = new File(inputDir + File.separator + UUID.randomUUID() + "_tempWork");
                        byte[] buffer = new byte[1024];
                        try (ZipArchiveInputStream inputStream = new ZipArchiveInputStream(new FileInputStream(Paths.get(fullPathToFile).toFile()))) {
                            ZipArchiveEntry zipEntry;
                            while ((zipEntry = inputStream.getNextZipEntry()) != null) {
                                decompressZipEntryToDisk(destDir, buffer, inputStream, zipEntry);
                            }
                        }
                        // ensure the file was extractable
                        if (Files.exists(destDir.toPath())) {
                            List<Path> tempFiles = Files.list(destDir.toPath()).collect(Collectors.toList());
                            for (Path path : tempFiles) {
                                if (path.toFile().isDirectory()) {
                                    try {
                                        List<Path> filesPaths = Files.list(path).collect(Collectors.toList());
                                        Document meta = null;
                                        byte[] associatedBytes = new byte[0];
                                        String filename = null;
                                        for (Path filePath : filesPaths) {
                                            try {
                                                if (filePath.toFile().getName().endsWith("_metadata.json")) {
                                                    meta = Document.parse(Files.readString(filePath));
                                                } else {
                                                    associatedBytes = Files.readAllBytes(filePath);
                                                    filename = filePath.toFile().getName();
                                                }
                                            } catch (Throwable t) {
                                                LOG.log(Level.SEVERE, "Could not restore associated file <" + filename + ">", t);
                                            }
                                        }
                                        if (skipExistingFiles) {
                                            if (!fileExists(workPool, id, filename, index)) {
                                                storeAssociatedDoc(index, workPool, id, filename, meta, associatedBytes);
                                            }
                                        } else {
                                            storeAssociatedDoc(index, workPool, id, filename, meta, associatedBytes);
                                        }
                                    } catch (Throwable t) {
                                        LOG.log(Level.SEVERE, "Could not list the individual files for dir <" + path.getFileName() + ">");
                                    }
                                } else {
                                    LOG.log(Level.SEVERE, "Top level file that shouldn't exist: " + path.getFileName());
                                }
                            }
                            // clean up temp work
                            Files.walk(destDir.toPath()).sorted(Comparator.reverseOrder()).map(Path::toFile).forEach(File::delete);
                        } else {
                        // LOG.log(Level.SEVERE, "Could not extract file <" + fullPathToFile + ">");
                        }
                    }
                    int i = count.incrementAndGet();
                    if (i % 10000 == 0) {
                        LOG.info("So far indexed <" + i + "> for index <" + index + ">");
                    }
                    return null;
                } catch (Exception e) {
                    LOG.log(Level.SEVERE, e.getMessage(), e);
                    return null;
                }
            });
        }
    } finally {
        threadPool.shutdown();
    }
}

Example 4 with Store

use of io.zulia.client.command.Store in project zuliasearch by zuliaio.

the class Mapper method createStore.

public Store createStore(String indexName, T object) throws Exception {
    ResultDocBuilder rd = toResultDocumentBuilder(object);
    Store store = new Store(rd.getUniqueId(), indexName);
    store.setResultDocument(rd);
    return store;
}

Example 5 with Store

use of io.zulia.client.command.Store in project zuliasearch by zuliaio.

the class FileStorageTest method storeFile.

private void storeFile(String documentId, String filename, Document meta, byte[] content) throws Exception {
    zuliaWorkPool.delete(new DeleteAssociated(documentId, TEST_INDEX, filename));
    if (content.length > 32 * 1024 * 1024) {
        StoreLargeAssociated storeLargeAssociated = new StoreLargeAssociated(documentId, TEST_INDEX, filename, content);
        storeLargeAssociated.setMeta(meta);
        zuliaWorkPool.storeLargeAssociated(storeLargeAssociated);
    } else {
        Store associatedDocStore = new Store(documentId, TEST_INDEX);
        associatedDocStore.addAssociatedDocument(AssociatedBuilder.newBuilder().setDocument(content).setFilename(filename).setMetadata(meta));
        zuliaWorkPool.store(associatedDocStore);
    }
}