Examples with ZuliaWorkPool io.zulia.client.pool.ZuliaWorkPool used on opensource projects

Example 1 with ZuliaWorkPool

use of io.zulia.client.pool.ZuliaWorkPool in project zuliasearch by zuliaio.

the class Zulia method main.

public static void main(String[] args) {
    LogUtil.init();
    ZuliaBaseArgs zuliaArgs = new ZuliaBaseArgs();
    GetIndexesCmd getIndexesCmd = new GetIndexesCmd();
    VersionCmd version = new VersionCmd();
    ClearCmd clear = new ClearCmd();
    OptimizeCmd optimize = new OptimizeCmd();
    GetCountCmd getCount = new GetCountCmd();
    GetCurrentNodesCmd getCurrentNodes = new GetCurrentNodesCmd();
    GetFieldsCmd getFields = new GetFieldsCmd();
    DeleteCmd delete = new DeleteCmd();
    ReindexCmd reindex = new ReindexCmd();
    QueryCmd query = new QueryCmd();
    JCommander jCommander = JCommander.newBuilder().addObject(zuliaArgs).addCommand(version).addCommand(getIndexesCmd).addCommand(query).addCommand(clear).addCommand(getCount).addCommand(getCurrentNodes).addCommand(getFields).addCommand(delete).addCommand(reindex).addCommand(optimize).build();
    try {
        jCommander.parse(args);
        if (jCommander.getParsedCommand() == null) {
            jCommander.usage();
            System.exit(2);
        }
        ZuliaPoolConfig config = new ZuliaPoolConfig().addNode(zuliaArgs.address, zuliaArgs.port);
        ZuliaWorkPool workPool = new ZuliaWorkPool(config);
        if ("--version".equalsIgnoreCase(jCommander.getParsedCommand())) {
            System.out.println(ZuliaVersion.getVersion());
            System.exit(0);
        }
        if ("getIndexes".equalsIgnoreCase(jCommander.getParsedCommand())) {
            GetIndexes getIndexes = new GetIndexes();
            GetIndexesResult execute = workPool.execute(getIndexes);
            System.out.println(execute.getIndexNames());
            System.exit(0);
        }
        String index = zuliaArgs.index;
        if (index == null) {
            System.err.println("Please pass in an index.");
            jCommander.usage();
            System.exit(2);
        }
        if ("query".equals(jCommander.getParsedCommand())) {
            Search search;
            if (query.indexes != null) {
                search = new Search(query.indexes);
            } else {
                search = new Search(index);
            }
            search.setAmount(query.rows);
            ScoredQuery scoredQuery = new ScoredQuery(query.q);
            if (query.qf != null) {
                query.qf.forEach(scoredQuery::addQueryField);
            }
            if (query.minimumNumberShouldMatch != null) {
                scoredQuery.setMinShouldMatch(query.minimumNumberShouldMatch);
            }
            search.addQuery(scoredQuery);
            search.setStart(query.start);
            if (query.fetch.equalsIgnoreCase("full")) {
                search.setResultFetchType(ZuliaQuery.FetchType.FULL);
            }
            if (query.facets != null) {
                for (String facet : query.facets) {
                    search.addCountFacet(new CountFacet(facet).setTopN(query.facetCount).setTopNShard(query.facetShardCount));
                }
            }
            if (query.sortFields != null) {
                for (String sortField : query.sortFields) {
                    search.addSort(new Sort(sortField));
                }
                ;
            }
            if (query.sortDescFields != null) {
                for (String sortDesc : query.sortDescFields) {
                    search.addSort(new Sort(sortDesc).descending());
                }
            }
            if (query.fq != null) {
                for (String filterQuery : query.fq) {
                    search.addQuery(new FilterQuery(filterQuery));
                }
            }
            if (query.fl != null) {
                query.fl.forEach(search::addDocumentField);
            }
            if (query.flMask != null) {
                query.flMask.forEach(search::addDocumentMaskedField);
            }
            SearchResult searchResult = workPool.search(search);
            List<ZuliaQuery.ScoredResult> srList = searchResult.getResults();
            System.out.println("QueryTime: " + (searchResult.getCommandTimeMs()) + "ms");
            System.out.println("TotalResults: " + searchResult.getTotalHits());
            System.out.println("Results:");
            System.out.print("UniqueId");
            System.out.print("\t");
            System.out.print("Score");
            System.out.print("\t");
            System.out.print("Index");
            System.out.print("\t");
            System.out.print("Shard");
            System.out.print("\t");
            System.out.print("LuceneShardId");
            System.out.print("\t");
            System.out.print("Sort");
            System.out.print("\t");
            if (query.fetch.equalsIgnoreCase("full")) {
                System.out.print("Document");
            }
            System.out.println();
            for (ZuliaQuery.ScoredResult sr : srList) {
                System.out.print(sr.getUniqueId());
                System.out.print("\t");
                System.out.print(df.format(sr.getScore()));
                System.out.print("\t");
                System.out.print(sr.getIndexName());
                System.out.print("\t");
                System.out.print(sr.getShard());
                System.out.print("\t");
                System.out.print(sr.getLuceneShardId());
                System.out.print("\t");
                StringBuffer sb = new StringBuffer();
                if (sr.hasSortValues()) {
                    for (ZuliaQuery.SortValue sortValue : sr.getSortValues().getSortValueList()) {
                        if (sb.length() != 0) {
                            sb.append(",");
                        }
                        if (sortValue.getExists()) {
                            if (sortValue.getDateValue() != 0) {
                                sb.append(new Date(sortValue.getDateValue()));
                            } else if (sortValue.getDoubleValue() != 0) {
                                sb.append(sortValue.getDoubleValue());
                            } else if (sortValue.getFloatValue() != 0) {
                                sb.append(sortValue.getFloatValue());
                            } else if (sortValue.getIntegerValue() != 0) {
                                sb.append(sortValue.getIntegerValue());
                            } else if (sortValue.getLongValue() != 0) {
                                sb.append(sortValue.getLongValue());
                            } else if (sortValue.getStringValue() != null) {
                                sb.append(sortValue.getStringValue());
                            }
                        } else {
                            sb.append("!NULL!");
                        }
                    }
                }
                if (sb.length() != 0) {
                    System.out.print(sb);
                } else {
                    System.out.print("--");
                }
                if (query.fetch != null && query.fetch.equalsIgnoreCase("full")) {
                    System.out.print("\t");
                    if (sr.hasResultDocument()) {
                        ZuliaBase.ResultDocument resultDocument = sr.getResultDocument();
                        if (resultDocument.getDocument() != null) {
                            Document mongoDocument = new Document();
                            mongoDocument.putAll(ZuliaUtil.byteArrayToMongoDocument(resultDocument.getDocument().toByteArray()));
                            System.out.println(mongoDocument.toJson());
                        }
                    }
                }
                System.out.println();
            }
            if (!searchResult.getFacetGroups().isEmpty()) {
                System.out.println("Facets:");
                for (ZuliaQuery.FacetGroup fg : searchResult.getFacetGroups()) {
                    System.out.println();
                    System.out.println("--Facet on " + fg.getCountRequest().getFacetField().getLabel() + "--");
                    for (ZuliaQuery.FacetCount fc : fg.getFacetCountList()) {
                        System.out.print(fc.getFacet());
                        System.out.print("\t");
                        System.out.print(fc.getCount());
                        System.out.print("\t");
                        System.out.print("+" + fc.getMaxError());
                        System.out.println();
                    }
                    if (fg.getPossibleMissing()) {
                        System.out.println("Possible facets missing from top results for <" + fg.getCountRequest().getFacetField().getLabel() + "> with max count <" + fg.getMaxValuePossibleMissing() + ">");
                    }
                }
            }
        } else if ("clear".equals(jCommander.getParsedCommand())) {
            System.out.println("Clearing index: " + index);
            ClearIndexResult response = workPool.execute(new ClearIndex(index));
            System.out.println("Cleared index: " + index);
        } else if ("getCount".equals(jCommander.getParsedCommand())) {
            GetNumberOfDocsResult response = workPool.execute(new GetNumberOfDocs(index));
            System.out.println("Shards: " + response.getShardCountResponseCount());
            System.out.println("Count: " + response.getNumberOfDocs());
            for (ZuliaBase.ShardCountResponse scr : response.getShardCountResponses()) {
                System.out.println("Shard [" + scr.getShardNumber() + "] Count:\n" + scr.getNumberOfDocs());
            }
        } else if ("getCurrentNodes".equals(jCommander.getParsedCommand())) {
            GetNodesResult response = workPool.execute(new GetNodes());
            System.out.println("serverAddress\tservicePort\theartBeat\trestPort\tversion");
            for (ZuliaBase.Node val : response.getNodes()) {
                String nodeVersion = val.getVersion();
                if (nodeVersion == null || nodeVersion.isEmpty()) {
                    nodeVersion = "< " + ZuliaVersion.getVersionAdded();
                }
                System.out.println(val.getServerAddress() + "\t" + val.getServicePort() + "\t" + val.getHeartbeat() + "\t" + val.getRestPort() + "\t" + nodeVersion);
            }
        } else if ("getFields".equals(jCommander.getParsedCommand())) {
            GetFieldsResult response = workPool.execute(new io.zulia.client.command.GetFields(index));
            response.getFieldNames().forEach(System.out::println);
        } else if ("delete".equals(jCommander.getParsedCommand())) {
            System.out.println("Deleting index: " + index);
            DeleteIndexResult response = workPool.execute(new DeleteIndex(index));
            System.out.println("Deleted index: " + index);
        } else if ("reindex".equals(jCommander.getParsedCommand())) {
            if (index.contains("*")) {
                GetIndexesResult indexesResult = workPool.getIndexes();
                for (String ind : indexesResult.getIndexNames()) {
                    if (ind.startsWith(index.replace("*", ""))) {
                        System.out.println("Reindexing index: " + ind);
                        ReindexResult response = workPool.execute(new Reindex(ind));
                        System.out.println("Reindexed index: " + ind);
                    }
                }
            } else {
                System.out.println("Reindexing index: " + index);
                ReindexResult response = workPool.execute(new Reindex(index));
                System.out.println("Reindexed index: " + index);
            }
        } else if ("optimize".equals(jCommander.getParsedCommand())) {
            System.out.println("Optimizing index: " + index);
            OptimizeIndexResult response = workPool.execute(new OptimizeIndex(index));
            System.out.println("Optimized index: " + index);
        }
    } catch (Exception e) {
        if (e instanceof ParameterException) {
            System.err.println(e.getMessage());
            jCommander.usage();
            System.exit(2);
        } else {
            e.printStackTrace();
        }
    }
}

Example 2 with ZuliaWorkPool

use of io.zulia.client.pool.ZuliaWorkPool in project zuliasearch by zuliaio.

the class ZuliaCmdUtil method index.

public static void index(String inputDir, String recordsFilename, String idField, String index, ZuliaWorkPool workPool, AtomicInteger count, Integer threads, Boolean skipExistingFiles) throws Exception {
    WorkPool threadPool = new WorkPool(threads);
    try (BufferedReader b = new BufferedReader(new FileReader(recordsFilename))) {
        String line;
        while ((line = b.readLine()) != null) {
            final String record = line;
            threadPool.executeAsync((Callable<Void>) () -> {
                try {
                    Document document = Document.parse(record);
                    String id = null;
                    if (idField != null) {
                        id = document.getString(idField);
                    }
                    if (id == null) {
                        // fall through to just "id"
                        id = document.getString("id");
                    }
                    if (id == null) {
                        // if still null, throw exception
                        throw new RuntimeException("No id for record: " + document.toJson());
                    }
                    document.put("indexTime", new Date());
                    Store store = new Store(id, index);
                    store.setResultDocument(new ResultDocBuilder().setDocument(document));
                    workPool.store(store);
                    String fullPathToFile = inputDir + File.separator + id.replaceAll("/", "_") + ".zip";
                    if (Files.exists(Paths.get(fullPathToFile))) {
                        File destDir = new File(inputDir + File.separator + UUID.randomUUID() + "_tempWork");
                        byte[] buffer = new byte[1024];
                        try (ZipArchiveInputStream inputStream = new ZipArchiveInputStream(new FileInputStream(Paths.get(fullPathToFile).toFile()))) {
                            ZipArchiveEntry zipEntry;
                            while ((zipEntry = inputStream.getNextZipEntry()) != null) {
                                decompressZipEntryToDisk(destDir, buffer, inputStream, zipEntry);
                            }
                        }
                        // ensure the file was extractable
                        if (Files.exists(destDir.toPath())) {
                            List<Path> tempFiles = Files.list(destDir.toPath()).collect(Collectors.toList());
                            for (Path path : tempFiles) {
                                if (path.toFile().isDirectory()) {
                                    try {
                                        List<Path> filesPaths = Files.list(path).collect(Collectors.toList());
                                        Document meta = null;
                                        byte[] associatedBytes = new byte[0];
                                        String filename = null;
                                        for (Path filePath : filesPaths) {
                                            try {
                                                if (filePath.toFile().getName().endsWith("_metadata.json")) {
                                                    meta = Document.parse(Files.readString(filePath));
                                                } else {
                                                    associatedBytes = Files.readAllBytes(filePath);
                                                    filename = filePath.toFile().getName();
                                                }
                                            } catch (Throwable t) {
                                                LOG.log(Level.SEVERE, "Could not restore associated file <" + filename + ">", t);
                                            }
                                        }
                                        if (skipExistingFiles) {
                                            if (!fileExists(workPool, id, filename, index)) {
                                                storeAssociatedDoc(index, workPool, id, filename, meta, associatedBytes);
                                            }
                                        } else {
                                            storeAssociatedDoc(index, workPool, id, filename, meta, associatedBytes);
                                        }
                                    } catch (Throwable t) {
                                        LOG.log(Level.SEVERE, "Could not list the individual files for dir <" + path.getFileName() + ">");
                                    }
                                } else {
                                    LOG.log(Level.SEVERE, "Top level file that shouldn't exist: " + path.getFileName());
                                }
                            }
                            // clean up temp work
                            Files.walk(destDir.toPath()).sorted(Comparator.reverseOrder()).map(Path::toFile).forEach(File::delete);
                        } else {
                        // LOG.log(Level.SEVERE, "Could not extract file <" + fullPathToFile + ">");
                        }
                    }
                    int i = count.incrementAndGet();
                    if (i % 10000 == 0) {
                        LOG.info("So far indexed <" + i + "> for index <" + index + ">");
                    }
                    return null;
                } catch (Exception e) {
                    LOG.log(Level.SEVERE, e.getMessage(), e);
                    return null;
                }
            });
        }
    } finally {
        threadPool.shutdown();
    }
}

Example 3 with ZuliaWorkPool

use of io.zulia.client.pool.ZuliaWorkPool in project zuliasearch by zuliaio.

the class ZuliaImport method main.

public static void main(String[] args) {
    LogUtil.init();
    ZuliaImport.ZuliaImportArgs zuliaImportArgs = new ZuliaImportArgs();
    JCommander jCommander = JCommander.newBuilder().addObject(zuliaImportArgs).build();
    try {
        jCommander.parse(args);
        ZuliaPoolConfig zuliaPoolConfig = new ZuliaPoolConfig().addNode(zuliaImportArgs.address, zuliaImportArgs.port).setNodeUpdateEnabled(false);
        ZuliaWorkPool workPool = new ZuliaWorkPool(zuliaPoolConfig);
        String dir = zuliaImportArgs.dir;
        String index = zuliaImportArgs.index;
        String idField = zuliaImportArgs.idField;
        Integer threads = zuliaImportArgs.threads;
        Boolean skipExistingFiles = zuliaImportArgs.skipExistingFiles;
        if (index != null) {
            doImport(workPool, dir, index, idField, threads, skipExistingFiles);
        }
    } catch (ParameterException e) {
        System.err.println(e.getMessage());
        jCommander.usage();
        System.exit(2);
    } catch (UnsupportedOperationException e) {
        System.err.println("Error: " + e.getMessage());
        System.exit(2);
    } catch (Exception e) {
        System.err.println("Error: " + e.getMessage());
        e.printStackTrace();
        System.exit(1);
    }
}

Example 4 with ZuliaWorkPool

use of io.zulia.client.pool.ZuliaWorkPool in project zuliasearch by zuliaio.

the class ZuliaRestore method main.

public static void main(String[] args) {
    LogUtil.init();
    ZuliaRestoreArgs zuliaRestoreArgs = new ZuliaRestoreArgs();
    JCommander jCommander = JCommander.newBuilder().addObject(zuliaRestoreArgs).build();
    try {
        jCommander.parse(args);
        ZuliaPoolConfig zuliaPoolConfig = new ZuliaPoolConfig().addNode(zuliaRestoreArgs.address, zuliaRestoreArgs.port).setNodeUpdateEnabled(false);
        ZuliaWorkPool workPool = new ZuliaWorkPool(zuliaPoolConfig);
        String dir = zuliaRestoreArgs.dir;
        String index = zuliaRestoreArgs.index;
        String idField = zuliaRestoreArgs.idField;
        Boolean drop = zuliaRestoreArgs.drop;
        Integer threads = zuliaRestoreArgs.threads;
        Boolean skipExistingFiles = zuliaRestoreArgs.skipExistingFiles;
        if (index != null) {
            // restore only this index
            restore(workPool, dir, index, idField, drop, threads, skipExistingFiles);
        } else {
            // walk dir and restore everything
            Files.list(Paths.get(dir)).forEach(indexDir -> {
                try {
                    String ind = indexDir.getFileName().toString();
                    restore(workPool, dir, ind, idField, drop, threads, skipExistingFiles);
                } catch (Exception e) {
                    LOG.log(Level.SEVERE, "There was a problem restoring index <" + indexDir.getFileName() + ">", e);
                }
            });
        }
    } catch (ParameterException e) {
        System.err.println(e.getMessage());
        jCommander.usage();
        System.exit(2);
    } catch (UnsupportedOperationException e) {
        System.err.println("Error: " + e.getMessage());
        System.exit(2);
    } catch (Exception e) {
        System.err.println("Error: " + e.getMessage());
        e.printStackTrace();
        System.exit(1);
    }
}

Example 5 with ZuliaWorkPool

use of io.zulia.client.pool.ZuliaWorkPool in project zuliasearch by zuliaio.

the class ZuliaStoreFile method main.

public static void main(String[] args) {
    LogUtil.init();
    ZuliaStoreFileArgs zuliaStoreFileArgs = new ZuliaStoreFileArgs();
    JCommander jCommander = JCommander.newBuilder().addObject(zuliaStoreFileArgs).build();
    try {
        jCommander.parse(args);
        ZuliaPoolConfig zuliaPoolConfig = new ZuliaPoolConfig().addNode(zuliaStoreFileArgs.address, zuliaStoreFileArgs.port).setNodeUpdateEnabled(false);
        ZuliaWorkPool workPool = new ZuliaWorkPool(zuliaPoolConfig);
        // String uniqueId, String indexName, String fileName, File fileToStore
        File fileToStore = new File(zuliaStoreFileArgs.fileToStore);
        if (!fileToStore.exists()) {
            System.err.println("File " + fileToStore.getAbsolutePath() + " does not exist");
            System.exit(3);
        }
        StoreLargeAssociated storeLargeAssociated = new StoreLargeAssociated(zuliaStoreFileArgs.id, zuliaStoreFileArgs.index, zuliaStoreFileArgs.fileName, Files.readAllBytes(fileToStore.toPath()));
        workPool.storeLargeAssociated(storeLargeAssociated);
    } catch (ParameterException e) {
        System.err.println(e.getMessage());
        jCommander.usage();
        System.exit(2);
    } catch (UnsupportedOperationException e) {
        System.err.println("Error: " + e.getMessage());
        System.exit(2);
    } catch (Exception e) {
        System.err.println("Error: " + e.getMessage());
        e.printStackTrace();
        System.exit(1);
    }
}