Examples with CatalogFacade org.codice.ddf.commands.catalog.facade.CatalogFacade used on opensource projects

Example 1 with CatalogFacade

use of org.codice.ddf.commands.catalog.facade.CatalogFacade in project ddf by codice.

the class DuplicateCommands method duplicateInBatches.

/**
 * In batches, loops through a query of the queryFacade and an ingest to the ingestFacade of the
 * metacards from the response until there are no more metacards from the queryFacade or the
 * maxMetacards has been reached.
 *
 * @param queryFacade - the CatalogFacade to duplicate from
 * @param ingestFacade - the CatalogFacade to duplicate to
 * @param filter - the filter to query with
 */
protected void duplicateInBatches(CatalogFacade queryFacade, CatalogFacade ingestFacade, Filter filter, String sourceId) throws InterruptedException {
    AtomicInteger queryIndex = new AtomicInteger(1);
    final long originalQuerySize;
    if (maxMetacards > 0 && maxMetacards < batchSize) {
        originalQuerySize = maxMetacards;
    } else {
        originalQuerySize = batchSize;
    }
    Function<Integer, QueryRequest> queryTemplate = (index) -> new QueryRequestImpl(new QueryImpl(filter, index, (int) originalQuerySize, new SortByImpl(Metacard.EFFECTIVE, SortOrder.DESCENDING), true, TimeUnit.MINUTES.toMillis(5)), Collections.singletonList(sourceId));
    List<Metacard> initialMetacards = ResultIterable.resultIterable((queryRequest -> {
        SourceResponse response = queryFacade.query(queryRequest);
        if (response.getHits() != -1) {
            maxMetacards = (int) response.getHits();
        }
        return response;
    }), queryTemplate.apply(queryIndex.get()), (int) originalQuerySize).stream().map(Result::getMetacard).collect(Collectors.toList());
    if (initialMetacards.isEmpty()) {
        LOGGER.debug("Query returned 0 results.");
        console.println(String.format("No results were returned by the source [%s]", sourceId));
        return;
    }
    ingestMetacards(ingestFacade, initialMetacards);
    if (initialMetacards.size() < originalQuerySize) {
        // all done if results exhausted in the first batch
        printProgressAndFlush(start, maxMetacards < 1 ? initialMetacards.size() : maxMetacards, ingestedCount.get());
        return;
    }
    final long totalWanted = maxMetacards;
    final AtomicBoolean done = new AtomicBoolean(false);
    if (multithreaded > 1) {
        BlockingQueue<Runnable> blockingQueue = new ArrayBlockingQueue<>(multithreaded);
        RejectedExecutionHandler rejectedExecutionHandler = new ThreadPoolExecutor.CallerRunsPolicy();
        final ExecutorService executorService = new ThreadPoolExecutor(multithreaded, multithreaded, 0L, TimeUnit.MILLISECONDS, blockingQueue, StandardThreadFactoryBuilder.newThreadFactory("duplicateCommandsThread"), rejectedExecutionHandler);
        console.printf("Running a maximum of %d threads during replication.%n", multithreaded);
        printProgressAndFlush(start, Math.max(totalWanted, initialMetacards.size()), ingestedCount.get());
        int index;
        while (!done.get()) {
            index = queryIndex.addAndGet(batchSize);
            final int taskIndex = index;
            executorService.submit(() -> {
                int querySize = (int) getQuerySizeFromIndex(totalWanted, taskIndex);
                if (querySize < 1) {
                    // If we don't need any more metacards, we're finished
                    done.set(true);
                    return;
                }
                List<Metacard> metacards = ResultIterable.resultIterable(queryFacade::query, queryTemplate.apply(taskIndex), querySize).stream().map(Result::getMetacard).collect(Collectors.toList());
                if (metacards.size() < querySize) {
                    done.set(true);
                }
                if (!metacards.isEmpty()) {
                    ingestMetacards(ingestFacade, metacards);
                }
                printProgressAndFlush(start, Math.max(totalWanted, ingestedCount.get()), ingestedCount.get());
            });
        }
        executorService.shutdown();
        try {
            executorService.awaitTermination(1, TimeUnit.SECONDS);
        } catch (InterruptedException e) {
            executorService.shutdownNow();
            throw e;
        }
        printProgressAndFlush(start, Math.max(totalWanted, ingestedCount.get()), ingestedCount.get());
    } else {
        // Single threaded
        ResultIterable iter;
        if (maxMetacards > 0) {
            iter = ResultIterable.resultIterable(queryFacade::query, queryTemplate.apply(1 + batchSize), maxMetacards);
        } else {
            iter = ResultIterable.resultIterable(queryFacade::query, queryTemplate.apply(1 + batchSize));
        }
        Iterables.partition(iter, batchSize).forEach((batch) -> {
            printProgressAndFlush(start, totalWanted, ingestedCount.get());
            if (batch.isEmpty()) {
                return;
            }
            ingestMetacards(ingestFacade, batch.stream().map(Result::getMetacard).collect(Collectors.toList()));
        });
    }
    printProgressAndFlush(start, totalWanted, ingestedCount.get());
    if (failedCount.get() > 0) {
        LOGGER.info("Not all records were ingested. [{}] failed", failedCount.get());
        if (StringUtils.isNotBlank(failedDir)) {
            try {
                writeFailedMetacards(failedMetacards);
            } catch (IOException e) {
                console.println("Error occurred while writing failed metacards to failedDir.");
            }
        }
    }
}

Example 2 with CatalogFacade

use of org.codice.ddf.commands.catalog.facade.CatalogFacade in project ddf by codice.

the class IngestCommand method executeWithSubject.

@Override
protected Object executeWithSubject() throws Exception {
    if (this.verifier == null) {
        this.verifier = new DigitalSignature(security);
    }
    if (batchSize * multithreaded > MAX_QUEUE_SIZE) {
        throw new IngestException(String.format("batchsize * multithreaded cannot be larger than %d.", MAX_QUEUE_SIZE));
    }
    final File inputFile = getInputFile();
    if (inputFile == null) {
        return null;
    }
    int totalFiles = totalFileCount(inputFile);
    fileCount.set(totalFiles);
    final ArrayBlockingQueue<Metacard> metacardQueue = new ArrayBlockingQueue<>(batchSize * multithreaded);
    ExecutorService queueExecutor = Executors.newSingleThreadExecutor(StandardThreadFactoryBuilder.newThreadFactory(THREAD_NAME));
    final long start = System.currentTimeMillis();
    printProgressAndFlush(start, fileCount.get(), 0);
    // Registering for the main thread and on behalf of the buildQueue thread
    // the buildQueue thread will unregister itself when the files have all
    // been added to the blocking queue and the final registration will
    // be held for the await.
    phaser.register();
    phaser.register();
    queueExecutor.submit(() -> buildQueue(inputFile, metacardQueue, start));
    final ScheduledExecutorService batchScheduler = Executors.newSingleThreadScheduledExecutor(StandardThreadFactoryBuilder.newThreadFactory(THREAD_NAME));
    BlockingQueue<Runnable> blockingQueue = new ArrayBlockingQueue<>(multithreaded);
    RejectedExecutionHandler rejectedExecutionHandler = new ThreadPoolExecutor.CallerRunsPolicy();
    ExecutorService executorService = new ThreadPoolExecutor(multithreaded, multithreaded, 0L, TimeUnit.MILLISECONDS, blockingQueue, StandardThreadFactoryBuilder.newThreadFactory(THREAD_NAME), rejectedExecutionHandler);
    final CatalogFacade catalog = getCatalog();
    submitToCatalog(batchScheduler, executorService, metacardQueue, catalog, start);
    // await on catalog processing threads to complete emptying queue
    phaser.awaitAdvance(phaser.arrive());
    try {
        queueExecutor.shutdown();
        executorService.shutdown();
        batchScheduler.shutdown();
    } catch (SecurityException e) {
        LOGGER.info("Executor service shutdown was not permitted", e);
    }
    printProgressAndFlush(start, fileCount.get(), (long) ingestCount.get() + ignoreCount.get());
    long end = System.currentTimeMillis();
    console.println();
    String elapsedTime = timeFormatter.print(new Period(start, end).withMillis(0));
    console.println();
    console.printf(" %d file(s) ingested in %s %n", ingestCount.get(), elapsedTime);
    LOGGER.debug("{} file(s) ingested in {} [{} records/sec]", ingestCount.get(), elapsedTime, calculateRecordsPerSecond(ingestCount.get(), start, end));
    INGEST_LOGGER.info("{} file(s) ingested in {} [{} records/sec]", ingestCount.get(), elapsedTime, calculateRecordsPerSecond(ingestCount.get(), start, end));
    if (fileCount.get() != ingestCount.get()) {
        if ((fileCount.get() - ingestCount.get() - ignoreCount.get()) >= 1) {
            String failedAmount = Integer.toString(fileCount.get() - ingestCount.get() - ignoreCount.get());
            console.println();
            printErrorMessage(failedAmount + " file(s) failed to be ingested. See the ingest log for more details.");
            INGEST_LOGGER.warn("{} file(s) failed to be ingested.", failedAmount);
        }
        if (ignoreList != null) {
            String ignoredAmount = Integer.toString(ignoreCount.get());
            console.println();
            printColor(Ansi.Color.YELLOW, ignoredAmount + " file(s) ignored. See the ingest log for more details.");
            INGEST_LOGGER.warn("{} file(s) were ignored.", ignoredAmount);
        }
    }
    console.println();
    securityLogger.audit("Ingested {} file(s) from {}", ingestCount.get(), filePath);
    return null;
}

Example 3 with CatalogFacade

use of org.codice.ddf.commands.catalog.facade.CatalogFacade in project ddf by codice.

the class RemoveAllCommand method executeRemoveAllFromStore.

private void executeRemoveAllFromStore() throws InterruptedException, SourceUnavailableException, FederationException, UnsupportedQueryException, IngestException {
    CatalogFacade catalog = getCatalog();
    QueryRequest firstQuery = getIntendedQuery(filterBuilder, true);
    QueryRequest subsequentQuery = getIntendedQuery(filterBuilder, false);
    long totalAmountDeleted = 0;
    long start = System.currentTimeMillis();
    SourceResponse response;
    try {
        response = catalog.query(firstQuery);
    } catch (UnsupportedQueryException e) {
        firstQuery = getAlternateQuery(filterBuilder, true);
        subsequentQuery = getAlternateQuery(filterBuilder, false);
        response = catalog.query(firstQuery);
    }
    if (response == null) {
        printErrorMessage("No response from Catalog.");
        return;
    }
    if (needsAlternateQueryAndResponse(response)) {
        firstQuery = getAlternateQuery(filterBuilder, true);
        subsequentQuery = getAlternateQuery(filterBuilder, false);
        response = catalog.query(firstQuery);
    }
    String totalAmount = getTotalAmount(response.getHits());
    while (response.getResults().size() > 0) {
        // Add metacard ids to string array
        List<String> ids = response.getResults().stream().filter(Objects::nonNull).map(Result::getMetacard).filter(Objects::nonNull).map(Metacard::getId).collect(Collectors.toList());
        // Delete the records
        DeleteRequestImpl request = new DeleteRequestImpl(ids.toArray(new String[ids.size()]));
        DeleteResponse deleteResponse = catalog.delete(request);
        int amountDeleted = deleteResponse.getDeletedMetacards().size();
        totalAmountDeleted += amountDeleted;
        console.print(String.format(PROGRESS_FORMAT, totalAmountDeleted, totalAmount));
        console.flush();
        // Prevents additional query to catalog framework
        if (response.getResults().isEmpty()) {
            break;
        }
        // Re-query when necessary
        response = catalog.query(subsequentQuery);
    }
    long end = System.currentTimeMillis();
    String info = String.format("%d file(s) removed in %3.3f seconds", totalAmountDeleted, (end - start) / MS_PER_SECOND);
    LOGGER.info(info);
    LOGGER.info("{} file(s) removed using the \"{}\" command", totalAmountDeleted, COMMAND);
    console.println();
    console.println(info);
}

Example 4 with CatalogFacade

use of org.codice.ddf.commands.catalog.facade.CatalogFacade in project ddf by codice.

the class ReplicateCommand method executeWithSubject.

@Override
protected Object executeWithSubject() throws Exception {
    if (batchSize > MAX_BATCH_SIZE || batchSize < 1) {
        console.println("Batch Size must be between 1 and " + MAX_BATCH_SIZE + ".");
        return null;
    }
    final CatalogFacade catalog = getCatalog();
    final CatalogFacade framework = new Framework(catalogFramework);
    Set<String> sourceIds = framework.getSourceIds();
    while (true) {
        if (StringUtils.isBlank(sourceId) || !sourceIds.contains(sourceId)) {
            console.println("Please enter the Source ID you would like to replicate:");
            for (String id : sourceIds) {
                console.println("\t" + id);
            }
        } else {
            break;
        }
        sourceId = session.readLine("ID:  ", null);
    }
    start = System.currentTimeMillis();
    console.println("Starting replication.");
    duplicateInBatches(framework, catalog, getFilter(), sourceId);
    console.println();
    long end = System.currentTimeMillis();
    String completed = String.format(" %d record(s) replicated; %d record(s) failed; completed in %3.3f seconds.", ingestedCount.get(), failedCount.get(), (end - start) / MS_PER_SECOND);
    LOGGER.debug("Replication Complete: {}", completed);
    console.println(completed);
    return null;
}

Example 5 with CatalogFacade

use of org.codice.ddf.commands.catalog.facade.CatalogFacade in project ddf by codice.

the class SearchCommand method executeSearchStore.

private Object executeSearchStore(Filter filter) {
    String formatString = "%1$-33s %2$-26s %3$-" + TITLE_MAX_LENGTH + "s %4$-" + EXCERPT_MAX_LENGTH + "s%n";
    CatalogFacade catalogProvider = getCatalog();
    QueryImpl query = new QueryImpl(filter);
    if (numberOfItems > -1) {
        query.setPageSize(numberOfItems);
    }
    long start = System.currentTimeMillis();
    List<Result> results = Lists.newArrayList(resultIterable(catalogProvider::query, new QueryRequestImpl(query), numberOfItems > 0 ? numberOfItems : 1000));
    long end = System.currentTimeMillis();
    final long hits = getHits(filter, catalogProvider);
    console.println();
    console.printf(" %d result(s) out of %s%s%s in %3.3f seconds", results.size(), Ansi.ansi().fg(Ansi.Color.CYAN).toString(), hits == -1 ? "?" : hits, Ansi.ansi().reset().toString(), (end - start) / MS_PER_SECOND);
    console.printf(formatString, "", "", "", "");
    printHeaderMessage(String.format(formatString, ID, DATE, TITLE, EXCERPT));
    for (Result result : results) {
        Metacard metacard = result.getMetacard();
        String title = (metacard.getTitle() != null ? metacard.getTitle() : "N/A");
        String excerpt = "N/A";
        String modifiedDate = "";
        if (searchPhrase != null && metacard.getMetadata() != null) {
            XPathHelper helper = new XPathHelper(metacard.getMetadata());
            String indexedText = helper.getDocument().getDocumentElement().getTextContent();
            indexedText = indexedText.replaceAll("\\r\\n|\\r|\\n", " ");
            String normalizedSearchPhrase = searchPhrase.replaceAll("\\*", "");
            int index = -1;
            if (caseSensitive) {
                index = indexedText.indexOf(normalizedSearchPhrase);
            } else {
                index = indexedText.toLowerCase().indexOf(normalizedSearchPhrase.toLowerCase());
            }
            if (index != -1) {
                int contextLength = (EXCERPT_MAX_LENGTH - normalizedSearchPhrase.length() - 8) / 2;
                excerpt = "..." + indexedText.substring(Math.max(index - contextLength, 0), index);
                excerpt = excerpt + Ansi.ansi().fg(Ansi.Color.GREEN).toString();
                excerpt = excerpt + indexedText.substring(index, index + normalizedSearchPhrase.length());
                excerpt = excerpt + Ansi.ansi().reset().toString();
                excerpt = excerpt + indexedText.substring(index + normalizedSearchPhrase.length(), Math.min(indexedText.length(), index + normalizedSearchPhrase.length() + contextLength)) + "...";
            }
        }
        if (metacard.getModifiedDate() != null) {
            modifiedDate = new DateTime(metacard.getModifiedDate().getTime()).toString(DATETIME_FORMATTER);
        }
        console.printf(formatString, metacard.getId(), modifiedDate, title.substring(0, Math.min(title.length(), TITLE_MAX_LENGTH)), excerpt);
    }
    return null;
}