Examples with ResultIterable ddf.catalog.util.impl.ResultIterable used on opensource projects

Example 1 with ResultIterable

use of ddf.catalog.util.impl.ResultIterable in project ddf by codice.

the class DuplicateCommands method duplicateInBatches.

/**
 * In batches, loops through a query of the queryFacade and an ingest to the ingestFacade of the
 * metacards from the response until there are no more metacards from the queryFacade or the
 * maxMetacards has been reached.
 *
 * @param queryFacade - the CatalogFacade to duplicate from
 * @param ingestFacade - the CatalogFacade to duplicate to
 * @param filter - the filter to query with
 */
protected void duplicateInBatches(CatalogFacade queryFacade, CatalogFacade ingestFacade, Filter filter, String sourceId) throws InterruptedException {
    AtomicInteger queryIndex = new AtomicInteger(1);
    final long originalQuerySize;
    if (maxMetacards > 0 && maxMetacards < batchSize) {
        originalQuerySize = maxMetacards;
    } else {
        originalQuerySize = batchSize;
    }
    Function<Integer, QueryRequest> queryTemplate = (index) -> new QueryRequestImpl(new QueryImpl(filter, index, (int) originalQuerySize, new SortByImpl(Metacard.EFFECTIVE, SortOrder.DESCENDING), true, TimeUnit.MINUTES.toMillis(5)), Collections.singletonList(sourceId));
    List<Metacard> initialMetacards = ResultIterable.resultIterable((queryRequest -> {
        SourceResponse response = queryFacade.query(queryRequest);
        if (response.getHits() != -1) {
            maxMetacards = (int) response.getHits();
        }
        return response;
    }), queryTemplate.apply(queryIndex.get()), (int) originalQuerySize).stream().map(Result::getMetacard).collect(Collectors.toList());
    if (initialMetacards.isEmpty()) {
        LOGGER.debug("Query returned 0 results.");
        console.println(String.format("No results were returned by the source [%s]", sourceId));
        return;
    }
    ingestMetacards(ingestFacade, initialMetacards);
    if (initialMetacards.size() < originalQuerySize) {
        // all done if results exhausted in the first batch
        printProgressAndFlush(start, maxMetacards < 1 ? initialMetacards.size() : maxMetacards, ingestedCount.get());
        return;
    }
    final long totalWanted = maxMetacards;
    final AtomicBoolean done = new AtomicBoolean(false);
    if (multithreaded > 1) {
        BlockingQueue<Runnable> blockingQueue = new ArrayBlockingQueue<>(multithreaded);
        RejectedExecutionHandler rejectedExecutionHandler = new ThreadPoolExecutor.CallerRunsPolicy();
        final ExecutorService executorService = new ThreadPoolExecutor(multithreaded, multithreaded, 0L, TimeUnit.MILLISECONDS, blockingQueue, StandardThreadFactoryBuilder.newThreadFactory("duplicateCommandsThread"), rejectedExecutionHandler);
        console.printf("Running a maximum of %d threads during replication.%n", multithreaded);
        printProgressAndFlush(start, Math.max(totalWanted, initialMetacards.size()), ingestedCount.get());
        int index;
        while (!done.get()) {
            index = queryIndex.addAndGet(batchSize);
            final int taskIndex = index;
            executorService.submit(() -> {
                int querySize = (int) getQuerySizeFromIndex(totalWanted, taskIndex);
                if (querySize < 1) {
                    // If we don't need any more metacards, we're finished
                    done.set(true);
                    return;
                }
                List<Metacard> metacards = ResultIterable.resultIterable(queryFacade::query, queryTemplate.apply(taskIndex), querySize).stream().map(Result::getMetacard).collect(Collectors.toList());
                if (metacards.size() < querySize) {
                    done.set(true);
                }
                if (!metacards.isEmpty()) {
                    ingestMetacards(ingestFacade, metacards);
                }
                printProgressAndFlush(start, Math.max(totalWanted, ingestedCount.get()), ingestedCount.get());
            });
        }
        executorService.shutdown();
        try {
            executorService.awaitTermination(1, TimeUnit.SECONDS);
        } catch (InterruptedException e) {
            executorService.shutdownNow();
            throw e;
        }
        printProgressAndFlush(start, Math.max(totalWanted, ingestedCount.get()), ingestedCount.get());
    } else {
        // Single threaded
        ResultIterable iter;
        if (maxMetacards > 0) {
            iter = ResultIterable.resultIterable(queryFacade::query, queryTemplate.apply(1 + batchSize), maxMetacards);
        } else {
            iter = ResultIterable.resultIterable(queryFacade::query, queryTemplate.apply(1 + batchSize));
        }
        Iterables.partition(iter, batchSize).forEach((batch) -> {
            printProgressAndFlush(start, totalWanted, ingestedCount.get());
            if (batch.isEmpty()) {
                return;
            }
            ingestMetacards(ingestFacade, batch.stream().map(Result::getMetacard).collect(Collectors.toList()));
        });
    }
    printProgressAndFlush(start, totalWanted, ingestedCount.get());
    if (failedCount.get() > 0) {
        LOGGER.info("Not all records were ingested. [{}] failed", failedCount.get());
        if (StringUtils.isNotBlank(failedDir)) {
            try {
                writeFailedMetacards(failedMetacards);
            } catch (IOException e) {
                console.println("Error occurred while writing failed metacards to failedDir.");
            }
        }
    }
}

Example 2 with ResultIterable

use of ddf.catalog.util.impl.ResultIterable in project ddf by codice.

the class CswEndpoint method queryCsw.

private CswRecordCollection queryCsw(GetRecordsType request) throws CswException {
    if (LOGGER.isDebugEnabled()) {
        try {
            Writer writer = new StringWriter();
            try {
                Marshaller marshaller = CswQueryFactory.getJaxBContext().createMarshaller();
                marshaller.setProperty(Marshaller.JAXB_FORMATTED_OUTPUT, Boolean.TRUE);
                JAXBElement<GetRecordsType> jaxbElement = new ObjectFactory().createGetRecords(request);
                marshaller.marshal(jaxbElement, writer);
            } catch (JAXBException e) {
                LOGGER.debug("Unable to marshall {} to XML", GetRecordsType.class, e);
            }
            LOGGER.debug(writer.toString());
        } catch (Exception e) {
            LOGGER.debug("Unable to create debug message for getRecordsType", e);
        }
    }
    QueryType query = (QueryType) request.getAbstractQuery().getValue();
    CswRecordCollection response = new CswRecordCollection();
    response.setRequest(request);
    response.setOutputSchema(request.getOutputSchema());
    response.setMimeType(request.getOutputFormat());
    response.setElementName(query.getElementName());
    response.setElementSetType((query.getElementSetName() != null) ? query.getElementSetName().getValue() : null);
    response.setResultType((ResultType) ObjectUtils.defaultIfNull(request.getResultType(), ResultType.HITS));
    if (ResultType.HITS.equals(request.getResultType()) || ResultType.RESULTS.equals(request.getResultType())) {
        QueryRequest queryRequest = queryFactory.getQuery(request);
        try {
            queryRequest = queryFactory.updateQueryRequestTags(queryRequest, request.getOutputSchema());
            LOGGER.debug("Attempting to execute paged query: {}", queryRequest);
            AtomicLong hitCount = new AtomicLong(0);
            QueryFunction qf = qr -> {
                SourceResponse sr = framework.query(qr);
                hitCount.compareAndSet(0, sr.getHits());
                return sr;
            };
            ResultIterable results = ResultIterable.resultIterable(qf, queryRequest, request.getMaxRecords().intValue());
            List<Result> resultList = results.stream().collect(Collectors.toList());
            // The hitCount Atomic is used here instead of just defaulting
            // to the size of the resultList because the size of the resultList
            // can be limited to request.getMaxRecords().intValue() which would
            // lead to an incorrect response for hits.
            // hitCount is set within the QueryFunction and will correspond to
            // all responses.
            long totalHits = hitCount.get();
            totalHits = totalHits != 0 ? totalHits : resultList.size();
            QueryResponse queryResponse = new QueryResponseImpl(queryRequest, resultList, totalHits);
            response.setSourceResponse(queryResponse);
        } catch (UnsupportedQueryException | CatalogQueryException e) {
            LOGGER.debug("Unable to query", e);
            throw new CswException(e);
        }
    }
    return response;
}