Examples with Metacard ddf.catalog.data.Metacard used on opensource projects

Example 26 with Metacard

use of ddf.catalog.data.Metacard in project ddf by codice.

the class CatalogComponentFrameworkTest method testCreateWithInvalidOperation.

@Test
public /**
     * Operation: CREATE
     * Body contains:  Metacard
     */
void testCreateWithInvalidOperation() throws Exception {
    resetMocks();
    // Setup expectations to verify
    final MockEndpoint mockVerifierEndpoint = getMockEndpoint("mock:result");
    mockVerifierEndpoint.expectedMessageCount(1);
    final List<Metacard> metacards = new ArrayList<Metacard>();
    metacards.add(metacard1);
    // Mock catalog framework
    final CreateRequest createRequest = new CreateRequestImpl(metacards);
    final CreateResponse createResponse = new CreateResponseImpl(createRequest, new HashMap(), metacards);
    when(catalogFramework.create(any(CreateRequest.class))).thenReturn(createResponse);
    // Exercise the route with a CREATE operation
    template.sendBodyAndHeader("direct:sampleInput", metacard1, "Operation", "WRONG OPERATION");
    // Verify that the number of metacards in the exchange after the records
    // is identical to the input
    assertListSize(mockVerifierEndpoint.getExchanges(), 1);
    final Exchange exchange = mockVerifierEndpoint.getExchanges().get(0);
    final List<Metacard> cardsCreated = (List<Metacard>) exchange.getIn().getBody();
    assertListSize(cardsCreated, 0);
    mockVerifierEndpoint.assertIsSatisfied();
}

Example 27 with Metacard

use of ddf.catalog.data.Metacard in project ddf by codice.

the class CatalogComponentFrameworkTest method testUpdateWithListOfMetacards.

@Test
public /**
     * Operation: UPDATE
     * Body contains:  List<Metacard>
     */
void testUpdateWithListOfMetacards() throws Exception {
    resetMocks();
    // Setup expectations to verify
    final MockEndpoint mockVerifierEndpoint = getMockEndpoint("mock:result");
    mockVerifierEndpoint.expectedMessageCount(1);
    final List<Metacard> metacards = new ArrayList<Metacard>();
    metacards.add(metacard1);
    // setup mock catalog framework
    final Update update = new UpdateImpl(metacard1, metacard2);
    List<Update> updates = new ArrayList<Update>();
    updates.add(update);
    final String[] metacardIds = new String[metacards.size()];
    for (int i = 0; i < metacards.size(); i++) {
        metacardIds[i] = metacards.get(i).getId();
    }
    UpdateRequest updateRequest = new UpdateRequestImpl(metacardIds, metacards);
    UpdateResponse updateResponse = new UpdateResponseImpl(updateRequest, new HashMap(), updates);
    when(catalogFramework.update(any(UpdateRequest.class))).thenReturn(updateResponse);
    // Exercise the route with a UPDATE operation
    template.sendBodyAndHeader("direct:sampleInput", metacards, "Operation", "UPDATE");
    // Verify that the number of metacards in the exchange after the records
    // is identical to the input
    assertListSize(mockVerifierEndpoint.getExchanges(), 1);
    final Exchange exchange = mockVerifierEndpoint.getExchanges().get(0);
    final List<Update> cardsUpdated = (List<Update>) exchange.getIn().getBody();
    assertListSize(cardsUpdated, 1);
    mockVerifierEndpoint.assertIsSatisfied();
}

Example 28 with Metacard

use of ddf.catalog.data.Metacard in project ddf by codice.

the class FrameworkProducer method delete.

/**
     * Deletes metacard(s) in the catalog using the Catalog Framework.
     *
     * @param exchange
     *            The {@link org.apache.camel.Exchange} can contain a
     *            {@link org.apache.camel.Message} with a body of type {@link java.util.List} of
     *            {@link String} or a single {@link String}. Each String represents the ID of a
     *            Metacard to be deleted.
     * @throws ddf.catalog.source.SourceUnavailableException
     * @throws ddf.catalog.source.IngestException
     * @throws ddf.camel.component.catalog.framework.FrameworkProducerException
     */
private void delete(final Exchange exchange) throws SourceUnavailableException, IngestException, FrameworkProducerException {
    DeleteResponse deleteResponse = null;
    // read in data
    final List<String> metacardIdsToBeDeleted = readBodyDataAsMetacardIds(exchange);
    // process if data is valid
    if (!validateList(metacardIdsToBeDeleted, String.class)) {
        LOGGER.debug("Validation of Metacard id list failed");
        processCatalogResponse(deleteResponse, exchange);
        throw new FrameworkProducerException("Validation of Metacard id list failed");
    }
    LOGGER.debug("Validation of Metacard id list passed...");
    final String[] metacardIdsToBeDeletedArray = new String[metacardIdsToBeDeleted.size()];
    final DeleteRequest deleteRequest = new DeleteRequestImpl(metacardIdsToBeDeleted.toArray(metacardIdsToBeDeletedArray));
    final int expectedNumberOfDeletedMetacards = metacardIdsToBeDeleted.size();
    if (expectedNumberOfDeletedMetacards < 1) {
        LOGGER.debug("Empty list of Metacard id...nothing to process");
        processCatalogResponse(deleteResponse, exchange);
        return;
    }
    LOGGER.debug("Making DELETE call to Catalog Framework...");
    deleteResponse = catalogFramework.delete(deleteRequest);
    if (deleteResponse == null) {
        LOGGER.debug("DeleteResponse is null from catalog framework");
        processCatalogResponse(deleteResponse, exchange);
        return;
    }
    final List<Metacard> deletedMetacards = deleteResponse.getDeletedMetacards();
    if (deletedMetacards == null) {
        LOGGER.debug("DeleteResponse returned null metacards list");
        processCatalogResponse(deleteResponse, exchange);
        return;
    }
    final int numberOfDeletedMetacards = deletedMetacards.size();
    if (numberOfDeletedMetacards != expectedNumberOfDeletedMetacards) {
        LOGGER.debug("Expected {} metacards deleted but only {} were successfully deleted", expectedNumberOfDeletedMetacards, numberOfDeletedMetacards);
        processCatalogResponse(deleteResponse, exchange);
        return;
    }
    LOGGER.debug("Deleted {} metacards", numberOfDeletedMetacards);
    processCatalogResponse(deleteResponse, exchange);
}

Example 29 with Metacard

use of ddf.catalog.data.Metacard in project ddf by codice.

the class IngestCommand method executeWithSubject.

@Override
protected Object executeWithSubject() throws Exception {
    if (batchSize * multithreaded > MAX_QUEUE_SIZE) {
        throw new IngestException(String.format("batchsize * multithreaded cannot be larger than %d.", MAX_QUEUE_SIZE));
    }
    final File inputFile = getInputFile();
    if (inputFile == null) {
        return null;
    }
    int totalFiles = totalFileCount(inputFile);
    fileCount.set(totalFiles);
    final ArrayBlockingQueue<Metacard> metacardQueue = new ArrayBlockingQueue<>(batchSize * multithreaded);
    ExecutorService queueExecutor = Executors.newSingleThreadExecutor();
    final long start = System.currentTimeMillis();
    printProgressAndFlush(start, fileCount.get(), 0);
    // Registering for the main thread and on behalf of the buildQueue thread;
    // the buildQueue thread will unregister itself when the files have all
    // been added to the blocking queue and the final registration will
    // be held for the await.
    phaser.register();
    phaser.register();
    queueExecutor.submit(() -> buildQueue(inputFile, metacardQueue, start));
    final ScheduledExecutorService batchScheduler = Executors.newSingleThreadScheduledExecutor();
    BlockingQueue<Runnable> blockingQueue = new ArrayBlockingQueue<>(multithreaded);
    RejectedExecutionHandler rejectedExecutionHandler = new ThreadPoolExecutor.CallerRunsPolicy();
    ExecutorService executorService = new ThreadPoolExecutor(multithreaded, multithreaded, 0L, TimeUnit.MILLISECONDS, blockingQueue, rejectedExecutionHandler);
    final CatalogFacade catalog = getCatalog();
    submitToCatalog(batchScheduler, executorService, metacardQueue, catalog, start);
    // await on catalog processing threads to complete emptying queue
    phaser.awaitAdvance(phaser.arrive());
    try {
        queueExecutor.shutdown();
        executorService.shutdown();
        batchScheduler.shutdown();
    } catch (SecurityException e) {
        LOGGER.info("Executor service shutdown was not permitted: {}", e);
    }
    printProgressAndFlush(start, fileCount.get(), ingestCount.get() + ignoreCount.get());
    long end = System.currentTimeMillis();
    console.println();
    String elapsedTime = timeFormatter.print(new Period(start, end).withMillis(0));
    console.println();
    console.printf(" %d file(s) ingested in %s %n", ingestCount.get(), elapsedTime);
    LOGGER.debug("{} file(s) ingested in {} [{} records/sec]", ingestCount.get(), elapsedTime, calculateRecordsPerSecond(ingestCount.get(), start, end));
    INGEST_LOGGER.info("{} file(s) ingested in {} [{} records/sec]", ingestCount.get(), elapsedTime, calculateRecordsPerSecond(ingestCount.get(), start, end));
    if (fileCount.get() != ingestCount.get()) {
        console.println();
        if ((fileCount.get() - ingestCount.get() - ignoreCount.get()) >= 1) {
            String failedAmount = Integer.toString(fileCount.get() - ingestCount.get() - ignoreCount.get());
            printErrorMessage(failedAmount + " file(s) failed to be ingested.  See the ingest log for more details.");
            INGEST_LOGGER.warn("{} files(s) failed to be ingested.", failedAmount);
        }
        if (ignoreList != null) {
            String ignoredAmount = Integer.toString(ignoreCount.get());
            printColor(Ansi.Color.YELLOW, ignoredAmount + " file(s) ignored.  See the ingest log for more details.");
            INGEST_LOGGER.warn("{} files(s) were ignored.", ignoredAmount);
        }
    }
    console.println();
    SecurityLogger.audit("Ingested {} files from {}", ingestCount.get(), filePath);
    return null;
}

Example 30 with Metacard

use of ddf.catalog.data.Metacard in project ddf by codice.

the class IngestCommand method buildIngestLog.

/**
     * Helper method to build ingest log strings
     */
private String buildIngestLog(ArrayList<Metacard> metacards) {
    StringBuilder strBuilder = new StringBuilder();
    for (int i = 0; i < metacards.size(); i++) {
        Metacard card = metacards.get(i);
        strBuilder.append(NEW_LINE).append("Batch #: ").append(i + 1).append(" | ");
        if (card != null) {
            if (card.getTitle() != null) {
                strBuilder.append("Metacard Title: ").append(card.getTitle()).append(" | ");
            }
            if (card.getId() != null) {
                strBuilder.append("Metacard ID: ").append(card.getId()).append(" | ");
            }
        } else {
            strBuilder.append("Null Metacard");
        }
    }
    return strBuilder.toString();
}