Examples with CreateResponse ddf.catalog.operation.CreateResponse used on opensource projects

Example 36 with CreateResponse

use of ddf.catalog.operation.CreateResponse in project ddf by codice.

the class Historian method version.

/**
 * Versions deleted {@link Metacard}s.
 *
 * @param deleteResponse Versions this responses deleted metacards
 */
public DeleteResponse version(DeleteResponse deleteResponse) throws SourceUnavailableException, IngestException {
    if (doSkip(deleteResponse)) {
        return deleteResponse;
    }
    setSkipFlag(deleteResponse);
    if (LOGGER.isTraceEnabled()) {
        LOGGER.trace("Versioning Deleted Metacards {}", deleteResponse);
    }
    List<Metacard> originalMetacards = deleteResponse.getDeletedMetacards().stream().filter(isNotVersionNorDeleted).collect(Collectors.toList());
    if (originalMetacards.isEmpty()) {
        LOGGER.trace("No deleted metacards applicable to versioning");
        securityLogger.audit("Skipping versioning deleted metacards with ids: {}", deleteResponse.getDeletedMetacards().stream().map(Metacard::getId).collect(TO_A_STRING));
        return deleteResponse;
    }
    if (LOGGER.isTraceEnabled()) {
        LOGGER.trace("Versioning following deleted metacards: {}", originalMetacards.stream().map(Metacard::getId).collect(TO_A_STRING));
    }
    // [OriginalMetacardId: Original Metacard]
    Map<String, Metacard> originalMetacardsMap = originalMetacards.stream().collect(Collectors.toMap(Metacard::getId, Function.identity(), Historian::firstInWinsMerge));
    // [ContentItem.getId: content items]
    Map<String, List<ContentItem>> contentItems = getContent(getReadStorageRequests(originalMetacards));
    if (LOGGER.isTraceEnabled()) {
        LOGGER.trace("Got content under the following ID's: {}", contentItems.keySet().stream().collect(TO_A_STRING));
    }
    Function<String, Action> getAction = id -> contentItems.containsKey(id) ? Action.DELETED_CONTENT : Action.DELETED;
    // VERSION_OF_ID is equivalent to the Original Metacard ID
    // [MetacardVersion.VERSION_OF_ID: versioned metacard]
    Map<String, Metacard> versionedMap = getVersionMetacards(originalMetacards, getAction, (Subject) deleteResponse.getRequest().getProperties().get(SecurityConstants.SECURITY_SUBJECT));
    if (LOGGER.isDebugEnabled() && !versionedMap.keySet().equals(originalMetacardsMap.keySet())) {
        LOGGER.debug("There is not a one to one mapping between original metacards and their versions!" + " (Some metacards may not have been versioned or too many versions may have been created). " + "More information regarding the IDs is available by setting log level to trace " + "(log:set trace ddf.catalog.history)");
        if (LOGGER.isTraceEnabled()) {
            LOGGER.trace("Original Metacards: {}", originalMetacards.stream().map(Metacard::getId).collect(TO_A_STRING));
            LOGGER.trace("Version Metacards: {}", versionedMap.keySet().stream().collect(TO_A_STRING));
        }
    }
    CreateStorageResponse createStorageResponse = versionContentItems(contentItems, versionedMap);
    if (createStorageResponse != null) {
        setResourceUriForContent(/*Mutable*/
        versionedMap, createStorageResponse);
        if (LOGGER.isTraceEnabled()) {
            LOGGER.trace("Successfully stored content under ids: {}", createStorageResponse.getCreatedContentItems());
        }
    }
    executeAsSystem(() -> catalogProvider().create(new CreateRequestImpl(new ArrayList<>(versionedMap.values()))));
    if (LOGGER.isTraceEnabled()) {
        LOGGER.trace("Successfully created versioned metacards under ids: {}", versionedMap.values().stream().map(Metacard::getId).collect(TO_A_STRING));
    }
    String userid = subjectIdentity.getUniqueIdentifier((Subject) deleteResponse.getProperties().get(SecurityConstants.SECURITY_SUBJECT));
    List<Metacard> deletionMetacards = versionedMap.entrySet().stream().map(s -> new DeletedMetacardImpl(uuidGenerator.generateUuid(), s.getKey(), userid, s.getValue().getId(), originalMetacardsMap.get(s.getKey()))).collect(Collectors.toList());
    CreateResponse deletionMetacardsCreateResponse = executeAsSystem(() -> catalogProvider().create(new CreateRequestImpl(deletionMetacards, new HashMap<>())));
    if (LOGGER.isTraceEnabled()) {
        LOGGER.trace("Successfully created deletion metacards under ids: {}", deletionMetacardsCreateResponse.getCreatedMetacards().stream().map(Metacard::getId).collect(TO_A_STRING));
    }
    return deleteResponse;
}

Example 37 with CreateResponse

use of ddf.catalog.operation.CreateResponse in project ddf by codice.

the class CreateOperations method performRemoteCreate.

private CreateResponse performRemoteCreate(CreateRequest createRequest, CreateResponse createResponse) {
    if (!opsCatStoreSupport.isCatalogStoreRequest(createRequest)) {
        return createResponse;
    }
    CreateResponse remoteCreateResponse = doRemoteCreate(createRequest);
    if (createResponse == null) {
        createResponse = remoteCreateResponse;
    } else {
        createResponse.getProperties().putAll(remoteCreateResponse.getProperties());
        createResponse.getProcessingErrors().addAll(remoteCreateResponse.getProcessingErrors());
    }
    return createResponse;
}

Example 38 with CreateResponse

use of ddf.catalog.operation.CreateResponse in project ddf by codice.

the class CreateOperations method doRemoteCreate.

private CreateResponse doRemoteCreate(CreateRequest createRequest) {
    HashSet<ProcessingDetails> exceptions = new HashSet<>();
    Map<String, Serializable> properties = new HashMap<>();
    List<CatalogStore> stores = opsCatStoreSupport.getCatalogStoresForRequest(createRequest, exceptions);
    for (CatalogStore store : stores) {
        try {
            if (!store.isAvailable()) {
                exceptions.add(new ProcessingDetailsImpl(store.getId(), null, "CatalogStore is not available"));
            } else {
                CreateResponse response = store.create(createRequest);
                properties.put(store.getId(), new ArrayList<>(response.getCreatedMetacards()));
            }
        } catch (IngestException e) {
            INGEST_LOGGER.error("Error creating metacards for CatalogStore {}", store.getId(), e);
            exceptions.add(new ProcessingDetailsImpl(store.getId(), e));
        }
    }
    return new CreateResponseImpl(createRequest, properties, createRequest.getMetacards(), exceptions);
}

Example 39 with CreateResponse

use of ddf.catalog.operation.CreateResponse in project ddf by codice.

the class CreateOperations method doCreate.

// 
// Private helper methods
// 
private CreateResponse doCreate(CreateRequest createRequest) throws IngestException, SourceUnavailableException {
    CreateResponse createResponse;
    Exception ingestError = null;
    createRequest = queryOperations.setFlagsOnRequest(createRequest);
    createRequest = validateCreateRequest(createRequest);
    createRequest = validateLocalSource(createRequest);
    String fileNames = "";
    if (createRequest != null) {
        List<String> fileList = createRequest.getMetacards().stream().filter(Objects::nonNull).map(Metacard::getTitle).collect(Collectors.toList());
        fileNames = String.join(", ", fileList);
    }
    try {
        INGEST_LOGGER.info("Started ingesting metacard with titles: {}.", fileNames);
        createRequest = injectAttributes(createRequest);
        createRequest = setDefaultValues(createRequest);
        createRequest = processPreAuthorizationPlugins(createRequest);
        createRequest = updateCreateRequestPolicyMap(createRequest);
        createRequest = processPrecreateAccessPlugins(createRequest);
        createRequest.getProperties().put(Constants.OPERATION_TRANSACTION_KEY, new OperationTransactionImpl(OperationTransaction.OperationType.CREATE, Collections.emptyList()));
        createRequest = processPreIngestPlugins(createRequest);
        createRequest = validateCreateRequest(createRequest);
        createResponse = getCreateResponse(createRequest);
        createResponse = performRemoteCreate(createRequest, createResponse);
    } catch (IngestException iee) {
        ingestError = iee;
        throw iee;
    } catch (StopProcessingException see) {
        ingestError = see;
        throw new IngestException(PRE_INGEST_ERROR, see);
    } catch (RuntimeException re) {
        ingestError = re;
        throw new InternalIngestException("Exception during runtime while performing create", re);
    } finally {
        if (createRequest != null && ingestError != null && INGEST_LOGGER.isInfoEnabled()) {
            INGEST_LOGGER.info("Error on create operation. {} metacards failed to ingest. {}", createRequest.getMetacards().size(), buildIngestLog(createRequest), ingestError);
        }
    }
    try {
        createResponse = validateFixCreateResponse(createResponse, createRequest);
    } catch (RuntimeException re) {
        LOGGER.info("Exception during runtime while performing doing post create operations (plugins and pubsub)", re);
    }
    if (createResponse == null) {
        // suppress all npe findings for this method.
        throw new IngestException("CatalogProvider returned null CreateResponse Object.");
    }
    return createResponse;
}

Example 40 with CreateResponse

use of ddf.catalog.operation.CreateResponse in project ddf by codice.

the class CreateOperations method create.

public CreateResponse create(CreateStorageRequest streamCreateRequest, List<String> fanoutTagBlacklist) throws IngestException, SourceUnavailableException {
    Map<String, Metacard> metacardMap = new HashMap<>();
    List<ContentItem> contentItems = new ArrayList<>(streamCreateRequest.getContentItems().size());
    HashMap<String, Map<String, Path>> tmpContentPaths = new HashMap<>();
    CreateResponse createResponse;
    CreateStorageRequest createStorageRequest = null;
    CreateStorageResponse createStorageResponse;
    CreateRequest createRequest = null;
    Exception ingestError = null;
    String fileNames = "";
    streamCreateRequest = opsStorageSupport.prepareStorageRequest(streamCreateRequest, streamCreateRequest::getContentItems);
    if (!streamCreateRequest.getContentItems().isEmpty()) {
        List<String> fileList = streamCreateRequest.getContentItems().stream().map(ContentItem::getFilename).collect(Collectors.toList());
        fileNames = String.join(", ", fileList);
    }
    INGEST_LOGGER.info("Started ingesting resources with titles: {}.", fileNames);
    // Operation populates the metacardMap, contentItems, and tmpContentPaths
    opsMetacardSupport.generateMetacardAndContentItems(streamCreateRequest.getContentItems(), metacardMap, contentItems, tmpContentPaths);
    if (blockCreateMetacards(metacardMap.values(), fanoutTagBlacklist)) {
        String message = "Fanout proxy does not support create operations with blacklisted metacard tag";
        LOGGER.debug("{}. Tags blacklist: {}", message, fanoutTagBlacklist);
        throw new IngestException(message);
    }
    streamCreateRequest.getProperties().put(CONTENT_PATHS, tmpContentPaths);
    injectAttributes(metacardMap);
    setDefaultValues(metacardMap);
    streamCreateRequest = applyAttributeOverrides(streamCreateRequest, metacardMap);
    try {
        if (!contentItems.isEmpty()) {
            createStorageRequest = new CreateStorageRequestImpl(contentItems, streamCreateRequest.getId(), streamCreateRequest.getProperties());
            createStorageRequest = processPreCreateStoragePlugins(createStorageRequest);
            try {
                createStorageResponse = sourceOperations.getStorage().create(createStorageRequest);
                createStorageResponse.getProperties().put(CONTENT_PATHS, tmpContentPaths);
            } catch (StorageException e) {
                INGEST_LOGGER.debug("Could not store content items: {}.", fileNames, e);
                throw new IngestException("Could not store content items.", e);
            }
            createStorageResponse = processPostCreateStoragePlugins(createStorageResponse);
            populateMetacardMap(metacardMap, createStorageResponse);
        }
        createRequest = new CreateRequestImpl(new ArrayList<>(metacardMap.values()), Optional.ofNullable(createStorageRequest).map(StorageRequest::getProperties).orElseGet(HashMap::new));
        createResponse = doCreate(createRequest);
    } catch (IngestException e) {
        ingestError = e;
        rollbackStorage(createStorageRequest);
        throw e;
    } catch (IOException | RuntimeException e) {
        LOGGER.debug("Unhandled runtime exception or IOException during create", e);
        ingestError = e;
        rollbackStorage(createStorageRequest);
        throw new IngestException("Unable to store products for request: " + fileNames, e);
    } finally {
        opsStorageSupport.commitAndCleanup(createStorageRequest, tmpContentPaths);
        if (INGEST_LOGGER.isInfoEnabled()) {
            if (createRequest != null && ingestError != null) {
                INGEST_LOGGER.info("Error during ingesting resource: {}", fileNames, ingestError);
            }
            INGEST_LOGGER.info("Completed ingesting resource: {}.", fileNames);
        }
    }
    createResponse = doPostIngest(createResponse);
    return createResponse;
}