Search in sources :

Example 26 with Objects

use of java.util.Objects in project ddf by codice.

the class RemoveAllCommand method executeRemoveAllFromStore.

private Object executeRemoveAllFromStore() throws Exception {
    CatalogFacade catalog = getCatalog();
    QueryRequest firstQuery = getIntendedQuery(filterBuilder, true);
    QueryRequest subsequentQuery = getIntendedQuery(filterBuilder, false);
    long totalAmountDeleted = 0;
    long start = System.currentTimeMillis();
    SourceResponse response;
    try {
        response = catalog.query(firstQuery);
    } catch (UnsupportedQueryException e) {
        firstQuery = getAlternateQuery(filterBuilder, true);
        subsequentQuery = getAlternateQuery(filterBuilder, false);
        response = catalog.query(firstQuery);
    }
    if (response == null) {
        printErrorMessage("No response from Catalog.");
        return null;
    }
    if (needsAlternateQueryAndResponse(response)) {
        firstQuery = getAlternateQuery(filterBuilder, true);
        subsequentQuery = getAlternateQuery(filterBuilder, false);
        response = catalog.query(firstQuery);
    }
    String totalAmount = getTotalAmount(response.getHits());
    while (response.getResults().size() > 0) {
        // Add metacard ids to string array
        List<String> ids = response.getResults().stream().filter(Objects::nonNull).map(Result::getMetacard).filter(Objects::nonNull).map(Metacard::getId).collect(Collectors.toList());
        // Delete the records
        DeleteRequestImpl request = new DeleteRequestImpl(ids.toArray(new String[ids.size()]));
        DeleteResponse deleteResponse = catalog.delete(request);
        int amountDeleted = deleteResponse.getDeletedMetacards().size();
        totalAmountDeleted += amountDeleted;
        console.print(String.format(PROGRESS_FORMAT, totalAmountDeleted, totalAmount));
        console.flush();
        // Break out if there are no more records to delete
        if (amountDeleted < batchSize || batchSize < 1) {
            break;
        }
        // Re-query when necessary
        response = catalog.query(subsequentQuery);
    }
    long end = System.currentTimeMillis();
    String info = String.format(" %d file(s) removed in %3.3f seconds%n", totalAmountDeleted, (end - start) / MS_PER_SECOND);
    LOGGER.info(info);
    LOGGER.info(totalAmountDeleted + " files removed using cache:removeAll command");
    console.println();
    console.print(info);
    return null;
}
Also used : DeleteResponse(ddf.catalog.operation.DeleteResponse) QueryRequest(ddf.catalog.operation.QueryRequest) SourceResponse(ddf.catalog.operation.SourceResponse) UnsupportedQueryException(ddf.catalog.source.UnsupportedQueryException) DeleteRequestImpl(ddf.catalog.operation.impl.DeleteRequestImpl) Objects(java.util.Objects) CatalogFacade(org.codice.ddf.commands.catalog.facade.CatalogFacade)

Example 27 with Objects

use of java.util.Objects in project ddf by codice.

the class SolrCatalogProvider method queryPendingNrtIndex.

private void queryPendingNrtIndex(QueryRequest request, SourceResponse response) throws UnsupportedQueryException {
    if (request == null || request.getQuery() == null) {
        return;
    }
    Set<String> ids = filterAdapter.adapt(request.getQuery(), new MetacardIdEqualityFilterDelegate());
    if (ids.size() == 0) {
        return;
    }
    for (Result result : response.getResults()) {
        ids.remove(result.getMetacard().getId());
    }
    List<Result> pendingResults = pendingNrtIndex.getAllPresent(ids).values().stream().filter(Objects::nonNull).map(ResultImpl::new).collect(Collectors.toList());
    response.getResults().addAll(pendingResults);
}
Also used : Objects(java.util.Objects) Result(ddf.catalog.data.Result)

Example 28 with Objects

use of java.util.Objects in project ddf by codice.

the class RegistryStoreImpl method update.

@Override
public UpdateResponse update(UpdateRequest request) throws IngestException {
    if (request.getUpdates().stream().map(e -> RegistryUtility.getRegistryId(e.getValue())).anyMatch(Objects::isNull)) {
        throw new IngestException("One or more of the metacards is not a registry metacard");
    }
    Map<String, Metacard> updatedMetacards = request.getUpdates().stream().collect(Collectors.toMap(e -> RegistryUtility.getRegistryId(e.getValue()), Map.Entry::getValue));
    Map<String, Metacard> origMetacards = ((OperationTransaction) request.getPropertyValue(Constants.OPERATION_TRANSACTION_KEY)).getPreviousStateMetacards().stream().collect(Collectors.toMap(RegistryUtility::getRegistryId, e -> e));
    //update the new metacards with the id from the orig so that they can be found on the remote system
    try {
        for (Map.Entry<String, Metacard> entry : updatedMetacards.entrySet()) {
            setMetacardExtID(entry.getValue(), origMetacards.get(entry.getKey()).getId());
        }
    } catch (ParserException e) {
        throw new IngestException("Could not update metacards id", e);
    }
    return super.update(request);
}
Also used : CreateRequest(ddf.catalog.operation.CreateRequest) UnsupportedQueryException(ddf.catalog.source.UnsupportedQueryException) LoggerFactory(org.slf4j.LoggerFactory) TagsFilterDelegate(ddf.catalog.filter.delegate.TagsFilterDelegate) Locale(java.util.Locale) Map(java.util.Map) DeleteRequestImpl(ddf.catalog.operation.impl.DeleteRequestImpl) URI(java.net.URI) Bundle(org.osgi.framework.Bundle) Converter(com.thoughtworks.xstream.converters.Converter) ImmutableSet(com.google.common.collect.ImmutableSet) RegistryUtility(org.codice.ddf.registry.common.metacard.RegistryUtility) SourceResponseImpl(ddf.catalog.operation.impl.SourceResponseImpl) SourceMonitor(ddf.catalog.source.SourceMonitor) ParserException(org.codice.ddf.parser.ParserException) AbstractCswStore(org.codice.ddf.spatial.ogc.csw.catalog.common.source.AbstractCswStore) Collectors(java.util.stream.Collectors) BundleContext(org.osgi.framework.BundleContext) Serializable(java.io.Serializable) Objects(java.util.Objects) SecureCxfClientFactory(org.codice.ddf.cxf.SecureCxfClientFactory) DeleteRequest(ddf.catalog.operation.DeleteRequest) List(java.util.List) ExternalIdentifierType(oasis.names.tc.ebxml_regrep.xsd.rim._3.ExternalIdentifierType) ConfigurationAdmin(org.osgi.service.cm.ConfigurationAdmin) RegistryObjectMetacardType(org.codice.ddf.registry.common.metacard.RegistryObjectMetacardType) UpdateResponse(ddf.catalog.operation.UpdateResponse) Dictionary(java.util.Dictionary) QueryRequestImpl(ddf.catalog.operation.impl.QueryRequestImpl) HashMap(java.util.HashMap) DeleteResponse(ddf.catalog.operation.DeleteResponse) OperationTransaction(ddf.catalog.operation.OperationTransaction) MetaTypeInformation(org.osgi.service.metatype.MetaTypeInformation) ArrayList(java.util.ArrayList) MetacardMarshaller(org.codice.ddf.registry.schemabindings.helper.MetacardMarshaller) Configuration(org.osgi.service.cm.Configuration) CreateResponse(ddf.catalog.operation.CreateResponse) CollectionUtils(org.apache.commons.collections.CollectionUtils) Constants(ddf.catalog.Constants) Metacard(ddf.catalog.data.Metacard) SecurityConstants(ddf.security.SecurityConstants) QueryRequest(ddf.catalog.operation.QueryRequest) UpdateRequest(ddf.catalog.operation.UpdateRequest) EncryptionService(ddf.security.encryption.EncryptionService) RegistryConstants(org.codice.ddf.registry.common.RegistryConstants) Result(ddf.catalog.data.Result) CreateRequestImpl(ddf.catalog.operation.impl.CreateRequestImpl) QueryImpl(ddf.catalog.operation.impl.QueryImpl) Logger(org.slf4j.Logger) RegistryPackageType(oasis.names.tc.ebxml_regrep.xsd.rim._3.RegistryPackageType) IngestException(ddf.catalog.source.IngestException) RegistryStore(org.codice.ddf.registry.api.internal.RegistryStore) CswSourceConfiguration(org.codice.ddf.spatial.ogc.csw.catalog.common.CswSourceConfiguration) IOException(java.io.IOException) CreateResponseImpl(ddf.catalog.operation.impl.CreateResponseImpl) MetaTypeService(org.osgi.service.metatype.MetaTypeService) Consumer(java.util.function.Consumer) Query(ddf.catalog.operation.Query) SourceResponse(ddf.catalog.operation.SourceResponse) Filter(org.opengis.filter.Filter) Collections(java.util.Collections) FrameworkUtil(org.osgi.framework.FrameworkUtil) OperationTransaction(ddf.catalog.operation.OperationTransaction) ParserException(org.codice.ddf.parser.ParserException) Metacard(ddf.catalog.data.Metacard) Objects(java.util.Objects) IngestException(ddf.catalog.source.IngestException) Map(java.util.Map) HashMap(java.util.HashMap)

Example 29 with Objects

use of java.util.Objects in project ddf by codice.

the class RegistryStoreImpl method create.

@Override
public CreateResponse create(CreateRequest request) throws IngestException {
    if (request.getMetacards().stream().map(RegistryUtility::getRegistryId).anyMatch(Objects::isNull)) {
        throw new IngestException("One or more of the metacards is not a registry metacard");
    }
    validateOperation();
    List<Filter> regIdFilters = request.getMetacards().stream().map(e -> filterBuilder.attribute(RegistryObjectMetacardType.REMOTE_METACARD_ID).is().equalTo().text(e.getId())).collect(Collectors.toList());
    Filter tagFilter = filterBuilder.attribute(Metacard.TAGS).is().equalTo().text(RegistryConstants.REGISTRY_TAG_INTERNAL);
    Map<String, Serializable> queryProps = new HashMap<>();
    queryProps.put(SecurityConstants.SECURITY_SUBJECT, request.getPropertyValue(SecurityConstants.SECURITY_SUBJECT));
    QueryImpl query = new QueryImpl(filterBuilder.allOf(tagFilter, filterBuilder.attribute(RegistryObjectMetacardType.REGISTRY_LOCAL_NODE).empty(), filterBuilder.anyOf(regIdFilters)));
    QueryRequest queryRequest = new QueryRequestImpl(query, queryProps);
    try {
        SourceResponse queryResponse = super.query(queryRequest);
        Map<String, Metacard> responseMap = queryResponse.getResults().stream().collect(Collectors.toMap(e -> RegistryUtility.getRegistryId(e.getMetacard()), Result::getMetacard));
        List<Metacard> metacardsToCreate = request.getMetacards().stream().filter(e -> !responseMap.containsKey(RegistryUtility.getRegistryId(e))).collect(Collectors.toList());
        List<Metacard> allMetacards = new ArrayList<>(responseMap.values());
        if (CollectionUtils.isNotEmpty(metacardsToCreate)) {
            CreateResponse createResponse = super.create(new CreateRequestImpl(metacardsToCreate, request.getProperties()));
            allMetacards.addAll(createResponse.getCreatedMetacards());
        }
        return new CreateResponseImpl(request, request.getProperties(), allMetacards);
    } catch (UnsupportedQueryException e) {
        LOGGER.warn("Unable to perform pre-create remote query. Proceeding with original query. Error was {}", e.getMessage());
    }
    return super.create(request);
}
Also used : CreateRequest(ddf.catalog.operation.CreateRequest) UnsupportedQueryException(ddf.catalog.source.UnsupportedQueryException) LoggerFactory(org.slf4j.LoggerFactory) TagsFilterDelegate(ddf.catalog.filter.delegate.TagsFilterDelegate) Locale(java.util.Locale) Map(java.util.Map) DeleteRequestImpl(ddf.catalog.operation.impl.DeleteRequestImpl) URI(java.net.URI) Bundle(org.osgi.framework.Bundle) Converter(com.thoughtworks.xstream.converters.Converter) ImmutableSet(com.google.common.collect.ImmutableSet) RegistryUtility(org.codice.ddf.registry.common.metacard.RegistryUtility) SourceResponseImpl(ddf.catalog.operation.impl.SourceResponseImpl) SourceMonitor(ddf.catalog.source.SourceMonitor) ParserException(org.codice.ddf.parser.ParserException) AbstractCswStore(org.codice.ddf.spatial.ogc.csw.catalog.common.source.AbstractCswStore) Collectors(java.util.stream.Collectors) BundleContext(org.osgi.framework.BundleContext) Serializable(java.io.Serializable) Objects(java.util.Objects) SecureCxfClientFactory(org.codice.ddf.cxf.SecureCxfClientFactory) DeleteRequest(ddf.catalog.operation.DeleteRequest) List(java.util.List) ExternalIdentifierType(oasis.names.tc.ebxml_regrep.xsd.rim._3.ExternalIdentifierType) ConfigurationAdmin(org.osgi.service.cm.ConfigurationAdmin) RegistryObjectMetacardType(org.codice.ddf.registry.common.metacard.RegistryObjectMetacardType) UpdateResponse(ddf.catalog.operation.UpdateResponse) Dictionary(java.util.Dictionary) QueryRequestImpl(ddf.catalog.operation.impl.QueryRequestImpl) HashMap(java.util.HashMap) DeleteResponse(ddf.catalog.operation.DeleteResponse) OperationTransaction(ddf.catalog.operation.OperationTransaction) MetaTypeInformation(org.osgi.service.metatype.MetaTypeInformation) ArrayList(java.util.ArrayList) MetacardMarshaller(org.codice.ddf.registry.schemabindings.helper.MetacardMarshaller) Configuration(org.osgi.service.cm.Configuration) CreateResponse(ddf.catalog.operation.CreateResponse) CollectionUtils(org.apache.commons.collections.CollectionUtils) Constants(ddf.catalog.Constants) Metacard(ddf.catalog.data.Metacard) SecurityConstants(ddf.security.SecurityConstants) QueryRequest(ddf.catalog.operation.QueryRequest) UpdateRequest(ddf.catalog.operation.UpdateRequest) EncryptionService(ddf.security.encryption.EncryptionService) RegistryConstants(org.codice.ddf.registry.common.RegistryConstants) Result(ddf.catalog.data.Result) CreateRequestImpl(ddf.catalog.operation.impl.CreateRequestImpl) QueryImpl(ddf.catalog.operation.impl.QueryImpl) Logger(org.slf4j.Logger) RegistryPackageType(oasis.names.tc.ebxml_regrep.xsd.rim._3.RegistryPackageType) IngestException(ddf.catalog.source.IngestException) RegistryStore(org.codice.ddf.registry.api.internal.RegistryStore) CswSourceConfiguration(org.codice.ddf.spatial.ogc.csw.catalog.common.CswSourceConfiguration) IOException(java.io.IOException) CreateResponseImpl(ddf.catalog.operation.impl.CreateResponseImpl) MetaTypeService(org.osgi.service.metatype.MetaTypeService) Consumer(java.util.function.Consumer) Query(ddf.catalog.operation.Query) SourceResponse(ddf.catalog.operation.SourceResponse) Filter(org.opengis.filter.Filter) Collections(java.util.Collections) FrameworkUtil(org.osgi.framework.FrameworkUtil) Serializable(java.io.Serializable) QueryRequest(ddf.catalog.operation.QueryRequest) SourceResponse(ddf.catalog.operation.SourceResponse) HashMap(java.util.HashMap) CreateResponse(ddf.catalog.operation.CreateResponse) UnsupportedQueryException(ddf.catalog.source.UnsupportedQueryException) ArrayList(java.util.ArrayList) QueryImpl(ddf.catalog.operation.impl.QueryImpl) Metacard(ddf.catalog.data.Metacard) Filter(org.opengis.filter.Filter) QueryRequestImpl(ddf.catalog.operation.impl.QueryRequestImpl) Objects(java.util.Objects) CreateRequestImpl(ddf.catalog.operation.impl.CreateRequestImpl) IngestException(ddf.catalog.source.IngestException) CreateResponseImpl(ddf.catalog.operation.impl.CreateResponseImpl)

Example 30 with Objects

use of java.util.Objects in project ddf by codice.

the class Historian method version.

/**
     * Versions updated {@link Metacard}s and {@link ContentItem}s.
     *
     * @param streamUpdateRequest   Needed to pass {@link ddf.catalog.core.versioning.MetacardVersion#SKIP_VERSIONING}
     *                              flag into downstream update
     * @param updateStorageResponse Versions this response's updated items
     * @return the update response originally passed in
     * @throws UnsupportedQueryException
     * @throws SourceUnavailableException
     * @throws IngestException
     */
public UpdateStorageResponse version(UpdateStorageRequest streamUpdateRequest, UpdateStorageResponse updateStorageResponse, UpdateResponse updateResponse) throws UnsupportedQueryException, SourceUnavailableException, IngestException {
    if (doSkip(updateStorageResponse)) {
        return updateStorageResponse;
    }
    setSkipFlag(streamUpdateRequest);
    setSkipFlag(updateStorageResponse);
    List<Metacard> updatedMetacards = updateStorageResponse.getUpdatedContentItems().stream().filter(ci -> StringUtils.isBlank(ci.getQualifier())).map(ContentItem::getMetacard).filter(Objects::nonNull).filter(isNotVersionNorDeleted).collect(Collectors.toList());
    Map<String, Metacard> originalMetacards = query(forIds(updatedMetacards.stream().map(Metacard::getId).collect(Collectors.toList())));
    Collection<ReadStorageRequest> ids = getReadStorageRequests(updatedMetacards);
    Map<String, List<ContentItem>> content = getContent(ids);
    Function<String, Action> getAction = (id) -> content.containsKey(id) ? Action.VERSIONED_CONTENT : Action.VERSIONED;
    Map<String, Metacard> versionMetacards = getVersionMetacards(originalMetacards.values(), getAction, (Subject) updateResponse.getProperties().get(SecurityConstants.SECURITY_SUBJECT));
    CreateStorageResponse createStorageResponse = versionContentItems(content, versionMetacards);
    if (createStorageResponse == null) {
        LOGGER.debug("Could not version content items.");
        return updateStorageResponse;
    }
    setResourceUriForContent(/*mutable*/
    versionMetacards, createStorageResponse);
    storeVersionMetacards(versionMetacards);
    return updateStorageResponse;
}
Also used : UnsupportedQueryException(ddf.catalog.source.UnsupportedQueryException) LoggerFactory(org.slf4j.LoggerFactory) StringUtils(org.apache.commons.lang3.StringUtils) ReadStorageRequest(ddf.catalog.content.operation.ReadStorageRequest) UpdateStorageRequest(ddf.catalog.content.operation.UpdateStorageRequest) DeletedMetacardImpl(ddf.catalog.core.versioning.impl.DeletedMetacardImpl) Map(java.util.Map) Action(ddf.catalog.core.versioning.MetacardVersion.Action) SubjectUtils(ddf.security.SubjectUtils) Bundle(org.osgi.framework.Bundle) UpdateStorageResponse(ddf.catalog.content.operation.UpdateStorageResponse) MetacardVersionImpl(ddf.catalog.core.versioning.impl.MetacardVersionImpl) Predicate(java.util.function.Predicate) Collection(java.util.Collection) Collectors(java.util.stream.Collectors) BundleContext(org.osgi.framework.BundleContext) MetacardType(ddf.catalog.data.MetacardType) Objects(java.util.Objects) StorageException(ddf.catalog.content.StorageException) List(java.util.List) Operation(ddf.catalog.operation.Operation) CreateStorageResponse(ddf.catalog.content.operation.CreateStorageResponse) Optional(java.util.Optional) UpdateResponse(ddf.catalog.operation.UpdateResponse) QueryRequestImpl(ddf.catalog.operation.impl.QueryRequestImpl) ContentItemImpl(ddf.catalog.content.data.impl.ContentItemImpl) FilterBuilder(ddf.catalog.filter.FilterBuilder) SourceUnavailableException(ddf.catalog.source.SourceUnavailableException) AttributeImpl(ddf.catalog.data.impl.AttributeImpl) HashMap(java.util.HashMap) Callable(java.util.concurrent.Callable) DeleteResponse(ddf.catalog.operation.DeleteResponse) Function(java.util.function.Function) Update(ddf.catalog.operation.Update) ArrayList(java.util.ArrayList) Lists(com.google.common.collect.Lists) SKIP_VERSIONING(ddf.catalog.core.versioning.MetacardVersion.SKIP_VERSIONING) ContentItem(ddf.catalog.content.data.ContentItem) CreateResponse(ddf.catalog.operation.CreateResponse) Metacard(ddf.catalog.data.Metacard) SecurityConstants(ddf.security.SecurityConstants) StorageProvider(ddf.catalog.content.StorageProvider) ByteSource(com.google.common.io.ByteSource) Result(ddf.catalog.data.Result) Hashtable(java.util.Hashtable) Nullable(javax.annotation.Nullable) CreateRequestImpl(ddf.catalog.operation.impl.CreateRequestImpl) CreateStorageRequestImpl(ddf.catalog.content.operation.impl.CreateStorageRequestImpl) QueryImpl(ddf.catalog.operation.impl.QueryImpl) Logger(org.slf4j.Logger) Security(org.codice.ddf.security.common.Security) IngestException(ddf.catalog.source.IngestException) IOException(java.io.IOException) Subject(ddf.security.Subject) TimeUnit(java.util.concurrent.TimeUnit) StorageRequest(ddf.catalog.content.operation.StorageRequest) ReadStorageRequestImpl(ddf.catalog.content.operation.impl.ReadStorageRequestImpl) SourceResponse(ddf.catalog.operation.SourceResponse) CatalogProvider(ddf.catalog.source.CatalogProvider) Filter(org.opengis.filter.Filter) Collections(java.util.Collections) FrameworkUtil(org.osgi.framework.FrameworkUtil) InputStream(java.io.InputStream) UuidGenerator(org.codice.ddf.platform.util.uuidgenerator.UuidGenerator) ReadStorageResponse(ddf.catalog.content.operation.ReadStorageResponse) Action(ddf.catalog.core.versioning.MetacardVersion.Action) CreateStorageResponse(ddf.catalog.content.operation.CreateStorageResponse) Metacard(ddf.catalog.data.Metacard) ReadStorageRequest(ddf.catalog.content.operation.ReadStorageRequest) Objects(java.util.Objects) List(java.util.List) ArrayList(java.util.ArrayList)

Aggregations

Objects (java.util.Objects)55 List (java.util.List)29 Map (java.util.Map)24 Collectors (java.util.stream.Collectors)22 ArrayList (java.util.ArrayList)20 Set (java.util.Set)19 Optional (java.util.Optional)16 IOException (java.io.IOException)15 HashMap (java.util.HashMap)14 Collections (java.util.Collections)13 HashSet (java.util.HashSet)10 ImmutableSet (com.google.common.collect.ImmutableSet)9 Result (ddf.catalog.data.Result)9 Stream (java.util.stream.Stream)9 Metacard (ddf.catalog.data.Metacard)8 TimeUnit (java.util.concurrent.TimeUnit)8 LoggerFactory (org.slf4j.LoggerFactory)8 QueryImpl (ddf.catalog.operation.impl.QueryImpl)7 Path (java.nio.file.Path)7 Logger (org.slf4j.Logger)7