Examples with UpdateStorageRequestImpl ddf.catalog.content.operation.impl.UpdateStorageRequestImpl used on opensource projects

Example 1 with UpdateStorageRequestImpl

use of ddf.catalog.content.operation.impl.UpdateStorageRequestImpl in project ddf by codice.

the class InMemoryProcessingFramework method storeContentItemUpdates.

private void storeContentItemUpdates(Map<String, ContentItem> contentItemsToUpdate, Map<String, Serializable> properties) {
    if (MapUtils.isNotEmpty(contentItemsToUpdate)) {
        LOGGER.trace("Storing content item updates(s)");
        UpdateStorageRequest updateStorageRequest = new UpdateStorageRequestImpl(new ArrayList<>(contentItemsToUpdate.values()), properties);
        Subject subject = (Subject) updateStorageRequest.getProperties().get(SecurityConstants.SECURITY_SUBJECT);
        if (subject == null) {
            LOGGER.debug("No subject to send UpdateStorageRequest. Updates will not be sent back to the catalog");
        } else {
            subject.execute(() -> {
                try {
                    catalogFramework.update(updateStorageRequest);
                    LOGGER.debug("Successfully completed update storage request");
                } catch (IngestException | SourceUnavailableException | RuntimeException e) {
                    LOGGER.info("Unable to complete update storage request", e);
                }
                return null;
            });
        }
    } else {
        LOGGER.debug("No content items to update");
    }
}

Example 2 with UpdateStorageRequestImpl

use of ddf.catalog.content.operation.impl.UpdateStorageRequestImpl in project ddf by codice.

the class MetacardApplication method revertContentandMetacard.

private void revertContentandMetacard(Metacard latestContent, Metacard versionMetacard, String id) throws SourceUnavailableException, IngestException, ResourceNotFoundException, IOException, ResourceNotSupportedException, FederationException, UnsupportedQueryException {
    LOGGER.trace("Reverting content and metacard for metacard [{}]. \nLatest content: [{}] \nVersion metacard: [{}]", id, latestContent.getId(), versionMetacard.getId());
    Map<String, Serializable> properties = new HashMap<>();
    properties.put("no-default-tags", true);
    ResourceResponse latestResource = catalogFramework.getLocalResource(new ResourceRequestById(latestContent.getId(), properties));
    ContentItemImpl contentItem = new ContentItemImpl(id, new ByteSourceWrapper(() -> latestResource.getResource().getInputStream()), latestResource.getResource().getMimeTypeValue(), latestResource.getResource().getName(), latestResource.getResource().getSize(), MetacardVersionImpl.toMetacard(versionMetacard, types));
    // Try to delete the "deleted metacard" marker first.
    boolean alreadyCreated = false;
    Action action = Action.fromKey((String) versionMetacard.getAttribute(MetacardVersion.ACTION).getValue());
    if (DELETE_ACTIONS.contains(action)) {
        alreadyCreated = true;
        catalogFramework.create(new CreateStorageRequestImpl(Collections.singletonList(contentItem), id, new HashMap<>()));
    } else {
        // Currently we can't guarantee the metacard will exist yet because of the 1 second
        // soft commit in solr. this busy wait loop should be fixed when alternate solution
        // is found.
        tryUpdate(4, () -> {
            catalogFramework.update(new UpdateStorageRequestImpl(Collections.singletonList(contentItem), id, new HashMap<>()));
            return true;
        });
    }
    LOGGER.trace("Successfully reverted metacard content for [{}]", id);
    revertMetacard(versionMetacard, id, alreadyCreated);
}

Example 3 with UpdateStorageRequestImpl

use of ddf.catalog.content.operation.impl.UpdateStorageRequestImpl in project ddf by codice.

the class CatalogFrameworkImplTest method testUpdateStorage.

/**
 * Tests that the framework properly passes an update request to the local provider.
 */
@Test
public void testUpdateStorage() throws Exception {
    List<ContentItem> contentItems = new ArrayList<>();
    MetacardImpl metacard = new MetacardImpl();
    metacard.setId(null);
    ByteSource byteSource = new ByteSource() {

        @Override
        public InputStream openStream() throws IOException {
            return new ByteArrayInputStream("blah".getBytes());
        }
    };
    ContentItemImpl contentItem = new ContentItemImpl(uuidGenerator.generateUuid(), byteSource, "application/octet-stream", "blah", 0L, metacard);
    contentItems.add(contentItem);
    CreateResponse response = framework.create(new CreateStorageRequestImpl(contentItems, null));
    Metacard insertedCard = response.getCreatedMetacards().get(0);
    List<ContentItem> updatedContentItems = new ArrayList<>();
    updatedContentItems.add(new ContentItemImpl(insertedCard.getId(), byteSource, "application/octet-stream", insertedCard));
    UpdateStorageRequest request = new UpdateStorageRequestImpl(updatedContentItems, null);
    List<Result> mockFederationResults = Stream.of(insertedCard).map(m -> {
        Result mockResult = mock(Result.class);
        when(mockResult.getMetacard()).thenReturn(m);
        return mockResult;
    }).collect(Collectors.toList());
    QueryResponseImpl queryResponse = new QueryResponseImpl(mock(QueryRequest.class), mockFederationResults, 1);
    when(mockFederationStrategy.federate(anyList(), any())).thenReturn(queryResponse);
    // send update to framework
    List<Update> returnedCards = framework.update(request).getUpdatedMetacards();
    assertThat(returnedCards, hasSize(1));
    final Metacard newMetacard = returnedCards.get(0).getNewMetacard();
    assertThat(newMetacard.getId(), notNullValue());
    assertThat(newMetacard.getResourceURI().toString(), is(contentItem.getUri()));
    assertThat(newMetacard.getResourceSize(), is(Long.toString(byteSource.size())));
    assertThat(response.getCreatedMetacards(), hasSize(storageProvider.size()));
    // make sure that the event was posted correctly
    assertThat(eventAdmin.wasEventPosted(), is(true));
}

Example 4 with UpdateStorageRequestImpl

use of ddf.catalog.content.operation.impl.UpdateStorageRequestImpl in project ddf by codice.

the class ContentProducerDataAccessObject method createContentItem.

public void createContentItem(FileSystemPersistenceProvider fileIdMap, ContentEndpoint endpoint, File ingestedFile, WatchEvent.Kind<Path> eventType, String mimeType, Map<String, Object> headers) throws SourceUnavailableException, IngestException {
    LOGGER.debug("Creating content item.");
    if (!eventType.equals(ENTRY_DELETE) && ingestedFile == null) {
        if (LOGGER.isDebugEnabled()) {
            LOGGER.debug("Ingested File was null with eventType [{}]. Doing nothing.", eventType.name());
        }
        return;
    }
    String refKey = (String) headers.get(Constants.STORE_REFERENCE_KEY);
    String safeKey = null;
    String id = null;
    // not null if the file lives outside the content store (external reference)
    if (refKey != null) {
        // guards against impermissible filesystem characters
        safeKey = DigestUtils.sha1Hex(refKey);
        if (fileIdMap.loadAllKeys().contains(safeKey)) {
            id = String.valueOf(fileIdMap.loadFromPersistence(safeKey));
        } else if (!ENTRY_CREATE.equals(eventType)) {
            LOGGER.warn("Unable to look up id for {}, not performing {}", refKey, eventType.name());
            return;
        }
    }
    if (ENTRY_CREATE.equals(eventType)) {
        CreateStorageRequest createRequest = new CreateStorageRequestImpl(Collections.singletonList(new ContentItemImpl(uuidGenerator.generateUuid(), Files.asByteSource(ingestedFile), mimeType, ingestedFile.getName(), ingestedFile.length(), null)), getProperties(headers));
        CatalogFramework catalogFramework = endpoint.getComponent().getCatalogFramework();
        waitForAvailableSource(catalogFramework);
        CreateResponse createResponse = catalogFramework.create(createRequest);
        if (createResponse != null) {
            List<Metacard> createdMetacards = createResponse.getCreatedMetacards();
            if (safeKey != null) {
                fileIdMap.store(safeKey, createdMetacards.get(0).getId());
            }
            logIds(createdMetacards, "created");
        }
    } else if (ENTRY_MODIFY.equals(eventType)) {
        UpdateStorageRequest updateRequest = new UpdateStorageRequestImpl(Collections.singletonList(new ContentItemImpl(id, Files.asByteSource(ingestedFile), mimeType, ingestedFile.getName(), 0, null)), getProperties(headers));
        UpdateResponse updateResponse = endpoint.getComponent().getCatalogFramework().update(updateRequest);
        if (updateResponse != null) {
            List<Update> updatedMetacards = updateResponse.getUpdatedMetacards();
            logIds(updatedMetacards.stream().map(Update::getNewMetacard).collect(Collectors.toList()), "updated");
        }
    } else if (ENTRY_DELETE.equals(eventType)) {
        DeleteRequest deleteRequest = new DeleteRequestImpl(id);
        DeleteResponse deleteResponse = endpoint.getComponent().getCatalogFramework().delete(deleteRequest);
        if (deleteResponse != null) {
            List<Metacard> deletedMetacards = deleteResponse.getDeletedMetacards();
            if (safeKey != null) {
                fileIdMap.delete(safeKey);
            }
            logIds(deletedMetacards, "deleted");
        }
    }
}

Example 5 with UpdateStorageRequestImpl

use of ddf.catalog.content.operation.impl.UpdateStorageRequestImpl in project ddf by codice.

the class UpdateOperations method update.

public UpdateResponse update(UpdateStorageRequest streamUpdateRequest) throws IngestException, SourceUnavailableException {
    Map<String, Metacard> metacardMap = new HashMap<>();
    List<ContentItem> contentItems = new ArrayList<>(streamUpdateRequest.getContentItems().size());
    HashMap<String, Map<String, Path>> tmpContentPaths = new HashMap<>();
    UpdateResponse updateResponse = null;
    UpdateStorageRequest updateStorageRequest = null;
    UpdateStorageResponse updateStorageResponse = null;
    streamUpdateRequest = opsStorageSupport.prepareStorageRequest(streamUpdateRequest, streamUpdateRequest::getContentItems);
    // Operation populates the metacardMap, contentItems, and tmpContentPaths
    opsMetacardSupport.generateMetacardAndContentItems(streamUpdateRequest.getContentItems(), metacardMap, contentItems, tmpContentPaths);
    streamUpdateRequest.getProperties().put(CONTENT_PATHS, tmpContentPaths);
    streamUpdateRequest = applyAttributeOverrides(streamUpdateRequest, metacardMap);
    try {
        if (!contentItems.isEmpty()) {
            updateStorageRequest = new UpdateStorageRequestImpl(contentItems, streamUpdateRequest.getId(), streamUpdateRequest.getProperties());
            updateStorageRequest = processPreUpdateStoragePlugins(updateStorageRequest);
            try {
                updateStorageResponse = sourceOperations.getStorage().update(updateStorageRequest);
                updateStorageResponse.getProperties().put(CONTENT_PATHS, tmpContentPaths);
            } catch (StorageException e) {
                throw new IngestException("Could not store content items. Removed created metacards.", e);
            }
            updateStorageResponse = processPostUpdateStoragePlugins(updateStorageResponse);
            for (ContentItem contentItem : updateStorageResponse.getUpdatedContentItems()) {
                if (StringUtils.isBlank(contentItem.getQualifier())) {
                    Metacard metacard = metacardMap.get(contentItem.getId());
                    Metacard overrideMetacard = contentItem.getMetacard();
                    Metacard updatedMetacard = OverrideAttributesSupport.overrideMetacard(metacard, overrideMetacard, true);
                    updatedMetacard.setAttribute(new AttributeImpl(Metacard.RESOURCE_SIZE, String.valueOf(contentItem.getSize())));
                    metacardMap.put(contentItem.getId(), updatedMetacard);
                }
            }
        }
        UpdateRequestImpl updateRequest = new UpdateRequestImpl(Iterables.toArray(metacardMap.values().stream().map(Metacard::getId).collect(Collectors.toList()), String.class), new ArrayList<>(metacardMap.values()));
        updateRequest.setProperties(streamUpdateRequest.getProperties());
        historian.setSkipFlag(updateRequest);
        updateResponse = doUpdate(updateRequest);
        historian.version(streamUpdateRequest, updateStorageResponse, updateResponse);
    } catch (Exception e) {
        if (updateStorageRequest != null) {
            try {
                sourceOperations.getStorage().rollback(updateStorageRequest);
            } catch (StorageException e1) {
                LOGGER.info("Unable to remove temporary content for id: {}", updateStorageRequest.getId(), e1);
            }
        }
        throw new IngestException("Unable to store products for request: " + streamUpdateRequest.getId(), e);
    } finally {
        opsStorageSupport.commitAndCleanup(updateStorageRequest, tmpContentPaths);
    }
    updateResponse = doPostIngest(updateResponse);
    return updateResponse;
}