Examples with CreateStorageRequestImpl ddf.catalog.content.operation.impl.CreateStorageRequestImpl used on opensource projects

Example 6 with CreateStorageRequestImpl

use of ddf.catalog.content.operation.impl.CreateStorageRequestImpl in project ddf by codice.

the class ContentProducerDataAccessObject method createContentItem.

public void createContentItem(FileSystemPersistenceProvider fileIdMap, ContentEndpoint endpoint, File ingestedFile, WatchEvent.Kind<Path> eventType, String mimeType, Map<String, Object> headers) throws SourceUnavailableException, IngestException {
    LOGGER.debug("Creating content item.");
    String key = String.valueOf(ingestedFile.getAbsolutePath().hashCode());
    String id = fileIdMap.loadAllKeys().contains(key) ? String.valueOf(fileIdMap.loadFromPersistence(key)) : null;
    if (StandardWatchEventKinds.ENTRY_CREATE.equals(eventType)) {
        CreateStorageRequest createRequest = new CreateStorageRequestImpl(Collections.singletonList(new ContentItemImpl(uuidGenerator.generateUuid(), Files.asByteSource(ingestedFile), mimeType, ingestedFile.getName(), 0L, null)), null);
        processHeaders(headers, createRequest, ingestedFile);
        CreateResponse createResponse = endpoint.getComponent().getCatalogFramework().create(createRequest);
        if (createResponse != null) {
            List<Metacard> createdMetacards = createResponse.getCreatedMetacards();
            for (Metacard metacard : createdMetacards) {
                fileIdMap.store(key, metacard.getId());
                LOGGER.debug("content item created with id = {}", metacard.getId());
            }
        }
    } else if (StandardWatchEventKinds.ENTRY_MODIFY.equals(eventType)) {
        UpdateStorageRequest updateRequest = new UpdateStorageRequestImpl(Collections.singletonList(new ContentItemImpl(id, Files.asByteSource(ingestedFile), mimeType, ingestedFile.getName(), 0, null)), null);
        processHeaders(headers, updateRequest, ingestedFile);
        UpdateResponse updateResponse = endpoint.getComponent().getCatalogFramework().update(updateRequest);
        if (updateResponse != null) {
            List<Update> updatedMetacards = updateResponse.getUpdatedMetacards();
            for (Update update : updatedMetacards) {
                LOGGER.debug("content item updated with id = {}", update.getNewMetacard().getId());
            }
        }
    } else if (StandardWatchEventKinds.ENTRY_DELETE.equals(eventType)) {
        DeleteRequest deleteRequest = new DeleteRequestImpl(id);
        DeleteResponse deleteResponse = endpoint.getComponent().getCatalogFramework().delete(deleteRequest);
        if (deleteResponse != null) {
            List<Metacard> deletedMetacards = deleteResponse.getDeletedMetacards();
            for (Metacard delete : deletedMetacards) {
                fileIdMap.delete(ingestedFile.getAbsolutePath());
                LOGGER.debug("content item deleted with id = {}", delete.getId());
            }
        }
    }
}

Example 7 with CreateStorageRequestImpl

use of ddf.catalog.content.operation.impl.CreateStorageRequestImpl in project ddf by codice.

the class CreateOperations method create.

public CreateResponse create(CreateStorageRequest streamCreateRequest, List<String> fanoutTagBlacklist) throws IngestException, SourceUnavailableException {
    Map<String, Metacard> metacardMap = new HashMap<>();
    List<ContentItem> contentItems = new ArrayList<>(streamCreateRequest.getContentItems().size());
    HashMap<String, Map<String, Path>> tmpContentPaths = new HashMap<>();
    CreateResponse createResponse = null;
    CreateStorageRequest createStorageRequest = null;
    CreateStorageResponse createStorageResponse;
    streamCreateRequest = opsStorageSupport.prepareStorageRequest(streamCreateRequest, streamCreateRequest::getContentItems);
    // Operation populates the metacardMap, contentItems, and tmpContentPaths
    opsMetacardSupport.generateMetacardAndContentItems(streamCreateRequest.getContentItems(), metacardMap, contentItems, tmpContentPaths);
    if (blockCreateMetacards(metacardMap.values(), fanoutTagBlacklist)) {
        String message = "Fanout proxy does not support create operations with blacklisted metacard tag";
        LOGGER.debug("{}. Tags blacklist: {}", message, fanoutTagBlacklist);
        throw new IngestException(message);
    }
    streamCreateRequest.getProperties().put(CONTENT_PATHS, tmpContentPaths);
    injectAttributes(metacardMap);
    setDefaultValues(metacardMap);
    streamCreateRequest = applyAttributeOverrides(streamCreateRequest, metacardMap);
    try {
        if (!contentItems.isEmpty()) {
            createStorageRequest = new CreateStorageRequestImpl(contentItems, streamCreateRequest.getId(), streamCreateRequest.getProperties());
            createStorageRequest = processPreCreateStoragePlugins(createStorageRequest);
            try {
                createStorageResponse = sourceOperations.getStorage().create(createStorageRequest);
                createStorageResponse.getProperties().put(CONTENT_PATHS, tmpContentPaths);
            } catch (StorageException e) {
                throw new IngestException("Could not store content items.", e);
            }
            createStorageResponse = processPostCreateStoragePlugins(createStorageResponse);
            populateMetacardMap(metacardMap, createStorageResponse);
        }
        CreateRequest createRequest = new CreateRequestImpl(new ArrayList<>(metacardMap.values()), Optional.ofNullable(createStorageRequest).map(StorageRequest::getProperties).orElseGet(HashMap::new));
        createResponse = doCreate(createRequest);
    } catch (IOException | RuntimeException e) {
        if (createStorageRequest != null) {
            try {
                sourceOperations.getStorage().rollback(createStorageRequest);
            } catch (StorageException e1) {
                LOGGER.info("Unable to remove temporary content for id: {}", createStorageRequest.getId(), e1);
            }
        }
        throw new IngestException("Unable to store products for request: " + streamCreateRequest.getId(), e);
    } finally {
        opsStorageSupport.commitAndCleanup(createStorageRequest, tmpContentPaths);
    }
    createResponse = doPostIngest(createResponse);
    return createResponse;
}

Example 8 with CreateStorageRequestImpl

use of ddf.catalog.content.operation.impl.CreateStorageRequestImpl in project ddf by codice.

the class MetacardApplication method revertContentandMetacard.

private void revertContentandMetacard(Metacard latestContent, Metacard versionMetacard, String id) throws SourceUnavailableException, IngestException, ResourceNotFoundException, IOException, ResourceNotSupportedException, FederationException, UnsupportedQueryException {
    LOGGER.trace("Reverting content and metacard for metacard [{}]. \nLatest content: [{}] \nVersion metacard: [{}]", id, latestContent.getId(), versionMetacard.getId());
    Map<String, Serializable> properties = new HashMap<>();
    properties.put("no-default-tags", true);
    ResourceResponse latestResource = catalogFramework.getLocalResource(new ResourceRequestById(latestContent.getId(), properties));
    ContentItemImpl contentItem = new ContentItemImpl(id, new ByteSourceWrapper(() -> latestResource.getResource().getInputStream()), latestResource.getResource().getMimeTypeValue(), latestResource.getResource().getName(), latestResource.getResource().getSize(), MetacardVersionImpl.toMetacard(versionMetacard, types));
    // Try to delete the "deleted metacard" marker first.
    boolean alreadyCreated = false;
    Action action = Action.fromKey((String) versionMetacard.getAttribute(MetacardVersion.ACTION).getValue());
    if (DELETE_ACTIONS.contains(action)) {
        alreadyCreated = true;
        catalogFramework.create(new CreateStorageRequestImpl(Collections.singletonList(contentItem), id, new HashMap<>()));
    } else {
        // Currently we can't guarantee the metacard will exist yet because of the 1 second
        // soft commit in solr. this busy wait loop should be fixed when alternate solution
        // is found.
        tryUpdate(4, () -> {
            catalogFramework.update(new UpdateStorageRequestImpl(Collections.singletonList(contentItem), id, new HashMap<>()));
            return true;
        });
    }
    LOGGER.trace("Successfully reverted metacard content for [{}]", id);
    revertMetacard(versionMetacard, id, alreadyCreated);
}

Example 9 with CreateStorageRequestImpl

use of ddf.catalog.content.operation.impl.CreateStorageRequestImpl in project ddf by codice.

the class CatalogFrameworkImplTest method testCreateStorageWithAttributeOverridesInvalidType.

/**
     * Tests that the framework properly passes a create request to the local provider with attribute overrides.
     */
@Test
public void testCreateStorageWithAttributeOverridesInvalidType() throws Exception {
    List<ContentItem> contentItems = new ArrayList<>();
    Map<String, Serializable> propertiesMap = new HashMap<>();
    HashMap<String, Object> attributeMap = new HashMap<>();
    attributeMap.put(Metacard.CREATED, "bad date");
    propertiesMap.put(Constants.ATTRIBUTE_OVERRIDES_KEY, attributeMap);
    MetacardImpl newCard = new MetacardImpl();
    newCard.setId(null);
    MetacardType metacardType = mock(MetacardType.class);
    AttributeDescriptor dateAttributeDescriptor = new AttributeDescriptorImpl(Metacard.CREATED, true, true, true, true, new AttributeType<Date>() {

        private static final long serialVersionUID = 1L;

        @Override
        public Class<Date> getBinding() {
            return Date.class;
        }

        @Override
        public AttributeFormat getAttributeFormat() {
            return AttributeFormat.DATE;
        }
    });
    when(metacardType.getAttributeDescriptor(Metacard.TITLE)).thenReturn(dateAttributeDescriptor);
    newCard.setType(metacardType);
    ByteSource byteSource = new ByteSource() {

        @Override
        public InputStream openStream() throws IOException {
            return new ByteArrayInputStream("blah".getBytes());
        }
    };
    ContentItemImpl newItem = new ContentItemImpl(uuidGenerator.generateUuid(), byteSource, "application/octet-stream", "blah", 0L, newCard);
    contentItems.add(newItem);
    CreateResponse response = framework.create(new CreateStorageRequestImpl(contentItems, propertiesMap));
    assertEquals(response.getCreatedMetacards().size(), provider.size());
    assertEquals(response.getCreatedMetacards().size(), storageProvider.size());
    for (Metacard curCard : response.getCreatedMetacards()) {
        assertNotNull(curCard.getId());
        // Assert value is not set for invalid format
        assertThat(curCard.getCreatedDate(), nullValue());
    }
}

Example 10 with CreateStorageRequestImpl

use of ddf.catalog.content.operation.impl.CreateStorageRequestImpl in project ddf by codice.

the class FanoutCatalogFrameworkTest method testBlacklistedTagCreateStorageRequestFails.

@Test(expected = IngestException.class)
public void testBlacklistedTagCreateStorageRequestFails() throws Exception {
    Metacard metacard = new MetacardImpl();
    metacard.setAttribute(new AttributeImpl(Metacard.TAGS, "blacklisted"));
    CatalogProvider catalogProvider = mock(CatalogProvider.class);
    doReturn(true).when(catalogProvider).isAvailable();
    StorageProvider storageProvider = new MockMemoryStorageProvider();
    MimeTypeMapper mimeTypeMapper = mock(MimeTypeMapper.class);
    doReturn("extension").when(mimeTypeMapper).getFileExtensionForMimeType(anyString());
    InputTransformer transformer = mock(InputTransformer.class);
    doReturn(metacard).when(transformer).transform(any(InputStream.class));
    MimeTypeToTransformerMapper mimeTypeToTransformerMapper = mock(MimeTypeToTransformerMapper.class);
    doReturn(Collections.singletonList(transformer)).when(mimeTypeToTransformerMapper).findMatches(any(Class.class), any(MimeType.class));
    frameworkProperties.setCatalogProviders(Collections.singletonList(catalogProvider));
    frameworkProperties.setStorageProviders(Collections.singletonList(storageProvider));
    frameworkProperties.setMimeTypeMapper(mimeTypeMapper);
    frameworkProperties.setMimeTypeToTransformerMapper(mimeTypeToTransformerMapper);
    OperationsSecuritySupport opsSecurity = new OperationsSecuritySupport();
    MetacardFactory metacardFactory = new MetacardFactory(frameworkProperties.getMimeTypeToTransformerMapper(), uuidGenerator);
    OperationsMetacardSupport opsMetacard = new OperationsMetacardSupport(frameworkProperties, metacardFactory);
    SourceOperations sourceOperations = new SourceOperations(frameworkProperties);
    sourceOperations.bind(catalogProvider);
    sourceOperations.bind(storageProvider);
    TransformOperations transformOperations = new TransformOperations(frameworkProperties);
    QueryOperations queryOperations = new QueryOperations(frameworkProperties, sourceOperations, opsSecurity, opsMetacard);
    OperationsCatalogStoreSupport opsCatStore = new OperationsCatalogStoreSupport(frameworkProperties, sourceOperations);
    OperationsStorageSupport opsStorage = new OperationsStorageSupport(sourceOperations, queryOperations);
    ResourceOperations resourceOperations = new ResourceOperations(frameworkProperties, queryOperations, opsSecurity);
    OperationsMetacardSupport opsMetacardSupport = new OperationsMetacardSupport(frameworkProperties, metacardFactory);
    // Need to set these for InputValidation to work
    System.setProperty("bad.files", "none");
    System.setProperty("bad.file.extensions", "none");
    System.setProperty("bad.mime.types", "none");
    CreateOperations createOperations = new CreateOperations(frameworkProperties, queryOperations, sourceOperations, opsSecurity, opsMetacardSupport, opsCatStore, opsStorage);
    framework = new CatalogFrameworkImpl(createOperations, null, null, queryOperations, resourceOperations, sourceOperations, transformOperations);
    framework.setId(NEW_SOURCE_ID);
    framework.setFanoutEnabled(true);
    framework.setFanoutTagBlacklist(Collections.singletonList("blacklisted"));
    ContentItem item = new ContentItemImpl(uuidGenerator.generateUuid(), ByteSource.empty(), "text/xml", "filename.xml", 0L, metacard);
    CreateStorageRequest request = new CreateStorageRequestImpl(Collections.singletonList(item), new HashMap<>());
    framework.create(request);
}