Examples with CreateStorageRequest ddf.catalog.content.operation.CreateStorageRequest used on opensource projects

Example 6 with CreateStorageRequest

use of ddf.catalog.content.operation.CreateStorageRequest in project ddf by codice.

the class CreateOperations method create.

public CreateResponse create(CreateStorageRequest streamCreateRequest, List<String> fanoutTagBlacklist) throws IngestException, SourceUnavailableException {
    Map<String, Metacard> metacardMap = new HashMap<>();
    List<ContentItem> contentItems = new ArrayList<>(streamCreateRequest.getContentItems().size());
    HashMap<String, Map<String, Path>> tmpContentPaths = new HashMap<>();
    CreateResponse createResponse = null;
    CreateStorageRequest createStorageRequest = null;
    CreateStorageResponse createStorageResponse;
    streamCreateRequest = opsStorageSupport.prepareStorageRequest(streamCreateRequest, streamCreateRequest::getContentItems);
    // Operation populates the metacardMap, contentItems, and tmpContentPaths
    opsMetacardSupport.generateMetacardAndContentItems(streamCreateRequest.getContentItems(), metacardMap, contentItems, tmpContentPaths);
    if (blockCreateMetacards(metacardMap.values(), fanoutTagBlacklist)) {
        String message = "Fanout proxy does not support create operations with blacklisted metacard tag";
        LOGGER.debug("{}. Tags blacklist: {}", message, fanoutTagBlacklist);
        throw new IngestException(message);
    }
    streamCreateRequest.getProperties().put(CONTENT_PATHS, tmpContentPaths);
    injectAttributes(metacardMap);
    setDefaultValues(metacardMap);
    streamCreateRequest = applyAttributeOverrides(streamCreateRequest, metacardMap);
    try {
        if (!contentItems.isEmpty()) {
            createStorageRequest = new CreateStorageRequestImpl(contentItems, streamCreateRequest.getId(), streamCreateRequest.getProperties());
            createStorageRequest = processPreCreateStoragePlugins(createStorageRequest);
            try {
                createStorageResponse = sourceOperations.getStorage().create(createStorageRequest);
                createStorageResponse.getProperties().put(CONTENT_PATHS, tmpContentPaths);
            } catch (StorageException e) {
                throw new IngestException("Could not store content items.", e);
            }
            createStorageResponse = processPostCreateStoragePlugins(createStorageResponse);
            populateMetacardMap(metacardMap, createStorageResponse);
        }
        CreateRequest createRequest = new CreateRequestImpl(new ArrayList<>(metacardMap.values()), Optional.ofNullable(createStorageRequest).map(StorageRequest::getProperties).orElseGet(HashMap::new));
        createResponse = doCreate(createRequest);
    } catch (IOException | RuntimeException e) {
        if (createStorageRequest != null) {
            try {
                sourceOperations.getStorage().rollback(createStorageRequest);
            } catch (StorageException e1) {
                LOGGER.info("Unable to remove temporary content for id: {}", createStorageRequest.getId(), e1);
            }
        }
        throw new IngestException("Unable to store products for request: " + streamCreateRequest.getId(), e);
    } finally {
        opsStorageSupport.commitAndCleanup(createStorageRequest, tmpContentPaths);
    }
    createResponse = doPostIngest(createResponse);
    return createResponse;
}

Example 7 with CreateStorageRequest

use of ddf.catalog.content.operation.CreateStorageRequest in project ddf by codice.

the class TestChecksum method testProcessCreateWithValidInput.

@Test
public void testProcessCreateWithValidInput() throws PluginExecutionException {
    CreateStorageRequest request = checksum.process(mockCreateRequest);
    String checksumResult = (String) request.getContentItems().get(0).getMetacard().getAttribute(Metacard.CHECKSUM).getValue();
    String checksumAlgorithm = (String) request.getContentItems().get(0).getMetacard().getAttribute(Metacard.CHECKSUM_ALGORITHM).getValue();
    assertThat(checksumResult, is(SAMPLE_CHECKSUM_VALUE));
    assertThat(checksumAlgorithm, is(SAMPLE_CHECKSUM_ALGORITHM));
}

Example 8 with CreateStorageRequest

use of ddf.catalog.content.operation.CreateStorageRequest in project ddf by codice.

the class FileSystemStorageProviderTest method testRollback.

@Test(expected = StorageException.class)
public void testRollback() throws Exception {
    String id = UUID.randomUUID().toString().replaceAll("-", "");
    ByteSource byteSource = new ByteSource() {

        @Override
        public InputStream openStream() throws IOException {
            return IOUtils.toInputStream(TEST_INPUT_CONTENTS);
        }
    };
    Metacard metacard = mock(Metacard.class);
    when(metacard.getId()).thenReturn(id);
    ContentItem contentItem = new ContentItemImpl(id, byteSource, NITF_MIME_TYPE, TEST_INPUT_FILENAME, TEST_INPUT_CONTENTS.getBytes().length, metacard);
    CreateStorageRequest createRequest = new CreateStorageRequestImpl(Collections.singletonList(contentItem), null);
    CreateStorageResponse createStorageResponse = provider.create(createRequest);
    provider.rollback(createRequest);
    ReadStorageRequest readStorageRequest = new ReadStorageRequestImpl(new URI("content:" + id), null);
    ReadStorageResponse read = provider.read(readStorageRequest);
}

Example 9 with CreateStorageRequest

use of ddf.catalog.content.operation.CreateStorageRequest in project ddf by codice.

the class FileSystemStorageProviderTest method assertContentItemWithQualifier.

public CreateStorageResponse assertContentItemWithQualifier(String data, String mimeTypeRawData, String filename, String id, String qualifier, Map<String, Serializable> properties) throws Exception {
    // Simulates what ContentFrameworkImpl would do
    String uuid = StringUtils.defaultIfBlank(id, UUID.randomUUID().toString().replaceAll("-", ""));
    ByteSource byteSource = new ByteSource() {

        @Override
        public InputStream openStream() throws IOException {
            return IOUtils.toInputStream(data);
        }
    };
    ContentItem contentItem = new ContentItemImpl(uuid, qualifier, byteSource, mimeTypeRawData, filename, byteSource.size(), mock(Metacard.class));
    CreateStorageRequest createRequest = new CreateStorageRequestImpl(Collections.singletonList(contentItem), properties);
    CreateStorageResponse createResponse = provider.create(createRequest);
    List<ContentItem> createdContentItems = createResponse.getCreatedContentItems();
    ContentItem createdContentItem = createdContentItems.isEmpty() ? null : createdContentItems.get(0);
    assertNotNull(createdContentItem);
    String createdId = createdContentItem.getId();
    assertNotNull(createdId);
    assertThat(createdId, equalTo(uuid));
    String contentUri = createdContentItem.getUri();
    LOGGER.debug("contentUri = {}", contentUri);
    assertNotNull(contentUri);
    String expectedContentUri = ContentItem.CONTENT_SCHEME + ":" + uuid + ((StringUtils.isNotBlank(qualifier)) ? "#" + qualifier : "");
    assertThat(contentUri, equalTo(expectedContentUri));
    assertTrue(createdContentItem.getSize() > 0);
    String createdMimeType = createdContentItem.getMimeTypeRawData().replace(";", "");
    List<String> createdMimeTypeArr = new ArrayList<>(Arrays.asList(createdMimeType.split(" ")));
    List<String> givenMimeTypeArr = new ArrayList<>(Arrays.asList(mimeTypeRawData.replace(";", "").split(" ")));
    assertEquals(createdMimeTypeArr.size(), givenMimeTypeArr.size());
    givenMimeTypeArr.removeAll(createdMimeTypeArr);
    assertThat(givenMimeTypeArr.size(), is(0));
    provider.commit(createRequest);
    return createResponse;
}

Example 10 with CreateStorageRequest

use of ddf.catalog.content.operation.CreateStorageRequest in project ddf by codice.

the class FanoutCatalogFrameworkTest method testBlacklistedTagCreateStorageRequestFails.

@Test(expected = IngestException.class)
public void testBlacklistedTagCreateStorageRequestFails() throws Exception {
    Metacard metacard = new MetacardImpl();
    metacard.setAttribute(new AttributeImpl(Metacard.TAGS, "blacklisted"));
    CatalogProvider catalogProvider = mock(CatalogProvider.class);
    doReturn(true).when(catalogProvider).isAvailable();
    StorageProvider storageProvider = new MockMemoryStorageProvider();
    MimeTypeMapper mimeTypeMapper = mock(MimeTypeMapper.class);
    doReturn("extension").when(mimeTypeMapper).getFileExtensionForMimeType(anyString());
    InputTransformer transformer = mock(InputTransformer.class);
    doReturn(metacard).when(transformer).transform(any(InputStream.class));
    MimeTypeToTransformerMapper mimeTypeToTransformerMapper = mock(MimeTypeToTransformerMapper.class);
    doReturn(Collections.singletonList(transformer)).when(mimeTypeToTransformerMapper).findMatches(any(Class.class), any(MimeType.class));
    frameworkProperties.setCatalogProviders(Collections.singletonList(catalogProvider));
    frameworkProperties.setStorageProviders(Collections.singletonList(storageProvider));
    frameworkProperties.setMimeTypeMapper(mimeTypeMapper);
    frameworkProperties.setMimeTypeToTransformerMapper(mimeTypeToTransformerMapper);
    OperationsSecuritySupport opsSecurity = new OperationsSecuritySupport();
    MetacardFactory metacardFactory = new MetacardFactory(frameworkProperties.getMimeTypeToTransformerMapper(), uuidGenerator);
    OperationsMetacardSupport opsMetacard = new OperationsMetacardSupport(frameworkProperties, metacardFactory);
    SourceOperations sourceOperations = new SourceOperations(frameworkProperties);
    sourceOperations.bind(catalogProvider);
    sourceOperations.bind(storageProvider);
    TransformOperations transformOperations = new TransformOperations(frameworkProperties);
    QueryOperations queryOperations = new QueryOperations(frameworkProperties, sourceOperations, opsSecurity, opsMetacard);
    OperationsCatalogStoreSupport opsCatStore = new OperationsCatalogStoreSupport(frameworkProperties, sourceOperations);
    OperationsStorageSupport opsStorage = new OperationsStorageSupport(sourceOperations, queryOperations);
    ResourceOperations resourceOperations = new ResourceOperations(frameworkProperties, queryOperations, opsSecurity);
    OperationsMetacardSupport opsMetacardSupport = new OperationsMetacardSupport(frameworkProperties, metacardFactory);
    // Need to set these for InputValidation to work
    System.setProperty("bad.files", "none");
    System.setProperty("bad.file.extensions", "none");
    System.setProperty("bad.mime.types", "none");
    CreateOperations createOperations = new CreateOperations(frameworkProperties, queryOperations, sourceOperations, opsSecurity, opsMetacardSupport, opsCatStore, opsStorage);
    framework = new CatalogFrameworkImpl(createOperations, null, null, queryOperations, resourceOperations, sourceOperations, transformOperations);
    framework.setId(NEW_SOURCE_ID);
    framework.setFanoutEnabled(true);
    framework.setFanoutTagBlacklist(Collections.singletonList("blacklisted"));
    ContentItem item = new ContentItemImpl(uuidGenerator.generateUuid(), ByteSource.empty(), "text/xml", "filename.xml", 0L, metacard);
    CreateStorageRequest request = new CreateStorageRequestImpl(Collections.singletonList(item), new HashMap<>());
    framework.create(request);
}