Examples with ContentItem ddf.catalog.content.data.ContentItem used on opensource projects

Example 56 with ContentItem

use of ddf.catalog.content.data.ContentItem in project ddf by codice.

the class ImportCommand method executeWithSubject.

@Override
protected Object executeWithSubject() throws Exception {
    int metacards = 0;
    int content = 0;
    int derivedContent = 0;
    ZipValidator zipValidator = initZipValidator();
    File file = initImportFile(importFile);
    InputTransformer transformer = getServiceByFilter(InputTransformer.class, String.format("(%s=%s)", "id", DEFAULT_TRANSFORMER_ID)).orElseThrow(() -> new CatalogCommandRuntimeException("Could not get " + DEFAULT_TRANSFORMER_ID + " input transformer"));
    if (unsafe) {
        if (!force) {
            console.println("This will import data with no check to see if data is modified/corrupt. Do you wish to continue?");
            String input = getUserInputModifiable().toString();
            if (!input.matches("^[yY][eE]?[sS]?$")) {
                console.println("ABORTED IMPORT.");
                return null;
            }
        }
        SecurityLogger.audit("Skipping validation check of imported data. There are no " + "guarantees of integrity or authenticity of the imported data." + "File being imported: {}", importFile);
    } else {
        if (!zipValidator.validateZipFile(importFile)) {
            throw new CatalogCommandRuntimeException("Signature on zip file is not valid");
        }
    }
    SecurityLogger.audit("Called catalog:import command on the file: {}", importFile);
    console.println("Importing file");
    Instant start = Instant.now();
    try (InputStream fis = new FileInputStream(file);
        ZipInputStream zipInputStream = new ZipInputStream(fis)) {
        ZipEntry entry = zipInputStream.getNextEntry();
        while (entry != null) {
            String filename = entry.getName();
            if (filename.startsWith("META-INF")) {
                entry = zipInputStream.getNextEntry();
                continue;
            }
            String[] pathParts = filename.split("\\" + File.separator);
            if (pathParts.length < 5) {
                console.println("Entry is not valid! " + filename);
                entry = zipInputStream.getNextEntry();
                continue;
            }
            String id = pathParts[ID];
            String type = pathParts[TYPE];
            switch(type) {
                case "metacard":
                    {
                        metacards++;
                        String metacardName = pathParts[NAME];
                        Metacard metacard = null;
                        try {
                            metacard = transformer.transform(new UncloseableBufferedInputStreamWrapper(zipInputStream), id);
                        } catch (IOException | CatalogTransformerException e) {
                            LOGGER.debug("Could not transform metacard: {}", id);
                        }
                        catalogProvider.create(new CreateRequestImpl(metacard));
                        break;
                    }
                case "content":
                    {
                        content++;
                        String contentFilename = pathParts[NAME];
                        ContentItem contentItem = new ContentItemImpl(id, new ZipEntryByteSource(new UncloseableBufferedInputStreamWrapper(zipInputStream)), null, contentFilename, entry.getSize(), null);
                        CreateStorageRequestImpl createStorageRequest = new CreateStorageRequestImpl(Collections.singletonList(contentItem), id, new HashMap<>());
                        storageProvider.create(createStorageRequest);
                        storageProvider.commit(createStorageRequest);
                        break;
                    }
                case "derived":
                    {
                        derivedContent++;
                        String qualifier = pathParts[NAME];
                        String derivedContentName = pathParts[DERIVED_NAME];
                        ContentItem contentItem = new ContentItemImpl(id, qualifier, new ZipEntryByteSource(new UncloseableBufferedInputStreamWrapper(zipInputStream)), null, derivedContentName, entry.getSize(), null);
                        CreateStorageRequestImpl createStorageRequest = new CreateStorageRequestImpl(Collections.singletonList(contentItem), id, new HashMap<>());
                        storageProvider.create(createStorageRequest);
                        storageProvider.commit(createStorageRequest);
                        break;
                    }
                default:
                    {
                        LOGGER.debug("Cannot interpret type of {}", type);
                    }
            }
            entry = zipInputStream.getNextEntry();
        }
    } catch (Exception e) {
        printErrorMessage(String.format("Exception while importing metacards (%s)%nFor more information set the log level to INFO (log:set INFO org.codice.ddf.commands.catalog) ", e.getMessage()));
        LOGGER.info("Exception while importing metacards", e);
        throw e;
    }
    console.println("File imported successfully. Imported in: " + getFormattedDuration(start));
    console.println("Number of metacards imported: " + metacards);
    console.println("Number of content imported: " + content);
    console.println("Number of derived content imported: " + derivedContent);
    return null;
}

Example 57 with ContentItem

use of ddf.catalog.content.data.ContentItem in project ddf by codice.

the class IngestCommand method submitToStorageProvider.

private void submitToStorageProvider(List<Metacard> metacardList) {
    metacardList.stream().filter(metacard -> metacardFileMapping.containsKey(metacard.getId())).map(metacard -> {
        List<File> fileList = metacardFileMapping.get(metacard.getId());
        List<ContentItem> contentItemList = new ArrayList<>();
        ContentItem contentItem;
        for (File file : fileList) {
            ByteSource byteSource = com.google.common.io.Files.asByteSource(file);
            String fileName = file.getName().split("-")[1];
            String fragment = null;
            if (!file.getPath().contains(CONTENT + File.separator + metacard.getId())) {
                fragment = StringUtils.substringBetween(file.getPath(), CONTENT + File.separator, File.separator + metacard.getId());
            }
            contentItem = new ContentItemImpl(metacard.getId(), fragment, byteSource, metacard.getContentTypeName(), fileName, file.length(), metacard);
            contentItemList.add(contentItem);
        }
        return new CreateStorageRequestImpl(contentItemList, metacard.getId(), new HashMap<>());
    }).forEach(createStorageRequest -> {
        try {
            storageProvider.create(createStorageRequest);
            storageProvider.commit(createStorageRequest);
        } catch (StorageException e) {
            LOGGER.debug("Unable to create content for {}", createStorageRequest.getId(), e);
            try {
                storageProvider.rollback(createStorageRequest);
            } catch (StorageException e1) {
                LOGGER.debug("Unable to perform rollback on temporary content for {} ", createStorageRequest.getId(), e1);
            }
        }
    });
}

Example 58 with ContentItem

use of ddf.catalog.content.data.ContentItem in project ddf by codice.

the class OperationsMetacardSupport method generateMetacardAndContentItems.

void generateMetacardAndContentItems(List<ContentItem> incomingContentItems, Map<String, Metacard> metacardMap, List<ContentItem> contentItems, Map<String, Map<String, Path>> tmpContentPaths) throws IngestException {
    for (ContentItem contentItem : incomingContentItems) {
        try {
            Path tmpPath = null;
            long size;
            try (InputStream inputStream = contentItem.getInputStream()) {
                if (inputStream == null) {
                    throw new IngestException("Could not copy bytes of content message.  Message was NULL.");
                }
                String sanitizedFilename = InputValidation.sanitizeFilename(contentItem.getFilename());
                tmpPath = Files.createTempFile(FilenameUtils.getBaseName(sanitizedFilename), FilenameUtils.getExtension(sanitizedFilename));
                Files.copy(inputStream, tmpPath, StandardCopyOption.REPLACE_EXISTING);
                size = Files.size(tmpPath);
                final String key = contentItem.getId();
                Map<String, Path> pathAndQualifiers = tmpContentPaths.get(key);
                if (pathAndQualifiers == null) {
                    pathAndQualifiers = new HashMap<>();
                    pathAndQualifiers.put(contentItem.getQualifier(), tmpPath);
                    tmpContentPaths.put(key, pathAndQualifiers);
                } else {
                    pathAndQualifiers.put(contentItem.getQualifier(), tmpPath);
                }
            } catch (IOException e) {
                if (tmpPath != null) {
                    FileUtils.deleteQuietly(tmpPath.toFile());
                }
                throw new IngestException("Could not copy bytes of content message.", e);
            }
            String mimeTypeRaw = contentItem.getMimeTypeRawData();
            mimeTypeRaw = guessMimeType(mimeTypeRaw, contentItem.getFilename(), tmpPath);
            if (!InputValidation.checkForClientSideVulnerableMimeType(mimeTypeRaw)) {
                throw new IngestException("Unsupported mime type.");
            }
            String fileName = updateFileExtension(mimeTypeRaw, contentItem.getFilename());
            Metacard metacard = metacardFactory.generateMetacard(mimeTypeRaw, contentItem.getId(), fileName, tmpPath);
            metacardMap.put(metacard.getId(), metacard);
            ContentItem generatedContentItem = new ContentItemImpl(metacard.getId(), StringUtils.isNotEmpty(contentItem.getQualifier()) ? contentItem.getQualifier() : "", com.google.common.io.Files.asByteSource(tmpPath.toFile()), mimeTypeRaw, fileName, size, metacard);
            contentItems.add(generatedContentItem);
        } catch (Exception e) {
            tmpContentPaths.values().stream().flatMap(id -> id.values().stream()).forEach(path -> FileUtils.deleteQuietly(path.toFile()));
            tmpContentPaths.clear();
            throw new IngestException("Could not create metacard.", e);
        }
    }
}

Example 59 with ContentItem

use of ddf.catalog.content.data.ContentItem in project ddf by codice.

the class FileSystemStorageProviderTest method testRollback.

@Test(expected = StorageException.class)
public void testRollback() throws Exception {
    String id = UUID.randomUUID().toString().replaceAll("-", "");
    ByteSource byteSource = new ByteSource() {

        @Override
        public InputStream openStream() throws IOException {
            return IOUtils.toInputStream(TEST_INPUT_CONTENTS);
        }
    };
    Metacard metacard = mock(Metacard.class);
    when(metacard.getId()).thenReturn(id);
    ContentItem contentItem = new ContentItemImpl(id, byteSource, NITF_MIME_TYPE, TEST_INPUT_FILENAME, TEST_INPUT_CONTENTS.getBytes().length, metacard);
    CreateStorageRequest createRequest = new CreateStorageRequestImpl(Collections.singletonList(contentItem), null);
    CreateStorageResponse createStorageResponse = provider.create(createRequest);
    provider.rollback(createRequest);
    ReadStorageRequest readStorageRequest = new ReadStorageRequestImpl(new URI("content:" + id), null);
    ReadStorageResponse read = provider.read(readStorageRequest);
}

Example 60 with ContentItem

use of ddf.catalog.content.data.ContentItem in project ddf by codice.

the class FileSystemStorageProviderTest method submitAndVerifySuccessfulUpdateStorageRequest.

private void submitAndVerifySuccessfulUpdateStorageRequest(ContentItem... requestContentItems) throws Exception {
    final UpdateStorageRequest updateStorageRequest = new UpdateStorageRequestImpl(Arrays.asList(requestContentItems), null);
    final UpdateStorageResponse updateStorageResponse = provider.update(updateStorageRequest);
    final List<ContentItem> responseContentItems = updateStorageResponse.getUpdatedContentItems();
    assertThat("Expect number of ContentItems in UpdateStorageResponse", responseContentItems, hasSize(requestContentItems.length));
    for (final ContentItem responseContentItem : responseContentItems) {
        // assert file exists
        final URI uri = new URI(responseContentItem.getUri());
        final List<String> parts = provider.getContentFilePathParts(uri.getSchemeSpecificPart(), uri.getFragment());
        final String expectedFilePath = baseDir + File.separator + FileSystemStorageProvider.DEFAULT_CONTENT_REPOSITORY + File.separator + FileSystemStorageProvider.DEFAULT_CONTENT_STORE + File.separator + FileSystemStorageProvider.DEFAULT_TMP + File.separator + updateStorageRequest.getId() + File.separator + parts.get(0) + File.separator + parts.get(1) + File.separator + parts.get(2) + (StringUtils.isNotBlank(responseContentItem.getQualifier()) ? File.separator + responseContentItem.getQualifier() : "") + File.separator + responseContentItem.getFilename();
        assertThat("Expect file exists at " + expectedFilePath, Files.exists(Paths.get(expectedFilePath)));
        // assert metacard attributes set
        final ArgumentCaptor<Attribute> captor = ArgumentCaptor.forClass(Attribute.class);
        final Metacard metacard = responseContentItem.getMetacard();
        if (StringUtils.isBlank(responseContentItem.getQualifier())) {
            verify(metacard, times(2)).setAttribute(captor.capture());
            Attribute resourceUriAttribute = captor.getAllValues().get(0);
            assertThat(resourceUriAttribute.getName(), is(Metacard.RESOURCE_URI));
            assertThat(resourceUriAttribute.getValue(), is(uri.toString()));
            Attribute resourceSizeAttribute = captor.getAllValues().get(1);
            assertThat(resourceSizeAttribute.getName(), is(Metacard.RESOURCE_SIZE));
            assertThat(resourceSizeAttribute.getValue(), is(responseContentItem.getSize()));
        } else {
            verify(metacard, never()).setAttribute(any());
        }
    }
    provider.commit(updateStorageRequest);
    for (ContentItem responseContentItem : responseContentItems) {
        assertReadRequest(responseContentItem.getUri(), responseContentItem.getMimeType().toString());
    }
}