Examples with CreateStorageRequest ddf.catalog.content.operation.CreateStorageRequest used on opensource projects

Example 1 with CreateStorageRequest

use of ddf.catalog.content.operation.CreateStorageRequest in project ddf by codice.

the class FtpRequestHandler method onRenameStart.

@Override
public FtpletResult onRenameStart(FtpSession session, FtpRequest request) throws FtpException, IOException {
    FtpFile fromFtpFile = session.getRenameFrom();
    String toFilename = request.getArgument();
    if (isDotFile(fromFtpFile.getName())) {
        Optional<TemporaryFileBackedOutputStream> tfbosOpt = findTempFileInSession(session, fromFtpFile.getAbsolutePath());
        if (!tfbosOpt.isPresent()) {
            session.write(new DefaultFtpReply(FtpReply.REPLY_501_SYNTAX_ERROR_IN_PARAMETERS_OR_ARGUMENTS, "file not found: " + fromFtpFile.getAbsolutePath()));
            return FtpletResult.SKIP;
        }
        try (TemporaryFileBackedOutputStream tfbos = tfbosOpt.get()) {
            Subject shiroSubject = (Subject) session.getAttribute(SUBJECT);
            if (shiroSubject == null) {
                return FtpletResult.DISCONNECT;
            }
            CreateStorageRequest createRequest = getCreateStorageRequest(toFilename, tfbos);
            storeObject(shiroSubject, toFilename, createRequest);
        } finally {
            removeTempFileFromSession(session, fromFtpFile.getAbsolutePath());
        }
    }
    session.write(new DefaultFtpReply(FtpReply.REPLY_250_REQUESTED_FILE_ACTION_OKAY, "RNTO successful"));
    return FtpletResult.SKIP;
}

Example 2 with CreateStorageRequest

use of ddf.catalog.content.operation.CreateStorageRequest in project ddf by codice.

the class FtpRequestHandler method store.

private FtpletResult store(FtpSession session, FtpRequest request, boolean isStoreUnique) throws FtpException, IOException {
    LOGGER.debug("Beginning FTP ingest of {}", request.getArgument());
    Subject shiroSubject = (Subject) session.getAttribute(SUBJECT);
    if (shiroSubject == null) {
        return FtpletResult.DISCONNECT;
    }
    FtpFile ftpFile = null;
    String fileName = request.getArgument();
    try {
        ftpFile = session.getFileSystemView().getFile(fileName);
    } catch (FtpException e) {
        LOGGER.debug("Failed to retrieve file from FTP session");
    }
    String requestTypeString = isStoreUnique ? STOU_REQUEST : STOR_REQUEST;
    if (ftpFile == null) {
        LOGGER.debug("Sending FTP status code 501 to client - syntax errors in request parameters");
        session.write(new DefaultFtpReply(FtpReply.REPLY_501_SYNTAX_ERROR_IN_PARAMETERS_OR_ARGUMENTS, requestTypeString));
        throw new FtpException("File to be transferred from client did not exist");
    }
    DataConnectionFactory connFactory = session.getDataConnection();
    if (connFactory instanceof IODataConnectionFactory) {
        InetAddress address = ((IODataConnectionFactory) connFactory).getInetAddress();
        if (address == null) {
            session.write(new DefaultFtpReply(FtpReply.REPLY_503_BAD_SEQUENCE_OF_COMMANDS, "PORT or PASV must be issued first"));
            LOGGER.debug("Sending FTP status code 503 to client - PORT or PASV must be issued before STOR");
            throw new FtpException("FTP client address was null");
        }
    }
    if (!ftpFile.isWritable()) {
        session.write(new DefaultFtpReply(FtpReply.REPLY_550_REQUESTED_ACTION_NOT_TAKEN, "Insufficient permissions"));
        LOGGER.debug("Sending FTP status code 550 to client - insufficient permissions to write file.");
        throw new FtpException("Insufficient permissions to write file");
    }
    session.write(new DefaultFtpReply(FtpReply.REPLY_150_FILE_STATUS_OKAY, requestTypeString + " " + fileName));
    LOGGER.debug("Replying to client with code 150 - file status okay");
    if (isDotFile(request.getArgument())) {
        DataConnection dataConnection;
        try {
            dataConnection = connFactory.openConnection();
        } catch (Exception e) {
            throw new IOException("Error getting the output stream from FTP session", e);
        }
        dataConnection.transferFromClient(session, addTempFileToSession(session, ftpFile.getAbsolutePath(), new TemporaryFileBackedOutputStream()));
        if (isStoreUnique) {
            session.write(new DefaultFtpReply(FtpReply.REPLY_125_DATA_CONNECTION_ALREADY_OPEN, "Storing data with unique name: " + fileName));
        }
        session.write(new DefaultFtpReply(FtpReply.REPLY_226_CLOSING_DATA_CONNECTION, "Closing data connection"));
        LOGGER.debug("Sending FTP status code 226 to client - closing data connection");
    } else {
        try (TemporaryFileBackedOutputStream outputStream = new TemporaryFileBackedOutputStream()) {
            DataConnection dataConnection = connFactory.openConnection();
            dataConnection.transferFromClient(session, outputStream);
            CreateStorageRequest createRequest = getCreateStorageRequest(fileName, outputStream);
            List<Metacard> storedMetacards = storeObject(shiroSubject, fileName, createRequest);
            if (isStoreUnique && !storedMetacards.isEmpty()) {
                String ids = storedMetacards.stream().map(Metacard::getId).collect(Collectors.joining(","));
                session.write(new DefaultFtpReply(FtpReply.REPLY_125_DATA_CONNECTION_ALREADY_OPEN, "Storing data with unique name: " + ids));
            }
            session.write(new DefaultFtpReply(FtpReply.REPLY_226_CLOSING_DATA_CONNECTION, "Closing data connection"));
            LOGGER.debug("Sending FTP status code 226 to client - closing data connection");
        } catch (FtpException fe) {
            throw new FtpException("Failure to create metacard for file " + fileName, fe);
        } catch (Exception e) {
            throw new IOException("Error getting the output stream from FTP session", e);
        } finally {
            session.getDataConnection().closeDataConnection();
        }
    }
    return FtpletResult.SKIP;
}

Example 3 with CreateStorageRequest

use of ddf.catalog.content.operation.CreateStorageRequest in project ddf by codice.

the class CreateOperations method create.

public CreateResponse create(CreateStorageRequest streamCreateRequest, List<String> fanoutTagBlacklist) throws IngestException, SourceUnavailableException {
    Map<String, Metacard> metacardMap = new HashMap<>();
    List<ContentItem> contentItems = new ArrayList<>(streamCreateRequest.getContentItems().size());
    HashMap<String, Map<String, Path>> tmpContentPaths = new HashMap<>();
    CreateResponse createResponse;
    CreateStorageRequest createStorageRequest = null;
    CreateStorageResponse createStorageResponse;
    CreateRequest createRequest = null;
    Exception ingestError = null;
    String fileNames = "";
    streamCreateRequest = opsStorageSupport.prepareStorageRequest(streamCreateRequest, streamCreateRequest::getContentItems);
    if (!streamCreateRequest.getContentItems().isEmpty()) {
        List<String> fileList = streamCreateRequest.getContentItems().stream().map(ContentItem::getFilename).collect(Collectors.toList());
        fileNames = String.join(", ", fileList);
    }
    INGEST_LOGGER.info("Started ingesting resources with titles: {}.", fileNames);
    // Operation populates the metacardMap, contentItems, and tmpContentPaths
    opsMetacardSupport.generateMetacardAndContentItems(streamCreateRequest.getContentItems(), metacardMap, contentItems, tmpContentPaths);
    if (blockCreateMetacards(metacardMap.values(), fanoutTagBlacklist)) {
        String message = "Fanout proxy does not support create operations with blacklisted metacard tag";
        LOGGER.debug("{}. Tags blacklist: {}", message, fanoutTagBlacklist);
        throw new IngestException(message);
    }
    streamCreateRequest.getProperties().put(CONTENT_PATHS, tmpContentPaths);
    injectAttributes(metacardMap);
    setDefaultValues(metacardMap);
    streamCreateRequest = applyAttributeOverrides(streamCreateRequest, metacardMap);
    try {
        if (!contentItems.isEmpty()) {
            createStorageRequest = new CreateStorageRequestImpl(contentItems, streamCreateRequest.getId(), streamCreateRequest.getProperties());
            createStorageRequest = processPreCreateStoragePlugins(createStorageRequest);
            try {
                createStorageResponse = sourceOperations.getStorage().create(createStorageRequest);
                createStorageResponse.getProperties().put(CONTENT_PATHS, tmpContentPaths);
            } catch (StorageException e) {
                INGEST_LOGGER.debug("Could not store content items: {}.", fileNames, e);
                throw new IngestException("Could not store content items.", e);
            }
            createStorageResponse = processPostCreateStoragePlugins(createStorageResponse);
            populateMetacardMap(metacardMap, createStorageResponse);
        }
        createRequest = new CreateRequestImpl(new ArrayList<>(metacardMap.values()), Optional.ofNullable(createStorageRequest).map(StorageRequest::getProperties).orElseGet(HashMap::new));
        createResponse = doCreate(createRequest);
    } catch (IngestException e) {
        ingestError = e;
        rollbackStorage(createStorageRequest);
        throw e;
    } catch (IOException | RuntimeException e) {
        LOGGER.debug("Unhandled runtime exception or IOException during create", e);
        ingestError = e;
        rollbackStorage(createStorageRequest);
        throw new IngestException("Unable to store products for request: " + fileNames, e);
    } finally {
        opsStorageSupport.commitAndCleanup(createStorageRequest, tmpContentPaths);
        if (INGEST_LOGGER.isInfoEnabled()) {
            if (createRequest != null && ingestError != null) {
                INGEST_LOGGER.info("Error during ingesting resource: {}", fileNames, ingestError);
            }
            INGEST_LOGGER.info("Completed ingesting resource: {}.", fileNames);
        }
    }
    createResponse = doPostIngest(createResponse);
    return createResponse;
}

Example 4 with CreateStorageRequest

use of ddf.catalog.content.operation.CreateStorageRequest in project ddf by codice.

the class ContentProducerDataAccessObject method createContentItem.

public void createContentItem(FileSystemPersistenceProvider fileIdMap, ContentEndpoint endpoint, File ingestedFile, WatchEvent.Kind<Path> eventType, String mimeType, Map<String, Object> headers) throws SourceUnavailableException, IngestException {
    LOGGER.debug("Creating content item.");
    if (!eventType.equals(ENTRY_DELETE) && ingestedFile == null) {
        if (LOGGER.isDebugEnabled()) {
            LOGGER.debug("Ingested File was null with eventType [{}]. Doing nothing.", eventType.name());
        }
        return;
    }
    String refKey = (String) headers.get(Constants.STORE_REFERENCE_KEY);
    String safeKey = null;
    String id = null;
    // not null if the file lives outside the content store (external reference)
    if (refKey != null) {
        // guards against impermissible filesystem characters
        safeKey = DigestUtils.sha1Hex(refKey);
        if (fileIdMap.loadAllKeys().contains(safeKey)) {
            id = String.valueOf(fileIdMap.loadFromPersistence(safeKey));
        } else if (!ENTRY_CREATE.equals(eventType)) {
            LOGGER.warn("Unable to look up id for {}, not performing {}", refKey, eventType.name());
            return;
        }
    }
    if (ENTRY_CREATE.equals(eventType)) {
        CreateStorageRequest createRequest = new CreateStorageRequestImpl(Collections.singletonList(new ContentItemImpl(uuidGenerator.generateUuid(), Files.asByteSource(ingestedFile), mimeType, ingestedFile.getName(), ingestedFile.length(), null)), getProperties(headers));
        CatalogFramework catalogFramework = endpoint.getComponent().getCatalogFramework();
        waitForAvailableSource(catalogFramework);
        CreateResponse createResponse = catalogFramework.create(createRequest);
        if (createResponse != null) {
            List<Metacard> createdMetacards = createResponse.getCreatedMetacards();
            if (safeKey != null) {
                fileIdMap.store(safeKey, createdMetacards.get(0).getId());
            }
            logIds(createdMetacards, "created");
        }
    } else if (ENTRY_MODIFY.equals(eventType)) {
        UpdateStorageRequest updateRequest = new UpdateStorageRequestImpl(Collections.singletonList(new ContentItemImpl(id, Files.asByteSource(ingestedFile), mimeType, ingestedFile.getName(), 0, null)), getProperties(headers));
        UpdateResponse updateResponse = endpoint.getComponent().getCatalogFramework().update(updateRequest);
        if (updateResponse != null) {
            List<Update> updatedMetacards = updateResponse.getUpdatedMetacards();
            logIds(updatedMetacards.stream().map(Update::getNewMetacard).collect(Collectors.toList()), "updated");
        }
    } else if (ENTRY_DELETE.equals(eventType)) {
        DeleteRequest deleteRequest = new DeleteRequestImpl(id);
        DeleteResponse deleteResponse = endpoint.getComponent().getCatalogFramework().delete(deleteRequest);
        if (deleteResponse != null) {
            List<Metacard> deletedMetacards = deleteResponse.getDeletedMetacards();
            if (safeKey != null) {
                fileIdMap.delete(safeKey);
            }
            logIds(deletedMetacards, "deleted");
        }
    }
}

Example 5 with CreateStorageRequest

use of ddf.catalog.content.operation.CreateStorageRequest in project ddf by codice.

the class FileSystemStorageProviderTest method testCreateWithQualifierAndOneInvalidItem.

@Test
public void testCreateWithQualifierAndOneInvalidItem() throws Exception {
    String uuid = UUID.randomUUID().toString().replaceAll("-", "");
    ByteSource byteSource = new ByteSource() {

        @Override
        public InputStream openStream() throws IOException {
            return IOUtils.toInputStream("This data is my data, this data is your data.", StandardCharsets.UTF_8);
        }
    };
    ContentItem contentItem = new ContentItemImpl(uuid, null, byteSource, "application/text", "datadatadata", byteSource.size(), mock(Metacard.class));
    String invalidUuid = "wow-this-isn't-a-valid-uuid-right?!@#%";
    ContentItem badContentItem = new ContentItemImpl(invalidUuid, null, byteSource, "application/text", "datadatadata", byteSource.size(), mock(Metacard.class));
    CreateStorageRequest createRequest = new CreateStorageRequestImpl(Lists.newArrayList(contentItem, badContentItem), null);
    CreateStorageResponse createResponse = provider.create(createRequest);
    assertThat(createResponse.getCreatedContentItems().size(), is(1));
    assertThat(createResponse.getCreatedContentItems().get(0).getId(), is(uuid));
    ContentItem updateContentItem = new ContentItemImpl(invalidUuid, null, byteSource, "application/text", "datadatadata", byteSource.size(), mock(Metacard.class));
    UpdateStorageRequest updateRequest = new UpdateStorageRequestImpl(Lists.newArrayList(updateContentItem), null);
    UpdateStorageResponse updateResponse = provider.update(updateRequest);
    assertThat(updateResponse.getUpdatedContentItems().size(), is(0));
}