Examples with DataFile edu.harvard.iq.dataverse.DataFile used on opensource projects

Example 81 with DataFile

use of edu.harvard.iq.dataverse.DataFile in project dataverse by IQSS.

the class FileAccessIO method getAuxObjectAsPath.

@Override
public Path getAuxObjectAsPath(String auxItemTag) throws IOException {
    if (auxItemTag == null || "".equals(auxItemTag)) {
        throw new IOException("Null or invalid Auxiliary Object Tag.");
    }
    String datasetDirectory = getDatasetDirectory();
    if (dvObject.getStorageIdentifier() == null || "".equals(dvObject.getStorageIdentifier())) {
        throw new IOException("Data Access: No local storage identifier defined for this datafile.");
    }
    Path auxPath = null;
    if (dvObject instanceof DataFile) {
        auxPath = Paths.get(datasetDirectory, dvObject.getStorageIdentifier() + "." + auxItemTag);
    } else if (dvObject instanceof Dataset) {
        auxPath = Paths.get(datasetDirectory, auxItemTag);
    } else if (dvObject instanceof Dataverse) {
    } else {
        throw new IOException("Aux path could not be generated for " + auxItemTag);
    }
    if (auxPath == null) {
        throw new IOException("Invalid Path location for the auxiliary file " + dvObject.getStorageIdentifier() + "." + auxItemTag);
    }
    return auxPath;
}

Example 82 with DataFile

use of edu.harvard.iq.dataverse.DataFile in project dataverse by IQSS.

the class FileAccessIO method open.

@Override
public void open(DataAccessOption... options) throws IOException {
    DataFile dataFile;
    Dataset dataset;
    Dataverse dataverse = null;
    DataAccessRequest req = this.getRequest();
    if (isWriteAccessRequested(options)) {
        isWriteAccess = true;
        isReadAccess = false;
    } else {
        isWriteAccess = false;
        isReadAccess = true;
    }
    if (dvObject instanceof DataFile) {
        dataFile = this.getDataFile();
        if (req != null && req.getParameter("noVarHeader") != null) {
            this.setNoVarHeader(true);
        }
        if (dataFile.getStorageIdentifier() == null || "".equals(dataFile.getStorageIdentifier())) {
            throw new IOException("Data Access: No local storage identifier defined for this datafile.");
        }
        if (isReadAccess) {
            FileInputStream fin = openLocalFileAsInputStream();
            if (fin == null) {
                throw new IOException("Failed to open local file " + getStorageLocation());
            }
            this.setInputStream(fin);
            setChannel(fin.getChannel());
            this.setSize(getLocalFileSize());
            if (dataFile.getContentType() != null && dataFile.getContentType().equals("text/tab-separated-values") && dataFile.isTabularData() && dataFile.getDataTable() != null && (!this.noVarHeader())) {
                List<DataVariable> datavariables = dataFile.getDataTable().getDataVariables();
                String varHeaderLine = generateVariableHeader(datavariables);
                this.setVarHeader(varHeaderLine);
            }
        } else if (isWriteAccess) {
            // Creates a new directory as needed for a dataset.
            if (dataFile.getOwner().getFileSystemDirectory() != null && !Files.exists(dataFile.getOwner().getFileSystemDirectory())) {
                Files.createDirectories(dataFile.getOwner().getFileSystemDirectory());
            }
            FileOutputStream fout = openLocalFileAsOutputStream();
            if (fout == null) {
                throw new IOException("Failed to open local file " + getStorageLocation() + " for writing.");
            }
            this.setOutputStream(fout);
            setChannel(fout.getChannel());
        }
        this.setMimeType(dataFile.getContentType());
        try {
            this.setFileName(dataFile.getFileMetadata().getLabel());
        } catch (Exception ex) {
            this.setFileName("unknown");
        }
    } else if (dvObject instanceof Dataset) {
        // This case is for uploading a dataset related auxiliary file
        // e.g. image thumbnails/metadata exports
        // TODO: do we really need to do anything here? should we return the dataset directory?
        dataset = this.getDataset();
        if (isReadAccess) {
        // TODO: Not necessary for dataset as there is no files associated with this
        // FileInputStream fin = openLocalFileAsInputStream();
        // Path path= dataset.getFileSystemDirectory();
        // if (path == null) {
        // throw new IOException("Failed to locate Dataset"+dataset.getIdentifier());
        // }
        // 
        // this.setInputStream(fin);
        } else if (isWriteAccess) {
            // this checks whether a directory for a dataset exists
            if (dataset.getFileSystemDirectory() != null && !Files.exists(dataset.getFileSystemDirectory())) {
                Files.createDirectories(dataset.getFileSystemDirectory());
            }
            dataset.setStorageIdentifier("file://" + dataset.getAuthority() + dataset.getDoiSeparator() + dataset.getIdentifier());
        }
    } else if (dvObject instanceof Dataverse) {
        dataverse = this.getDataverse();
    } else {
        throw new IOException("Data Access: Invalid DvObject type");
    }
    // This "status" is a leftover from 3.6; we don't have a use for it
    // in 4.0 yet; and we may not need it at all.
    // -- L.A. 4.0.2
    this.setStatus(200);
}

Example 83 with DataFile

use of edu.harvard.iq.dataverse.DataFile in project dataverse by IQSS.

the class S3AccessIO method generateTemporaryS3Url.

public String generateTemporaryS3Url() throws IOException {
    // A. 1 hour by default seems like an OK number. Making it configurable seems like a good idea too. -- L.A.
    if (s3 == null) {
        throw new IOException("ERROR: s3 not initialised. ");
    }
    if (dvObject instanceof DataFile) {
        key = getMainFileKey();
        java.util.Date expiration = new java.util.Date();
        long msec = expiration.getTime();
        msec += 1000 * getUrlExpirationMinutes();
        expiration.setTime(msec);
        GeneratePresignedUrlRequest generatePresignedUrlRequest = new GeneratePresignedUrlRequest(bucketName, key);
        // Default.
        generatePresignedUrlRequest.setMethod(HttpMethod.GET);
        generatePresignedUrlRequest.setExpiration(expiration);
        ResponseHeaderOverrides responseHeaders = new ResponseHeaderOverrides();
        // responseHeaders.setContentDisposition("attachment; filename="+this.getDataFile().getDisplayName());
        // Encode the file name explicitly specifying the encoding as UTF-8:
        // (otherwise S3 may not like non-ASCII characters!)
        // Most browsers are happy with just "filename="+URLEncoder.encode(this.getDataFile().getDisplayName(), "UTF-8")
        // in the header. But Firefox appears to require that "UTF8" is
        // specified explicitly, as below:
        responseHeaders.setContentDisposition("attachment; filename*=UTF-8''" + URLEncoder.encode(this.getDataFile().getDisplayName(), "UTF-8"));
        // - without it, download will work, but Firefox will leave the special
        // characters in the file name encoded. For example, the file name
        // will look like "1976%E2%80%932016.txt" instead of "1976–2016.txt",
        // where the dash is the "long dash", represented by a 3-byte UTF8
        // character "\xE2\x80\x93"
        responseHeaders.setContentType(this.getDataFile().getContentType());
        generatePresignedUrlRequest.setResponseHeaders(responseHeaders);
        URL s = s3.generatePresignedUrl(generatePresignedUrlRequest);
        return s.toString();
    } else if (dvObject instanceof Dataset) {
        throw new IOException("Data Access: GenerateTemporaryS3Url: Invalid DvObject type : Dataset");
    } else if (dvObject instanceof Dataverse) {
        throw new IOException("Data Access: Invalid DvObject type : Dataverse");
    } else {
        throw new IOException("Data Access: Invalid DvObject type");
    }
}

Example 84 with DataFile

use of edu.harvard.iq.dataverse.DataFile in project dataverse by IQSS.

the class SwiftAccessIO method open.

@Override
public void open(DataAccessOption... options) throws IOException {
    DataAccessRequest req = this.getRequest();
    if (isWriteAccessRequested(options)) {
        isWriteAccess = true;
        isReadAccess = false;
    } else {
        isWriteAccess = false;
        isReadAccess = true;
    }
    if (dvObject instanceof DataFile) {
        DataFile dataFile = this.getDataFile();
        if (req != null && req.getParameter("noVarHeader") != null) {
            this.setNoVarHeader(true);
        }
        if (dataFile.getStorageIdentifier() == null || "".equals(dataFile.getStorageIdentifier())) {
            throw new IOException("Data Access: No local storage identifier defined for this datafile.");
        }
        if (isReadAccess) {
            InputStream fin = openSwiftFileAsInputStream();
            if (fin == null) {
                throw new IOException("Failed to open Swift file " + getStorageLocation());
            }
            this.setInputStream(fin);
            setChannel(Channels.newChannel(fin));
            if (dataFile.getContentType() != null && dataFile.getContentType().equals("text/tab-separated-values") && dataFile.isTabularData() && dataFile.getDataTable() != null && (!this.noVarHeader())) {
                List<DataVariable> datavariables = dataFile.getDataTable().getDataVariables();
                String varHeaderLine = generateVariableHeader(datavariables);
                this.setVarHeader(varHeaderLine);
            }
        } else if (isWriteAccess) {
            swiftFileObject = initializeSwiftFileObject(true);
        }
        this.setMimeType(dataFile.getContentType());
        try {
            this.setFileName(dataFile.getFileMetadata().getLabel());
        } catch (Exception ex) {
            this.setFileName("unknown");
        }
    } else if (dvObject instanceof Dataset) {
        // such as a dataset thumbnail or a metadata export
        if (isReadAccess) {
            // TODO: fix this
            InputStream fin = openSwiftFileAsInputStream();
            if (fin == null) {
                throw new IOException("Failed to open Swift file " + getStorageLocation());
            }
            this.setInputStream(fin);
        } else if (isWriteAccess) {
            swiftFileObject = initializeSwiftFileObject(true);
        }
    } else if (dvObject instanceof Dataverse) {
    } else {
        throw new IOException("Data Access: Invalid DvObject type");
    }
}

Example 85 with DataFile

use of edu.harvard.iq.dataverse.DataFile in project dataverse by IQSS.

the class DatasetUtil method getThumbnailCandidates.

public static List<DatasetThumbnail> getThumbnailCandidates(Dataset dataset, boolean considerDatasetLogoAsCandidate) {
    List<DatasetThumbnail> thumbnails = new ArrayList<>();
    if (dataset == null) {
        return thumbnails;
    }
    if (considerDatasetLogoAsCandidate) {
        // Path path = Paths.get(dataset.getFileSystemDirectory() + File.separator + datasetLogoThumbnail + thumb48addedByImageThumbConverter);
        // if (Files.exists(path)) {
        // logger.fine("Thumbnail created from dataset logo exists!");
        // File file = path.toFile();
        // try {
        // byte[] bytes = Files.readAllBytes(file.toPath());
        StorageIO<Dataset> dataAccess = null;
        try {
            dataAccess = DataAccess.getStorageIO(dataset);
        } catch (IOException ioex) {
        }
        InputStream in = null;
        try {
            if (dataAccess.getAuxFileAsInputStream(datasetLogoThumbnail + thumb48addedByImageThumbConverter) != null) {
                in = dataAccess.getAuxFileAsInputStream(datasetLogoThumbnail + thumb48addedByImageThumbConverter);
            }
        } catch (Exception ioex) {
        }
        if (in != null) {
            logger.fine("Thumbnail created from dataset logo exists!");
            try {
                byte[] bytes = IOUtils.toByteArray(in);
                String base64image = Base64.getEncoder().encodeToString(bytes);
                DatasetThumbnail datasetThumbnail = new DatasetThumbnail(FileUtil.DATA_URI_SCHEME + base64image, null);
                thumbnails.add(datasetThumbnail);
            } catch (IOException ex) {
                logger.warning("Unable to rescale image: " + ex);
            }
        } else {
            logger.fine("There is no thumbnail created from a dataset logo");
        }
    }
    for (FileMetadata fileMetadata : dataset.getLatestVersion().getFileMetadatas()) {
        DataFile dataFile = fileMetadata.getDataFile();
        if (dataFile != null && FileUtil.isThumbnailSupported(dataFile) && ImageThumbConverter.isThumbnailAvailable(dataFile) && !dataFile.isRestricted()) {
            String imageSourceBase64 = null;
            imageSourceBase64 = ImageThumbConverter.getImageThumbnailAsBase64(dataFile, ImageThumbConverter.DEFAULT_CARDIMAGE_SIZE);
            if (imageSourceBase64 != null) {
                DatasetThumbnail datasetThumbnail = new DatasetThumbnail(imageSourceBase64, dataFile);
                thumbnails.add(datasetThumbnail);
            }
        }
    }
    return thumbnails;
}