Examples with DataFile edu.harvard.iq.dataverse.DataFile used on opensource projects

Example 96 with DataFile

use of edu.harvard.iq.dataverse.DataFile in project dataverse by IQSS.

the class IngestUtilTest method testCheckForDuplicateFileNamesNoDirectories.

@Test
public /**
 * Test adding duplicate file name labels to a dataset version with no
 * subdirectories.
 */
void testCheckForDuplicateFileNamesNoDirectories() throws Exception {
    SimpleDateFormat dateFmt = new SimpleDateFormat("yyyyMMdd");
    // create dataset
    Dataset dataset = makeDataset();
    // create dataset version
    DatasetVersion datasetVersion = dataset.getEditVersion();
    datasetVersion.setCreateTime(dateFmt.parse("20001012"));
    datasetVersion.setLastUpdateTime(datasetVersion.getLastUpdateTime());
    datasetVersion.setId(MocksFactory.nextId());
    datasetVersion.setReleaseTime(dateFmt.parse("20010101"));
    datasetVersion.setVersionState(DatasetVersion.VersionState.RELEASED);
    datasetVersion.setMinorVersionNumber(0L);
    datasetVersion.setVersionNumber(1L);
    datasetVersion.setFileMetadatas(new ArrayList<>());
    // create datafiles
    List<DataFile> dataFileList = new ArrayList<>();
    DataFile datafile1 = new DataFile("application/octet-stream");
    datafile1.setStorageIdentifier("datafile1.txt");
    datafile1.setFilesize(200);
    datafile1.setModificationTime(new Timestamp(new Date().getTime()));
    datafile1.setCreateDate(new Timestamp(new Date().getTime()));
    datafile1.setPermissionModificationTime(new Timestamp(new Date().getTime()));
    datafile1.setOwner(dataset);
    datafile1.setIngestDone();
    datafile1.setChecksumType(DataFile.ChecksumType.SHA1);
    datafile1.setChecksumValue("Unknown");
    // set metadata and add verson
    FileMetadata fmd1 = new FileMetadata();
    fmd1.setId(1L);
    fmd1.setLabel("datafile1.txt");
    fmd1.setDataFile(datafile1);
    datafile1.getFileMetadatas().add(fmd1);
    datasetVersion.getFileMetadatas().add(fmd1);
    fmd1.setDatasetVersion(datasetVersion);
    dataFileList.add(datafile1);
    DataFile datafile2 = new DataFile("application/octet-stream");
    datafile2.setStorageIdentifier("datafile2.txt");
    datafile2.setFilesize(200);
    datafile2.setModificationTime(new Timestamp(new Date().getTime()));
    datafile2.setCreateDate(new Timestamp(new Date().getTime()));
    datafile2.setPermissionModificationTime(new Timestamp(new Date().getTime()));
    datafile2.setOwner(dataset);
    datafile2.setIngestDone();
    datafile2.setChecksumType(DataFile.ChecksumType.SHA1);
    datafile2.setChecksumValue("Unknown");
    // set metadata and add version
    FileMetadata fmd2 = new FileMetadata();
    fmd2.setId(2L);
    fmd2.setLabel("datafile2.txt");
    fmd2.setDataFile(datafile2);
    datafile2.getFileMetadatas().add(fmd2);
    datasetVersion.getFileMetadatas().add(fmd2);
    fmd2.setDatasetVersion(datasetVersion);
    dataFileList.add(datafile2);
    IngestUtil.checkForDuplicateFileNamesFinal(datasetVersion, dataFileList);
    boolean file1NameAltered = false;
    boolean file2NameAltered = false;
    for (DataFile df : dataFileList) {
        if (df.getFileMetadata().getLabel().equals("datafile1-1.txt")) {
            file1NameAltered = true;
        }
        if (df.getFileMetadata().getLabel().equals("datafile2-1.txt")) {
            file2NameAltered = true;
        }
    }
    // check filenames are unique and unaltered
    assertEquals(file1NameAltered, true);
    assertEquals(file2NameAltered, true);
    // try to add data files with "-1" duplicates and see if it gets incremented to "-2"
    IngestUtil.checkForDuplicateFileNamesFinal(datasetVersion, dataFileList);
    for (DataFile df : dataFileList) {
        if (df.getFileMetadata().getLabel().equals("datafile1-2.txt")) {
            file1NameAltered = true;
        }
        if (df.getFileMetadata().getLabel().equals("datafile2-2.txt")) {
            file2NameAltered = true;
        }
    }
    // check filenames are unique and unaltered
    assertEquals(file1NameAltered, true);
    assertEquals(file2NameAltered, true);
}

Example 97 with DataFile

use of edu.harvard.iq.dataverse.DataFile in project dataverse by IQSS.

the class MocksFactory method makeDataFile.

public static DataFile makeDataFile() {
    DataFile retVal = new DataFile();
    retVal.setId(nextId());
    retVal.setContentType("application/unitTests");
    retVal.setCreateDate(new Timestamp(System.currentTimeMillis()));
    addFileMetadata(retVal);
    retVal.setModificationTime(retVal.getCreateDate());
    return retVal;
}

Example 98 with DataFile

use of edu.harvard.iq.dataverse.DataFile in project dataverse by IQSS.

the class TabularSubsetGenerator method subsetObjectVector.

public Object[] subsetObjectVector(DataFile dataFile, int column, int columntype) throws IOException {
    if (!dataFile.isTabularData()) {
        throw new IOException("DataFile is not tabular data.");
    }
    int varcount = dataFile.getDataTable().getVarQuantity().intValue();
    int casecount = dataFile.getDataTable().getCaseQuantity().intValue();
    if (column >= varcount) {
        throw new IOException("Column " + column + " is out of bounds.");
    }
    StorageIO<DataFile> dataAccess = dataFile.getStorageIO();
    if (!dataAccess.isLocalFile()) {
        throw new IOException("Subsetting is supported on local files only!");
    }
    // File tabfile = datafile.getFileSystemLocation().toFile();
    File tabfile = dataAccess.getFileSystemPath().toFile();
    if (columntype == COLUMN_TYPE_STRING) {
        String filename = dataFile.getFileMetadata().getLabel();
        if (filename != null) {
            filename = filename.replaceFirst("^_", "");
            Integer fnumvalue = null;
            try {
                fnumvalue = new Integer(filename);
            } catch (Exception ex) {
                fnumvalue = null;
            }
            if (fnumvalue != null) {
                // if ((fnumvalue.intValue() < 112497)) { // && (fnumvalue.intValue() > 60015)) {
                if ((fnumvalue.intValue() < 111931)) {
                    // && (fnumvalue.intValue() > 60015)) {
                    if (!(fnumvalue.intValue() == 60007 || fnumvalue.intValue() == 59997 || fnumvalue.intValue() == 60015 || fnumvalue.intValue() == 59948 || fnumvalue.intValue() == 60012 || fnumvalue.intValue() == 52585 || fnumvalue.intValue() == 60005 || fnumvalue.intValue() == 60002 || fnumvalue.intValue() == 59954 || fnumvalue.intValue() == 60008 || fnumvalue.intValue() == 54972 || fnumvalue.intValue() == 55010 || fnumvalue.intValue() == 54996 || fnumvalue.intValue() == 53527 || fnumvalue.intValue() == 53546 || fnumvalue.intValue() == 55002 || fnumvalue.intValue() == 55006 || fnumvalue.intValue() == 54998 || fnumvalue.intValue() == 52552 || // || fnumvalue.intValue() == 101826 // temporary - tricky file with accents and v. 16...
                    fnumvalue.intValue() == // another SAV file, with long strings...
                    54618 || // [same]
                    fnumvalue.intValue() == 54619 || fnumvalue.intValue() == 57983 || fnumvalue.intValue() == 58262 || fnumvalue.intValue() == 58288 || fnumvalue.intValue() == 58656 || fnumvalue.intValue() == 59144)) {
                        dbgLog.info("\"Old\" file name detected; using \"compatibility mode\" for a character vector subset;");
                        return subsetObjectVector(tabfile, column, varcount, casecount, columntype, true);
                    }
                }
            }
        }
    }
    return subsetObjectVector(tabfile, column, varcount, casecount, columntype);
}

Example 99 with DataFile

use of edu.harvard.iq.dataverse.DataFile in project dataverse by IQSS.

the class AddReplaceFileHelper method step_080_run_update_dataset_command_for_replace.

private boolean step_080_run_update_dataset_command_for_replace() {
    if (!isFileReplaceOperation()) {
        // Shouldn't happen!
        this.addErrorSevere(getBundleErr("only_replace_operation") + " (step_080_run_update_dataset_command_for_replace)");
        return false;
    }
    if (this.hasError()) {
        return false;
    }
    // -----------------------------------------------------------
    if (!step_085_auto_remove_filemetadata_to_replace_from_working_version()) {
        return false;
    }
    /*
            Check the root file id on fileToReplace, updating it if necessary
        */
    if (fileToReplace.getRootDataFileId().equals(DataFile.ROOT_DATAFILE_ID_DEFAULT)) {
        fileToReplace.setRootDataFileId(fileToReplace.getId());
        fileToReplace = fileService.save(fileToReplace);
    }
    /*
            Go through the final file list, settting the rootFileId and previousFileId
        */
    for (DataFile df : finalFileList) {
        df.setPreviousDataFileId(fileToReplace.getId());
        df.setRootDataFileId(fileToReplace.getRootDataFileId());
    }
    // 
    return step_070_run_update_dataset_command();
}

Example 100 with DataFile

use of edu.harvard.iq.dataverse.DataFile in project dataverse by IQSS.

the class AddReplaceFileHelper method step_005_loadFileToReplaceById.

/**
 * Optional: old file to replace
 *
 * @param oldFile
 * @return
 */
private boolean step_005_loadFileToReplaceById(Long dataFileId) {
    if (this.hasError()) {
        return false;
    }
    // 
    if (dataFileId == null) {
        this.addErrorSevere(getBundleErr("existing_file_to_replace_id_is_null"));
        return false;
    }
    // Does the file exist?
    // 
    DataFile existingFile = fileService.find(dataFileId);
    if (existingFile == null) {
        this.addError(BundleUtil.getStringFromBundle("file.addreplace.error.existing_file_to_replace_not_found_by_id", Collections.singletonList(dataFileId.toString())));
        return false;
    }
    // 
    if (!step_015_auto_check_permissions(existingFile.getOwner())) {
        return false;
    }
    ;
    // 
    if (!existingFile.isReleased()) {
        addError(getBundleErr("unpublished_file_cannot_be_replaced"));
        return false;
    }
    // 
    if (!step_007_auto_isReplacementInLatestVersion(existingFile)) {
        return false;
    }
    fileToReplace = existingFile;
    return true;
}