Examples with FileMeta org.molgenis.data.file.model.FileMeta used on opensource projects

Example 26 with FileMeta

use of org.molgenis.data.file.model.FileMeta in project molgenis by molgenis.

the class AmazonBucketIngester method ingest.

public FileMeta ingest(String jobExecutionID, String targetEntityTypeName, String bucket, String key, String extension, String accessKey, String secretKey, String region, boolean isExpression, Progress progress) {
    FileMeta fileMeta;
    try {
        progress.setProgressMax(3);
        progress.progress(0, "Connection to Amazon Bucket with accessKey '" + accessKey + "'");
        AmazonS3 client = amazonBucketClient.getClient(accessKey, secretKey, region);
        progress.progress(1, "downloading...");
        File file = amazonBucketClient.downloadFile(client, fileStore, jobExecutionID, bucket, key, extension, isExpression, targetEntityTypeName);
        if (targetEntityTypeName != null && ExcelUtils.isExcelFile(file.getName())) {
            if (ExcelUtils.getNumberOfSheets(file) == 1) {
                ExcelUtils.renameSheet(targetEntityTypeName, file, 0);
            } else {
                throw new MolgenisDataException("Amazon Bucket imports to a specified entityType are only possible with CSV files or Excel files with one sheet");
            }
        }
        progress.progress(2, "Importing...");
        ImportService importService = importServiceFactory.getImportService(file.getName());
        File renamed = new File(String.format("%s%s%s.%s", file.getParent(), File.separatorChar, targetEntityTypeName, extension));
        Files.copy(file.toPath(), renamed.toPath(), StandardCopyOption.REPLACE_EXISTING);
        RepositoryCollection repositoryCollection = fileRepositoryCollectionFactory.createFileRepositoryCollection(renamed);
        EntityImportReport report = importService.doImport(repositoryCollection, DatabaseAction.ADD_UPDATE_EXISTING, "base");
        progress.status("Download and import from Amazon Bucket done.");
        progress.progress(3, "Successfully imported " + report.getNrImportedEntitiesMap().keySet().toString() + " entities.");
        fileMeta = createFileMeta(jobExecutionID, file);
    } catch (Exception e) {
        throw new MolgenisDataException(e);
    }
    return fileMeta;
}

Example 27 with FileMeta

use of org.molgenis.data.file.model.FileMeta in project molgenis by molgenis.

the class FileIngester method createFileMeta.

private FileMeta createFileMeta(String jobExecutionID, File file) {
    FileMeta fileMeta = fileMetaFactory.create(jobExecutionID);
    fileMeta.setContentType("text/csv");
    fileMeta.setSize(file.length());
    fileMeta.setFilename(jobExecutionID + '/' + file.getName());
    fileMeta.setUrl(FileDownloadController.URI + '/' + jobExecutionID);
    return fileMeta;
}

Example 28 with FileMeta

use of org.molgenis.data.file.model.FileMeta in project molgenis by molgenis.

the class FileIngester method ingest.

/**
 * Imports a csv file defined in the fileIngest entity
 *
 * @see FileIngestJobExecutionMetaData
 */
public FileMeta ingest(String entityTypeId, String url, String loader, String jobExecutionID, Progress progress) {
    if (!"CSV".equals(loader)) {
        throw new FileIngestException("Unknown loader '" + loader + "'");
    }
    progress.setProgressMax(2);
    progress.progress(0, "Downloading url '" + url + "'");
    File file = fileStoreDownload.downloadFile(url, jobExecutionID, entityTypeId + ".csv");
    progress.progress(1, "Importing...");
    FileRepositoryCollection repoCollection = fileRepositoryCollectionFactory.createFileRepositoryCollection(file);
    ImportService importService = importServiceFactory.getImportService(file, repoCollection);
    EntityImportReport report = importService.doImport(repoCollection, ADD_UPDATE_EXISTING, PACKAGE_DEFAULT);
    progress.status("Ingestion of url '" + url + "' done.");
    Integer count = report.getNrImportedEntitiesMap().get(entityTypeId);
    count = count != null ? count : 0;
    progress.progress(2, "Successfully imported " + count + " " + entityTypeId + " entities.");
    FileMeta fileMeta = createFileMeta(jobExecutionID, file);
    FileIngestJobExecution fileIngestJobExecution = (FileIngestJobExecution) progress.getJobExecution();
    fileIngestJobExecution.setFile(fileMeta);
    dataService.add(FILE_META, fileMeta);
    return fileMeta;
}

Example 29 with FileMeta

use of org.molgenis.data.file.model.FileMeta in project molgenis by molgenis.

the class SavedScriptRunner method createFileMeta.

private FileMeta createFileMeta(String fileMetaId, File file) {
    FileMeta fileMeta = fileMetaFactory.create(fileMetaId);
    fileMeta.setContentType(guessContentTypeFromName(file.getName()));
    fileMeta.setSize(file.length());
    fileMeta.setFilename(file.getName());
    fileMeta.setUrl(format("{0}/{1}", "/files", fileMetaId));
    return fileMeta;
}

Example 30 with FileMeta

use of org.molgenis.data.file.model.FileMeta in project molgenis by molgenis.

the class PostgreSqlUtilsTest method setUpBeforeClass.

@BeforeClass
public static void setUpBeforeClass() throws ParseException {
    // create ref entities
    String attrRefStringIdName = "refStringId";
    Attribute attrRefStringId = mock(Attribute.class);
    when(attrRefStringId.getName()).thenReturn(attrRefStringIdName);
    when(attrRefStringId.getDataType()).thenReturn(STRING);
    EntityType refStringIdEntityType = mock(EntityType.class);
    when(refStringIdEntityType.getIdAttribute()).thenReturn(attrRefStringId);
    when(refStringIdEntityType.toString()).thenReturn("refStringId");
    String attrRefIntIdName = "refIntId";
    Attribute attrRefIntId = mock(Attribute.class);
    when(attrRefIntId.getName()).thenReturn(attrRefIntIdName);
    when(attrRefIntId.getDataType()).thenReturn(INT);
    EntityType refIntIdEntityType = mock(EntityType.class);
    when(refIntIdEntityType.getIdAttribute()).thenReturn(attrRefIntId);
    when(refIntIdEntityType.toString()).thenReturn("refIntId");
    String attrRefFileIdName = FileMetaMetaData.ID;
    Attribute attrRefFileId = mock(Attribute.class);
    when(attrRefFileId.getName()).thenReturn(attrRefFileIdName);
    when(attrRefFileId.getDataType()).thenReturn(STRING);
    FileMetaMetaData fileMetaMeta = mock(FileMetaMetaData.class);
    when(fileMetaMeta.getIdAttribute()).thenReturn(attrRefFileId);
    when(fileMetaMeta.toString()).thenReturn("fileMeta");
    // create attributes
    String attrBoolName = "attrBool";
    attrBool = createAttr(attrBoolName, BOOL);
    String attrBoolNillableName = "attrBoolNillable";
    attrBoolNillable = createAttr(attrBoolNillableName, BOOL);
    String attrCategoricalName = "attrCategorical";
    attrCategorical = createAttr(attrCategoricalName, CATEGORICAL, refStringIdEntityType);
    String attrCategoricalNillableName = "attrCategoricalNillable";
    attrCategoricalNillable = createAttr(attrCategoricalNillableName, CATEGORICAL, refStringIdEntityType);
    String attrCategoricalMrefName = "attrCategoricalMref";
    attrCategoricalMref = createAttr(attrCategoricalMrefName, CATEGORICAL_MREF, refStringIdEntityType);
    String attrCategoricalMrefNillableName = "attrCategoricalMrefNillable";
    attrCategoricalMrefNillable = createAttr(attrCategoricalMrefNillableName, CATEGORICAL_MREF, refStringIdEntityType);
    String attrDateName = "attrDate";
    attrDate = createAttr(attrDateName, DATE);
    String attrDateNillableName = "attrDateNillable";
    attrDateNillable = createAttr(attrDateNillableName, DATE);
    String attrDateTimeName = "attrDateTime";
    attrDateTime = createAttr(attrDateTimeName, DATE_TIME);
    String attrDateTimeNillableName = "attrDateTimeNillable";
    attrDateTimeNillable = createAttr(attrDateTimeNillableName, DATE_TIME);
    String attrDecimalName = "attrDecimal";
    attrDecimal = createAttr(attrDecimalName, DECIMAL);
    String attrDecimalNillableName = "attrDecimalNillable";
    attrDecimalNillable = createAttr(attrDecimalNillableName, DECIMAL);
    String attrEmailName = "attrEmail";
    attrEmail = createAttr(attrEmailName, EMAIL);
    String attrEmailNillableName = "attrEmailNillable";
    attrEmailNillable = createAttr(attrEmailNillableName, EMAIL);
    String attrEnumName = "attrEnum";
    attrEnum = createAttr(attrEnumName, ENUM);
    String attrEnumNillableName = "attrEnumNillable";
    attrEnumNillable = createAttr(attrEnumNillableName, ENUM);
    String attrFileName = "attrFile";
    attrFile = createAttr(attrFileName, FILE, fileMetaMeta);
    String attrFileNillableName = "attrFileNillable";
    attrFileNillable = createAttr(attrFileNillableName, FILE, fileMetaMeta);
    String attrHtmlName = "attrHtml";
    attrHtml = createAttr(attrHtmlName, HTML);
    String attrHtmlNillableName = "attrHtmlNillable";
    attrHtmlNillable = createAttr(attrHtmlNillableName, HTML);
    String attrHyperlinkName = "attrHyperlink";
    attrHyperlink = createAttr(attrHyperlinkName, HYPERLINK);
    String attrHyperlinkNillableName = "attrHyperlinkNillable";
    attrHyperlinkNillable = createAttr(attrHyperlinkNillableName, HYPERLINK);
    String attrIntName = "attrInt";
    attrInt = createAttr(attrIntName, INT);
    String attrIntNillableName = "attrIntNillable";
    attrIntNillable = createAttr(attrIntNillableName, INT);
    String attrLongName = "attrLong";
    attrLong = createAttr(attrLongName, LONG);
    String attrLongNillableName = "attrLongNillable";
    attrLongNillable = createAttr(attrLongNillableName, LONG);
    String attrMrefName = "attrMref";
    attrMref = createAttr(attrMrefName, MREF, refIntIdEntityType);
    String attrMrefNillableName = "attrMrefNillable";
    attrMrefNillable = createAttr(attrMrefNillableName, MREF, refIntIdEntityType);
    String attrScriptName = "attrScript";
    attrScript = createAttr(attrScriptName, SCRIPT);
    String attrScriptNillableName = "attrScriptNillable";
    attrScriptNillable = createAttr(attrScriptNillableName, SCRIPT);
    String attrStringName = "attrString";
    attrString = createAttr(attrStringName, STRING);
    String attrStringNillableName = "attrStringNillable";
    attrStringNillable = createAttr(attrStringNillableName, STRING);
    String attrTextName = "attrText";
    attrText = createAttr(attrTextName, TEXT);
    String attrTextNillableName = "attrTextNillable";
    attrTextNillable = createAttr(attrTextNillableName, TEXT);
    String attrXrefName = "attrXref";
    attrXref = createAttr(attrXrefName, XREF, refIntIdEntityType);
    String attrXrefNillableName = "attrXrefNillable";
    attrXrefNillable = createAttr(attrXrefNillableName, XREF, refIntIdEntityType);
    // create entity
    entity = mock(Entity.class);
    boolValue = true;
    when(entity.getBoolean(attrBoolName)).thenReturn(boolValue);
    when(entity.getBoolean(attrBoolNillableName)).thenReturn(null);
    categoricalValueId = "id0";
    categoricalValue = when(mock(Entity.class).getEntityType()).thenReturn(refStringIdEntityType).getMock();
    when(categoricalValue.getIdValue()).thenReturn(categoricalValueId);
    when(categoricalValue.toString()).thenReturn("categoricalValue");
    when(categoricalValue.getString(attrRefStringIdName)).thenReturn(categoricalValueId);
    when(entity.getEntity(attrCategoricalName)).thenReturn(categoricalValue);
    when(entity.getEntity(attrCategoricalNillableName)).thenReturn(null);
    categoricalMrefValueId0 = "id0";
    categoricalMrefValueId1 = "id1";
    categoricalMrefValue0 = when(mock(Entity.class).getEntityType()).thenReturn(refStringIdEntityType).getMock();
    when(categoricalMrefValue0.toString()).thenReturn("categoricalMrefValue0");
    when(categoricalMrefValue0.getIdValue()).thenReturn(categoricalMrefValueId0);
    Entity categoricalMrefValue1 = when(mock(Entity.class).getEntityType()).thenReturn(refStringIdEntityType).getMock();
    when(categoricalMrefValue1.toString()).thenReturn("categoricalMrefValue1");
    when(categoricalMrefValue1.getIdValue()).thenReturn(categoricalMrefValueId1);
    when(categoricalMrefValue0.getString(attrRefStringIdName)).thenReturn(categoricalMrefValueId0);
    when(categoricalMrefValue1.getString(attrRefStringIdName)).thenReturn(categoricalMrefValueId1);
    when(entity.getEntities(attrCategoricalMrefName)).thenReturn(asList(categoricalMrefValue0, categoricalMrefValue1));
    when(entity.getEntities(attrCategoricalMrefNillableName)).thenReturn(emptyList());
    dateValue = LocalDate.of(2012, Month.DECEMBER, 21);
    when(entity.getLocalDate(attrDateName)).thenReturn(dateValue);
    when(entity.getLocalDate(attrDateNillableName)).thenReturn(null);
    offsetDateTime = OffsetDateTime.parse("1985-08-12T06:12:13Z");
    dateTimeValue = offsetDateTime.toInstant();
    when(entity.getInstant(attrDateTimeName)).thenReturn(dateTimeValue);
    when(entity.getInstant(attrDateTimeNillableName)).thenReturn(null);
    decimalValue = 1.23;
    when(entity.getDouble(attrDecimalName)).thenReturn(decimalValue);
    when(entity.getDouble(attrDecimalNillableName)).thenReturn(null);
    emailValue = "my@mail.com";
    when(entity.getString(attrEmailName)).thenReturn(emailValue);
    when(entity.getString(attrEmailNillableName)).thenReturn(null);
    enumValue = "enum0";
    when(entity.getString(attrEnumName)).thenReturn(enumValue);
    when(entity.getString(attrEnumNillableName)).thenReturn(null);
    fileValueId = "id0";
    fileValue = when(mock(FileMeta.class).getEntityType()).thenReturn(fileMetaMeta).getMock();
    when(fileValue.toString()).thenReturn("fileValue");
    when(fileValue.getIdValue()).thenReturn(fileValueId);
    when(fileValue.getString(attrRefFileIdName)).thenReturn(fileValueId);
    when(entity.getEntity(attrFileName, FileMeta.class)).thenReturn(fileValue);
    when(entity.getEntity(attrFileNillableName, FileMeta.class)).thenReturn(null);
    htmlValue = "<p>text</p>";
    when(entity.getString(attrHtmlName)).thenReturn(htmlValue);
    when(entity.getString(attrHtmlNillableName)).thenReturn(null);
    hyperlinkValue = "<p>text</p>";
    when(entity.getString(attrHyperlinkName)).thenReturn(hyperlinkValue);
    when(entity.getString(attrHyperlinkNillableName)).thenReturn(null);
    intValue = 123;
    when(entity.getInt(attrIntName)).thenReturn(intValue);
    when(entity.getInt(attrIntNillableName)).thenReturn(null);
    longValue = 123456L;
    when(entity.getLong(attrLongName)).thenReturn(longValue);
    when(entity.getLong(attrLongNillableName)).thenReturn(null);
    mrefValueId0 = 3;
    mrefValueId1 = 4;
    mrefValue0 = when(mock(Entity.class).getEntityType()).thenReturn(refIntIdEntityType).getMock();
    when(mrefValue0.toString()).thenReturn("mrefValue0");
    when(mrefValue0.getIdValue()).thenReturn(mrefValueId0);
    Entity mrefValue1 = when(mock(Entity.class).getEntityType()).thenReturn(refIntIdEntityType).getMock();
    when(mrefValue1.toString()).thenReturn("mrefValue1");
    when(mrefValue1.getIdValue()).thenReturn(mrefValueId1);
    when(mrefValue0.getInt(attrRefIntIdName)).thenReturn(mrefValueId0);
    when(mrefValue1.getInt(attrRefIntIdName)).thenReturn(mrefValueId1);
    when(entity.getEntities(attrMrefName)).thenReturn(asList(mrefValue0, mrefValue1));
    when(entity.getEntities(attrMrefNillableName)).thenReturn(emptyList());
    scriptValue = "int x = 2";
    when(entity.getString(attrScriptName)).thenReturn(scriptValue);
    when(entity.getString(attrScriptNillableName)).thenReturn(null);
    stringValue = "str";
    when(entity.getString(attrStringName)).thenReturn(stringValue);
    when(entity.getString(attrStringNillableName)).thenReturn(null);
    textValue = "text";
    when(entity.getString(attrTextName)).thenReturn(textValue);
    when(entity.getString(attrTextNillableName)).thenReturn(null);
    xrefValueId = 3;
    xrefValue = when(mock(Entity.class).getEntityType()).thenReturn(refIntIdEntityType).getMock();
    when(xrefValue.toString()).thenReturn("xrefValue");
    when(xrefValue.getIdValue()).thenReturn(xrefValueId);
    when(xrefValue.getInt(attrRefIntIdName)).thenReturn(xrefValueId);
    when(entity.getEntity(attrXrefName)).thenReturn(xrefValue);
    when(entity.getEntity(attrXrefNillableName)).thenReturn(null);
    when(entity.toString()).thenReturn("entity");
}