Examples with DataCollection org.molgenis.oneclickimporter.model.DataCollection used on opensource projects

Example 6 with DataCollection

use of org.molgenis.oneclickimporter.model.DataCollection in project molgenis by molgenis.

the class OneClickImporterServiceTest method testBuildDataSheetWithComplexFile.

@Test
public void testBuildDataSheetWithComplexFile() throws IOException, InvalidFormatException, URISyntaxException, EmptySheetException {
    List<Sheet> sheets = loadSheetFromFile(OneClickImporterServiceTest.class, "/complex-valid.xlsx");
    DataCollection actual = oneClickImporterService.buildDataCollectionsFromExcel(sheets).get(0);
    Column c1 = Column.create("first name", 0, newArrayList("Mark", "Fleur", "Dennis", "Bart", "Sido", "Mariska", "Tommy", "Connor", "Piet", "Jan"));
    Column c2 = Column.create("last name", 1, newArrayList("de Haan", "Kelpin", "Hendriksen", "Charbon", "Haakma", "Slofstra", "de Boer", "Stroomberg", "Klaassen", null));
    Column c3 = Column.create("full name", 2, newArrayList("Mark de Haan", "Fleur Kelpin", "Dennis Hendriksen", "Bart Charbon", "Sido Haakma", "Mariska Slofstra", "Tommy de Boer", "Connor Stroomberg", "Piet Klaassen", null));
    Column c4 = Column.create("UMCG employee", 3, newArrayList(true, true, true, true, true, true, true, true, false, false));
    Column c5 = Column.create("Age", 4, newArrayList(26.0, null, null, null, null, 22.0, 27.0, null, 53.0, 32.0));
    DataCollection expected = DataCollection.create("Sheet1", newArrayList(c1, c2, c3, c4, c5));
    assertEquals(actual, expected);
}

Example 7 with DataCollection

use of org.molgenis.oneclickimporter.model.DataCollection in project molgenis by molgenis.

the class OneClickImportJob method getEntityType.

@Transactional
public List<EntityType> getEntityType(Progress progress, String filename) throws UnknownFileTypeException, IOException, InvalidFormatException, EmptySheetException {
    File file = fileStore.getFile(filename);
    String fileExtension = findExtensionFromPossibilities(filename, newHashSet("csv", "xlsx", "zip", "xls"));
    progress.status("Preparing import");
    List<DataCollection> dataCollections = newArrayList();
    if (fileExtension == null) {
        throw new UnknownFileTypeException(String.format("File [%s] does not have a valid extension, supported: [csv, xlsx, zip, xls]", filename));
    } else if (fileExtension.equals("xls") || fileExtension.equals("xlsx")) {
        List<Sheet> sheets = excelService.buildExcelSheetsFromFile(file);
        dataCollections.addAll(oneClickImporterService.buildDataCollectionsFromExcel(sheets));
    } else if (fileExtension.equals("csv")) {
        List<String[]> lines = csvService.buildLinesFromFile(file);
        dataCollections.add(oneClickImporterService.buildDataCollectionFromCsv(oneClickImporterNamingService.createValidIdFromFileName(filename), lines));
    } else if (fileExtension.equals("zip")) {
        List<File> filesInZip = unzip(file);
        for (File fileInZip : filesInZip) {
            String fileInZipExtension = findExtensionFromPossibilities(fileInZip.getName(), newHashSet("csv"));
            if (fileInZipExtension != null) {
                List<String[]> lines = csvService.buildLinesFromFile(fileInZip);
                dataCollections.add(oneClickImporterService.buildDataCollectionFromCsv(oneClickImporterNamingService.createValidIdFromFileName(fileInZip.getName()), lines));
            } else {
                throw new UnknownFileTypeException("Zip file contains files which are not of type CSV");
            }
        }
    }
    List<EntityType> entityTypes = newArrayList();
    String packageName = oneClickImporterNamingService.createValidIdFromFileName(filename);
    dataCollections.forEach(dataCollection -> {
        progress.status("Importing [" + dataCollection.getName() + "] into package [" + packageName + "]");
        entityTypes.add(entityService.createEntityType(dataCollection, packageName));
    });
    return entityTypes;
}

Example 8 with DataCollection

use of org.molgenis.oneclickimporter.model.DataCollection in project molgenis by molgenis.

the class EntityServiceImplTest method testCreateEntity.

@Test
public void testCreateEntity() throws Exception {
    String tableName = "super-powers";
    List<Object> userNames = Arrays.asList("Mark", "Mariska", "Bart");
    List<Object> superPowers = Arrays.asList("Arrow functions", "Cookies", "Knots");
    List<Column> columns = Arrays.asList(Column.create("user name", 0, userNames), Column.create("super power", 1, superPowers));
    DataCollection dataCollection = DataCollection.create(tableName, columns);
    // mock auto id
    String generatedId = "id_0";
    when(idGenerator.generateId()).thenReturn(generatedId);
    // mock attributes
    Attribute idAttr = mock(Attribute.class);
    when(idAttr.setName(anyString())).thenReturn(idAttr);
    when(idAttr.setVisible(anyBoolean())).thenReturn(idAttr);
    when(idAttr.setAuto(anyBoolean())).thenReturn(idAttr);
    when(idAttr.setIdAttribute(anyBoolean())).thenReturn(idAttr);
    Attribute nameAttr = mock(Attribute.class);
    when(nameAttr.getDataType()).thenReturn(STRING);
    Attribute powerAttr = mock(Attribute.class);
    when(powerAttr.getDataType()).thenReturn(STRING);
    when(attributeFactory.create()).thenReturn(idAttr, nameAttr, powerAttr);
    // mock table
    EntityType table = mock(EntityType.class);
    when(entityTypeFactory.create()).thenReturn(table);
    when(table.getId()).thenReturn(generatedId);
    when(table.getAttribute("user_name")).thenReturn(nameAttr);
    when(table.getAttribute("super_power")).thenReturn(powerAttr);
    // mock package
    Package package_ = mock(Package.class);
    when(metaDataService.getPackage("package_")).thenReturn(package_);
    when(dataService.getMeta()).thenReturn(metaDataService);
    // mock rows
    Entity row1 = mock(Entity.class);
    when(row1.getEntityType()).thenReturn(table);
    Entity row2 = mock(Entity.class);
    when(row2.getEntityType()).thenReturn(table);
    Entity row3 = mock(Entity.class);
    when(row3.getEntityType()).thenReturn(table);
    when(entityManager.create(table, NO_POPULATE)).thenReturn(row1, row2, row3);
    when(oneClickImporterNamingService.asValidColumnName("user name")).thenReturn("user_name");
    when(oneClickImporterNamingService.asValidColumnName("super power")).thenReturn("super_power");
    when(oneClickImporterNamingService.getLabelWithPostFix("super-powers")).thenReturn("super-powers");
    when(attributeTypeService.guessAttributeType(any())).thenReturn(STRING);
    entityService = new EntityServiceImpl(entityTypeFactory, attributeFactory, idGenerator, dataService, metaDataService, entityManager, attributeTypeService, oneClickImporterService, oneClickImporterNamingService, packageFactory, permissionSystemService);
    EntityType entityType = entityService.createEntityType(dataCollection, "package_");
    assertEquals(entityType.getId(), generatedId);
    verify(table).setPackage(package_);
    verify(table).setId(generatedId);
    verify(table).setLabel(tableName);
    verify(permissionSystemService).giveUserWriteMetaPermissions(table);
}

Example 9 with DataCollection

use of org.molgenis.oneclickimporter.model.DataCollection in project molgenis by molgenis.

the class OneClickImportJobTest method testGetEntityTypeWithCsv.

@Test
public void testGetEntityTypeWithCsv() throws UnknownFileTypeException, InvalidFormatException, IOException, URISyntaxException, EmptySheetException {
    Progress progress = mock(Progress.class);
    String filename = "simple-valid.csv";
    when(oneClickImporterNamingService.createValidIdFromFileName(filename)).thenReturn("simple_valid");
    File file = loadFile(OneClickImportJobTest.class, "/" + filename);
    when(fileStore.getFile(filename)).thenReturn(file);
    List<String[]> lines = new ArrayList<>();
    lines.add(new String[] { "name,age", "piet,25" });
    when(csvService.buildLinesFromFile(file)).thenReturn(lines);
    DataCollection dataCollection = mock(DataCollection.class);
    when(dataCollection.getName()).thenReturn("file_1");
    when(oneClickImporterService.buildDataCollectionFromCsv("simple_valid", lines)).thenReturn(dataCollection);
    EntityType entityType = mock(EntityType.class);
    when(entityService.createEntityType(dataCollection, "simple_valid")).thenReturn(entityType);
    oneClickImporterJob = new OneClickImportJob(excelService, csvService, oneClickImporterService, oneClickImporterNamingService, entityService, fileStore);
    oneClickImporterJob.getEntityType(progress, filename);
    verify(progress).status("Preparing import");
    verify(csvService).buildLinesFromFile(file);
    verify(oneClickImporterService).buildDataCollectionFromCsv("simple_valid", lines);
    verify(progress).status("Importing [file_1] into package [simple_valid]");
    verify(entityService).createEntityType(dataCollection, "simple_valid");
}

Example 10 with DataCollection

use of org.molgenis.oneclickimporter.model.DataCollection in project molgenis by molgenis.

the class OneClickImporterServiceTest method testBuildDataCollectionWithSimpleValidExcelFile.

@Test
public void testBuildDataCollectionWithSimpleValidExcelFile() throws IOException, InvalidFormatException, URISyntaxException, EmptySheetException {
    List<Sheet> sheets = loadSheetFromFile(OneClickImporterServiceTest.class, "/simple-valid.xlsx");
    DataCollection actual = oneClickImporterService.buildDataCollectionsFromExcel(sheets).get(0);
    Column c1 = Column.create("name", 0, newArrayList("Mark", "Connor", "Fleur", "Dennis"));
    Column c2 = Column.create("superpower", 1, newArrayList("arrow functions", "Oldschool syntax", "Lambda Magician", "Root access"));
    DataCollection expected = DataCollection.create("Sheet1", newArrayList(c1, c2));
    assertEquals(actual, expected);
}