Examples with EntityImportReport org.molgenis.data.importer.EntityImportReport used on opensource projects

Example 11 with EntityImportReport

use of org.molgenis.data.importer.EntityImportReport in project molgenis by molgenis.

the class OntologyImportServiceIT method testDoImportObo.

private void testDoImportObo() {
    String fileName = "ontology-small.obo.zip";
    File file = getFile("/obo/" + fileName);
    FileRepositoryCollection repoCollection = fileRepositoryCollectionFactory.createFileRepositoryCollection(file);
    ImportService importService = importServiceFactory.getImportService(file, repoCollection);
    EntityImportReport importReport = importService.doImport(repoCollection, ADD, PACKAGE_DEFAULT);
    validateImportReport(importReport, ImmutableMap.of("sys_ont_OntologyTermDynamicAnnotation", 0, "sys_ont_OntologyTermSynonym", 5, "sys_ont_OntologyTermNodePath", 5, "sys_ont_Ontology", 1, "sys_ont_OntologyTerm", 5), emptySet());
    // Verify the import as system as we need write permissions on sys tables to carry out the verification
    runAsSystem(this::verifyOboAsSystem);
}

Example 12 with EntityImportReport

use of org.molgenis.data.importer.EntityImportReport in project molgenis by molgenis.

the class OntologyImportServiceTest method testDoImport.

@SuppressWarnings("unchecked")
@Test
public void testDoImport() throws Exception {
    String entityTypeId0 = "entityTypeId0";
    String entityTypeId1 = "entityTypeId1";
    Entity entity0 = mock(Entity.class);
    Repository<Entity> sourceRepository0 = mock(Repository.class);
    when(sourceRepository0.spliterator()).thenReturn(singletonList(entity0).spliterator());
    Entity entity1 = mock(Entity.class);
    Repository<Entity> sourceRepository1 = mock(Repository.class);
    when(sourceRepository1.spliterator()).thenReturn(singletonList(entity1).spliterator());
    RepositoryCollection repositoryCollection = mock(RepositoryCollection.class);
    when(repositoryCollection.getEntityTypeIds()).thenReturn(asList(entityTypeId0, entityTypeId1));
    when(repositoryCollection.getRepository(entityTypeId0)).thenReturn(sourceRepository0);
    when(repositoryCollection.getRepository(entityTypeId1)).thenReturn(sourceRepository1);
    Repository<Entity> targetRepository0 = mock(Repository.class);
    when(targetRepository0.add(any(Stream.class))).thenReturn(1);
    Repository<Entity> targetRepository1 = mock(Repository.class);
    when(targetRepository1.add(any(Stream.class))).thenReturn(1);
    when(dataService.getRepository(entityTypeId0)).thenReturn(targetRepository0);
    when(dataService.getRepository(entityTypeId1)).thenReturn(targetRepository1);
    EntityImportReport entityImportReport = ontologyImportService.doImport(repositoryCollection, DatabaseAction.ADD, null);
    assertEquals(entityImportReport.getNewEntities(), emptyList());
    assertEquals(entityImportReport.getNrImportedEntitiesMap(), of(entityTypeId0, 1, entityTypeId1, 1));
    @SuppressWarnings("unchecked") ArgumentCaptor<Stream<Entity>> entityTypeId0Captor = ArgumentCaptor.forClass(Stream.class);
    verify(targetRepository0).add(entityTypeId0Captor.capture());
    assertEquals(entityTypeId0Captor.getValue().collect(toList()), singletonList(entity0));
    @SuppressWarnings("unchecked") ArgumentCaptor<Stream<Entity>> entityTypeId1Captor = ArgumentCaptor.forClass(Stream.class);
    verify(targetRepository1).add(entityTypeId1Captor.capture());
    assertEquals(entityTypeId1Captor.getValue().collect(toList()), singletonList(entity1));
}

Example 13 with EntityImportReport

use of org.molgenis.data.importer.EntityImportReport in project molgenis by molgenis.

the class VcfImporterService method importVcf.

private EntityImportReport importVcf(Repository<Entity> inRepository, List<EntityType> addedEntities, String packageId) throws IOException {
    EntityImportReport report = new EntityImportReport();
    String entityTypeId = inRepository.getName();
    if (runAsSystem(() -> dataService.hasRepository(entityTypeId))) {
        throw new MolgenisDataException("Can't overwrite existing " + entityTypeId);
    }
    EntityType entityType = inRepository.getEntityType();
    entityType.setBackend(metaDataService.getDefaultBackend().getName());
    Package targetPackage = determineTargetPackage(packageId, entityType);
    Repository<Entity> sampleRepository = createSampleRepository(addedEntities, entityType, targetPackage);
    Iterator<Entity> inIterator = inRepository.iterator();
    try (Repository<Entity> outRepository = runAsSystem(() -> dataService.getMeta().createRepository(entityType))) {
        permissionSystemService.giveUserWriteMetaPermissions(entityType);
        addedEntities.add(entityType);
        if (sampleRepository != null) {
            int sampleEntityCount = addSampleEntities(sampleRepository, inIterator);
            report.addNewEntity(sampleRepository.getName());
            if (sampleEntityCount > 0) {
                report.addEntityCount(sampleRepository.getName(), sampleEntityCount);
            }
        }
        AtomicInteger vcfEntityCount = new AtomicInteger();
        runAsSystem(() -> outRepository.add(StreamSupport.stream(inRepository.spliterator(), false).filter(entity -> {
            vcfEntityCount.incrementAndGet();
            return true;
        })));
        if (vcfEntityCount.get() > 0) {
            report.addEntityCount(entityTypeId, vcfEntityCount.get());
        }
    }
    report.addNewEntity(entityTypeId);
    return report;
}

Example 14 with EntityImportReport

use of org.molgenis.data.importer.EntityImportReport in project molgenis by molgenis.

the class VcfImporterServiceTest method doImportVcfWithSamples.

@SuppressWarnings("unchecked")
@Test
public void doImportVcfWithSamples() {
    // Test with multiple input repositories not possible due to
    // https://github.com/molgenis/molgenis/issues/4544
    String entityTypeId0 = "entity0";
    List<String> entityTypeIds = Arrays.asList(entityTypeId0);
    String sampleEntityName0 = "entity0sample";
    EntityType sampleEntityType0 = mock(EntityType.class);
    when(sampleEntityType0.getId()).thenReturn(sampleEntityName0);
    when(sampleEntityType0.getOwnAttributes()).thenReturn(emptyList());
    when(sampleEntityType0.getOwnLookupAttributes()).thenReturn(emptyList());
    Repository<Entity> outSampleRepo0 = mock(Repository.class);
    when(outSampleRepo0.getName()).thenReturn(sampleEntityName0);
    when(metaDataService.createRepository(argThat(eqName(sampleEntityType0)))).thenReturn(outSampleRepo0);
    Attribute sampleAttr = mock(Attribute.class);
    when(sampleAttr.getName()).thenReturn(VcfAttributes.SAMPLES);
    when(sampleAttr.getRefEntity()).thenReturn(sampleEntityType0);
    when(sampleAttr.getDataType()).thenReturn(MREF);
    EntityType entityType0 = mock(EntityType.class);
    when(entityType0.getId()).thenReturn(entityTypeId0);
    when(entityType0.getAttribute(VcfAttributes.SAMPLES)).thenReturn(sampleAttr);
    when(entityType0.getOwnAttributes()).thenReturn(singletonList(sampleAttr));
    when(entityType0.getOwnLookupAttributes()).thenReturn(emptyList());
    Entity entity0Sample0 = mock(Entity.class);
    Entity entity0Sample1 = mock(Entity.class);
    Entity entity1Sample0 = mock(Entity.class);
    Entity entity1Sample1 = mock(Entity.class);
    Entity entity0 = mock(Entity.class);
    when(entity0.getEntities(VcfAttributes.SAMPLES)).thenReturn(Arrays.asList(entity0Sample0, entity0Sample1));
    Entity entity1 = mock(Entity.class);
    when(entity1.getEntities(VcfAttributes.SAMPLES)).thenReturn(Arrays.asList(entity1Sample0, entity1Sample1));
    List<Entity> entities = Arrays.asList(entity0, entity1);
    Repository<Entity> repo0 = Mockito.spy(new AbstractRepository() {

        @Override
        public Set<RepositoryCapability> getCapabilities() {
            return null;
        }

        public EntityType getEntityType() {
            return entityType0;
        }

        @Override
        public Iterator<Entity> iterator() {
            return entities.iterator();
        }

        @Override
        public Spliterator<Entity> spliterator() {
            return entities.spliterator();
        }

        @Override
        public String getName() {
            return entityTypeId0;
        }

        @Override
        public void forEachBatched(Consumer<List<Entity>> consumer, int batchSize) {
            this.forEachBatched(null, consumer, batchSize);
        }
    });
    when(dataService.hasRepository(entityTypeId0)).thenReturn(false);
    Repository<Entity> outRepo0 = mock(Repository.class);
    when(metaDataService.createRepository(argThat(eqName(entityType0)))).thenReturn(outRepo0);
    when(outRepo0.add(any(Stream.class))).thenAnswer(invocation -> {
        Stream<Entity> entities1 = (Stream<Entity>) invocation.getArguments()[0];
        List<Entity> entityList = entities1.collect(Collectors.toList());
        return entityList.size();
    });
    RepositoryCollection source = mock(RepositoryCollection.class);
    when(source.getEntityTypeIds()).thenReturn(entityTypeIds);
    when(source.getRepository(entityTypeId0)).thenReturn(repo0);
    String defaultPackage = "package";
    Package pack = mock(Package.class);
    when(metaDataService.getPackage(defaultPackage)).thenReturn(pack);
    EntityImportReport entityImportReport = vcfImporterService.doImport(source, DatabaseAction.ADD, defaultPackage);
    EntityImportReport expectedEntityImportReport = new EntityImportReport();
    expectedEntityImportReport.addNewEntity(sampleEntityName0);
    expectedEntityImportReport.addEntityCount(sampleEntityName0, 4);
    expectedEntityImportReport.addNewEntity(entityTypeId0);
    expectedEntityImportReport.addEntityCount(entityTypeId0, entities.size());
    assertEquals(entityImportReport, expectedEntityImportReport);
    verify(metaDataService).createRepository(argThat(eqName(sampleEntityType0)));
    verify(metaDataService).createRepository(argThat(eqName(entityType0)));
    verify(permissionSystemService).giveUserWriteMetaPermissions(entityType0);
    verify(permissionSystemService).giveUserWriteMetaPermissions(sampleEntityType0);
}

Example 15 with EntityImportReport

use of org.molgenis.data.importer.EntityImportReport in project molgenis by molgenis.

the class VcfImporterServiceTest method doImportVcfWithoutSamples.

@SuppressWarnings("unchecked")
@Test
public void doImportVcfWithoutSamples() {
    // Test with multiple input repositories not possible due to
    // https://github.com/molgenis/molgenis/issues/4544
    String entityTypeId0 = "entity0";
    List<String> entityTypeIds = Arrays.asList(entityTypeId0);
    EntityType entityType0 = mock(EntityType.class);
    when(entityType0.getId()).thenReturn(entityTypeId0);
    when(entityType0.getOwnAttributes()).thenReturn(emptyList());
    when(entityType0.getOwnLookupAttributes()).thenReturn(emptyList());
    Entity entity0 = mock(Entity.class);
    Entity entity1 = mock(Entity.class);
    List<Entity> entities = Arrays.asList(entity0, entity1);
    Repository<Entity> repo0 = Mockito.spy(new AbstractRepository() {

        @Override
        public Set<RepositoryCapability> getCapabilities() {
            return null;
        }

        public EntityType getEntityType() {
            return entityType0;
        }

        @Override
        public Iterator<Entity> iterator() {
            return entities.iterator();
        }

        @Override
        public Spliterator<Entity> spliterator() {
            return entities.spliterator();
        }

        @Override
        public String getName() {
            return entityTypeId0;
        }

        @Override
        public void forEachBatched(Consumer<List<Entity>> consumer, int batchSize) {
            this.forEachBatched(null, consumer, batchSize);
        }
    });
    when(dataService.hasRepository(entityTypeId0)).thenReturn(false);
    Repository<Entity> outRepo0 = mock(Repository.class);
    when(metaDataService.createRepository(argThat(eqName(entityType0)))).thenReturn(outRepo0);
    when(outRepo0.add(any(Stream.class))).thenAnswer(invocation -> {
        Stream<Entity> entities1 = (Stream<Entity>) invocation.getArguments()[0];
        List<Entity> entityList = entities1.collect(Collectors.toList());
        return entityList.size();
    });
    RepositoryCollection source = mock(RepositoryCollection.class);
    when(source.getEntityTypeIds()).thenReturn(entityTypeIds);
    when(source.getRepository(entityTypeId0)).thenReturn(repo0);
    String defaultPackage = "package";
    Package pack = mock(Package.class);
    when(metaDataService.getPackage(defaultPackage)).thenReturn(pack);
    EntityImportReport entityImportReport = vcfImporterService.doImport(source, DatabaseAction.ADD, defaultPackage);
    EntityImportReport expectedEntityImportReport = new EntityImportReport();
    expectedEntityImportReport.addEntityCount(entityTypeId0, entities.size());
    expectedEntityImportReport.addNewEntity(entityTypeId0);
    assertEquals(entityImportReport, expectedEntityImportReport);
    verify(metaDataService, times(1)).createRepository(argThat(eqName(entityType0)));
    verify(permissionSystemService, times(1)).giveUserWriteMetaPermissions(entityType0);
}