Search in sources :

Example 26 with DataValueSet

use of org.hisp.dhis.dxf2.datavalueset.DataValueSet in project dhis2-core by dhis2.

the class AnalyticsUtilsTest method testGetDataValueSetFromGridWithDuplicates.

@Test
public void testGetDataValueSetFromGridWithDuplicates() {
    Grid grid = new ListGrid();
    grid.addHeader(new GridHeader(DimensionalObject.DATA_X_DIM_ID));
    grid.addHeader(new GridHeader(DimensionalObject.ORGUNIT_DIM_ID));
    grid.addHeader(new GridHeader(DimensionalObject.PERIOD_DIM_ID));
    grid.addHeader(new GridHeader(DimensionalObject.CATEGORYOPTIONCOMBO_DIM_ID));
    grid.addHeader(new GridHeader(DimensionalObject.ATTRIBUTEOPTIONCOMBO_DIM_ID));
    grid.addHeader(new GridHeader(VALUE_ID, VALUE_HEADER_NAME, ValueType.NUMBER, Double.class.getName(), false, false));
    grid.addRow().addValuesAsList(Lists.newArrayList("dxA", "ouA", "peA", null, null, 1d));
    grid.addRow().addValuesAsList(Lists.newArrayList("dxA", "ouA", "peB", null, null, 2d));
    // Duplicate
    grid.addRow().addValuesAsList(Lists.newArrayList("dxA", "ouA", "peB", null, null, 2d));
    grid.addRow().addValuesAsList(Lists.newArrayList("dxA", "ouB", "peA", null, null, 3d));
    grid.addRow().addValuesAsList(Lists.newArrayList("dxA", "ouB", "peB", null, null, 4d));
    grid.addRow().addValuesAsList(Lists.newArrayList("dxB", "ouA", "peA", null, null, 5d));
    // Duplicate
    grid.addRow().addValuesAsList(Lists.newArrayList("dxB", "ouA", "peA", null, null, 5d));
    grid.addRow().addValuesAsList(Lists.newArrayList("dxB", "ouA", "peB", null, null, 6d));
    grid.addRow().addValuesAsList(Lists.newArrayList("dxC", "ouA", "peA", null, null, 7d));
    // Duplicate
    grid.addRow().addValuesAsList(Lists.newArrayList("dxC", "ouA", "peA", null, null, 7d));
    grid.addRow().addValuesAsList(Lists.newArrayList("dxC", "ouA", "peB", null, null, 8d));
    DataValueSet dvs = AnalyticsUtils.getDataValueSetFromGrid(DataQueryParams.newBuilder().withDuplicatesOnly(true).build(), grid);
    assertNotNull(dvs);
    assertNotNull(dvs.getDataValues());
    assertEquals(3, dvs.getDataValues().size());
}
Also used : DataValueSet(org.hisp.dhis.dxf2.datavalueset.DataValueSet) ListGrid(org.hisp.dhis.system.grid.ListGrid) ListGrid(org.hisp.dhis.system.grid.ListGrid) Test(org.junit.Test) DhisConvenienceTest(org.hisp.dhis.DhisConvenienceTest)

Example 27 with DataValueSet

use of org.hisp.dhis.dxf2.datavalueset.DataValueSet in project dhis2-core by dhis2.

the class AnalyticsUtilsTest method testGetDataValueSetFromGrid.

@Test
public void testGetDataValueSetFromGrid() {
    Grid grid = new ListGrid();
    grid.addHeader(new GridHeader(DimensionalObject.DATA_X_DIM_ID));
    grid.addHeader(new GridHeader(DimensionalObject.ORGUNIT_DIM_ID));
    grid.addHeader(new GridHeader(DimensionalObject.PERIOD_DIM_ID));
    grid.addHeader(new GridHeader(DimensionalObject.CATEGORYOPTIONCOMBO_DIM_ID));
    grid.addHeader(new GridHeader(DimensionalObject.ATTRIBUTEOPTIONCOMBO_DIM_ID));
    grid.addHeader(new GridHeader(VALUE_ID, VALUE_HEADER_NAME, ValueType.NUMBER, Double.class.getName(), false, false));
    grid.addRow().addValuesAsList(Lists.newArrayList("dxA", "ouA", "peA", "coA", "aoA", 1d));
    grid.addRow().addValuesAsList(Lists.newArrayList("dxA", "ouA", "peB", null, null, 2d));
    grid.addRow().addValuesAsList(Lists.newArrayList("dxA", "ouB", "peA", null, null, 3d));
    grid.addRow().addValuesAsList(Lists.newArrayList("dxA", "ouB", "peB", null, null, 4d));
    grid.addRow().addValuesAsList(Lists.newArrayList("dxB", "ouA", "peA", "coA", null, 5d));
    grid.addRow().addValuesAsList(Lists.newArrayList("dxB", "ouA", "peB", "coA", "aoB", 6d));
    grid.addRow().addValuesAsList(Lists.newArrayList("dxC", "ouA", "peA", null, "aoA", 7));
    grid.addRow().addValuesAsList(Lists.newArrayList("dxC", "ouA", "peB", null, null, 8d));
    grid.addRow().addValuesAsList(Lists.newArrayList("dxD", "ouA", "peA", "coB", null, 9d));
    grid.addRow().addValuesAsList(Lists.newArrayList("dxE", "ouA", "peB", null, null, 10));
    DataValueSet dvs = AnalyticsUtils.getDataValueSetFromGrid(DataQueryParams.newBuilder().build(), grid);
    assertNotNull(dvs);
    assertNotNull(dvs.getDataValues());
    assertEquals(10, dvs.getDataValues().size());
    assertEquals("dxA", dvs.getDataValues().get(1).getDataElement());
    assertEquals("ouA", dvs.getDataValues().get(1).getOrgUnit());
    assertEquals("peB", dvs.getDataValues().get(1).getPeriod());
    assertNull(dvs.getDataValues().get(1).getCategoryOptionCombo());
    assertNull(dvs.getDataValues().get(1).getAttributeOptionCombo());
    assertEquals("2.0", dvs.getDataValues().get(1).getValue());
    assertEquals("dxB", dvs.getDataValues().get(4).getDataElement());
    assertEquals("ouA", dvs.getDataValues().get(4).getOrgUnit());
    assertEquals("peA", dvs.getDataValues().get(4).getPeriod());
    assertEquals("coA", dvs.getDataValues().get(4).getCategoryOptionCombo());
    assertNull(dvs.getDataValues().get(4).getAttributeOptionCombo());
    assertEquals("5.0", dvs.getDataValues().get(4).getValue());
    assertEquals("dxC", dvs.getDataValues().get(6).getDataElement());
    assertEquals("ouA", dvs.getDataValues().get(6).getOrgUnit());
    assertEquals("peA", dvs.getDataValues().get(6).getPeriod());
    assertNull(dvs.getDataValues().get(6).getCategoryOptionCombo());
    assertEquals("aoA", dvs.getDataValues().get(6).getAttributeOptionCombo());
    assertEquals("7", dvs.getDataValues().get(6).getValue());
    assertEquals("dxD", dvs.getDataValues().get(8).getDataElement());
    assertEquals("ouA", dvs.getDataValues().get(8).getOrgUnit());
    assertEquals("peA", dvs.getDataValues().get(8).getPeriod());
    assertEquals("coB", dvs.getDataValues().get(8).getCategoryOptionCombo());
    assertNull(dvs.getDataValues().get(8).getAttributeOptionCombo());
    assertEquals("9.0", dvs.getDataValues().get(8).getValue());
    assertEquals("dxE", dvs.getDataValues().get(9).getDataElement());
    assertEquals("ouA", dvs.getDataValues().get(9).getOrgUnit());
    assertEquals("peB", dvs.getDataValues().get(9).getPeriod());
    assertNull(dvs.getDataValues().get(9).getCategoryOptionCombo());
    assertNull(dvs.getDataValues().get(9).getAttributeOptionCombo());
    assertEquals("10", dvs.getDataValues().get(9).getValue());
}
Also used : DataValueSet(org.hisp.dhis.dxf2.datavalueset.DataValueSet) ListGrid(org.hisp.dhis.system.grid.ListGrid) ListGrid(org.hisp.dhis.system.grid.ListGrid) Test(org.junit.Test) DhisConvenienceTest(org.hisp.dhis.DhisConvenienceTest)

Example 28 with DataValueSet

use of org.hisp.dhis.dxf2.datavalueset.DataValueSet in project dhis2-core by dhis2.

the class DefaultAdxDataService method saveDataValueSetInternal.

private ImportSummary saveDataValueSetInternal(InputStream in, ImportOptions importOptions, TaskId id) {
    notifier.clear(id).notify(id, "ADX parsing process started");
    ImportOptions adxImportOptions = ObjectUtils.firstNonNull(importOptions, ImportOptions.getDefaultImportOptions()).instance().setNotificationLevel(NotificationLevel.OFF);
    // Get import options
    IdScheme dataSetIdScheme = importOptions.getIdSchemes().getDataSetIdScheme();
    IdScheme dataElementIdScheme = importOptions.getIdSchemes().getDataElementIdScheme();
    // Create meta-data maps
    CachingMap<String, DataSet> dataSetMap = new CachingMap<>();
    CachingMap<String, DataElement> dataElementMap = new CachingMap<>();
    // Get meta-data maps
    IdentifiableObjectCallable<DataSet> dataSetCallable = new IdentifiableObjectCallable<>(identifiableObjectManager, DataSet.class, dataSetIdScheme, null);
    IdentifiableObjectCallable<DataElement> dataElementCallable = new IdentifiableObjectCallable<>(identifiableObjectManager, DataElement.class, dataElementIdScheme, null);
    // Heat cache
    if (importOptions.isPreheatCacheDefaultFalse()) {
        dataSetMap.load(identifiableObjectManager.getAll(DataSet.class), o -> o.getPropertyValue(dataSetIdScheme));
        dataElementMap.load(identifiableObjectManager.getAll(DataElement.class), o -> o.getPropertyValue(dataElementIdScheme));
    }
    XMLReader adxReader = XMLFactory.getXMLReader(in);
    ImportSummary importSummary;
    adxReader.moveToStartElement(AdxDataService.ROOT, AdxDataService.NAMESPACE);
    ExecutorService executor = Executors.newSingleThreadExecutor();
    // Give the DXF import a different notification task ID so it doesn't conflict with notifications from this level.
    TaskId dxfTaskId = new TaskId(TaskCategory.DATAVALUE_IMPORT_INTERNAL, id.getUser());
    int groupCount = 0;
    try (PipedOutputStream pipeOut = new PipedOutputStream()) {
        Future<ImportSummary> futureImportSummary = executor.submit(new AdxPipedImporter(dataValueSetService, adxImportOptions, dxfTaskId, pipeOut, sessionFactory));
        XMLOutputFactory factory = XMLOutputFactory.newInstance();
        XMLStreamWriter dxfWriter = factory.createXMLStreamWriter(pipeOut);
        List<ImportConflict> adxConflicts = new LinkedList<>();
        dxfWriter.writeStartDocument("1.0");
        dxfWriter.writeStartElement("dataValueSet");
        dxfWriter.writeDefaultNamespace("http://dhis2.org/schema/dxf/2.0");
        notifier.notify(id, "Starting to import ADX data groups.");
        while (adxReader.moveToStartElement(AdxDataService.GROUP, AdxDataService.NAMESPACE)) {
            notifier.update(id, "Importing ADX data group: " + groupCount);
            // note this returns conflicts which are detected at ADX level
            adxConflicts.addAll(parseAdxGroupToDxf(adxReader, dxfWriter, adxImportOptions, dataSetMap, dataSetCallable, dataElementMap, dataElementCallable));
            groupCount++;
        }
        // end dataValueSet
        dxfWriter.writeEndElement();
        dxfWriter.writeEndDocument();
        pipeOut.flush();
        importSummary = futureImportSummary.get(TOTAL_MINUTES_TO_WAIT, TimeUnit.MINUTES);
        importSummary.getConflicts().addAll(adxConflicts);
        importSummary.getImportCount().incrementIgnored(adxConflicts.size());
    } catch (AdxException ex) {
        importSummary = new ImportSummary();
        importSummary.setStatus(ImportStatus.ERROR);
        importSummary.setDescription("Data set import failed within group number: " + groupCount);
        importSummary.getConflicts().add(ex.getImportConflict());
        notifier.update(id, NotificationLevel.ERROR, "ADX data import done", true);
        log.warn("Import failed: " + DebugUtils.getStackTrace(ex));
    } catch (IOException | XMLStreamException | InterruptedException | ExecutionException | TimeoutException ex) {
        importSummary = new ImportSummary();
        importSummary.setStatus(ImportStatus.ERROR);
        importSummary.setDescription("Data set import failed within group number: " + groupCount);
        notifier.update(id, NotificationLevel.ERROR, "ADX data import done", true);
        log.warn("Import failed: " + DebugUtils.getStackTrace(ex));
    }
    executor.shutdown();
    notifier.update(id, INFO, "ADX data import done", true).addTaskSummary(id, importSummary);
    ImportCount c = importSummary.getImportCount();
    log.info("ADX data import done, imported: " + c.getImported() + ", updated: " + c.getUpdated() + ", deleted: " + c.getDeleted() + ", ignored: " + c.getIgnored());
    return importSummary;
}
Also used : XMLOutputFactory(javax.xml.stream.XMLOutputFactory) TaskId(org.hisp.dhis.scheduling.TaskId) DataSet(org.hisp.dhis.dataset.DataSet) ImportSummary(org.hisp.dhis.dxf2.importsummary.ImportSummary) PipedOutputStream(java.io.PipedOutputStream) DataElement(org.hisp.dhis.dataelement.DataElement) CachingMap(org.hisp.dhis.commons.collection.CachingMap) XMLStreamWriter(javax.xml.stream.XMLStreamWriter) ExecutionException(java.util.concurrent.ExecutionException) XMLReader(org.hisp.staxwax.reader.XMLReader) ImportConflict(org.hisp.dhis.dxf2.importsummary.ImportConflict) TimeoutException(java.util.concurrent.TimeoutException) ImportCount(org.hisp.dhis.dxf2.importsummary.ImportCount) IdScheme(org.hisp.dhis.common.IdScheme) IOException(java.io.IOException) IdentifiableObjectCallable(org.hisp.dhis.system.callable.IdentifiableObjectCallable) LinkedList(java.util.LinkedList) XMLStreamException(javax.xml.stream.XMLStreamException) ExecutorService(java.util.concurrent.ExecutorService) ImportOptions(org.hisp.dhis.dxf2.common.ImportOptions)

Example 29 with DataValueSet

use of org.hisp.dhis.dxf2.datavalueset.DataValueSet in project dhis2-core by dhis2.

the class DefaultDataValueSetService method saveDataValueSetCsv.

@Override
public ImportSummary saveDataValueSetCsv(InputStream in, ImportOptions importOptions, TaskId id) {
    try {
        in = StreamUtils.wrapAndCheckCompressionFormat(in);
        DataValueSet dataValueSet = new StreamingCsvDataValueSet(new CsvReader(in, Charset.forName("UTF-8")));
        return saveDataValueSet(importOptions, id, dataValueSet);
    } catch (Exception ex) {
        log.error(DebugUtils.getStackTrace(ex));
        notifier.clear(id).notify(id, ERROR, "Process failed: " + ex.getMessage(), true);
        return new ImportSummary(ImportStatus.ERROR, "The import process failed: " + ex.getMessage());
    }
}
Also used : CsvReader(com.csvreader.CsvReader) ImportSummary(org.hisp.dhis.dxf2.importsummary.ImportSummary)

Example 30 with DataValueSet

use of org.hisp.dhis.dxf2.datavalueset.DataValueSet in project dhis2-core by dhis2.

the class DefaultDataValueSetService method saveDataValueSetJson.

@Override
public ImportSummary saveDataValueSetJson(InputStream in, ImportOptions importOptions, TaskId id) {
    try {
        in = StreamUtils.wrapAndCheckCompressionFormat(in);
        DataValueSet dataValueSet = DefaultRenderService.getJsonMapper().readValue(in, DataValueSet.class);
        return saveDataValueSet(importOptions, id, dataValueSet);
    } catch (Exception ex) {
        log.error(DebugUtils.getStackTrace(ex));
        notifier.notify(id, ERROR, "Process failed: " + ex.getMessage(), true);
        return new ImportSummary(ImportStatus.ERROR, "The import process failed: " + ex.getMessage());
    }
}
Also used : ImportSummary(org.hisp.dhis.dxf2.importsummary.ImportSummary)

Aggregations

ImportSummary (org.hisp.dhis.dxf2.importsummary.ImportSummary)45 Test (org.junit.Test)43 ClassPathResource (org.springframework.core.io.ClassPathResource)39 DhisSpringTest (org.hisp.dhis.DhisSpringTest)31 DataValue (org.hisp.dhis.datavalue.DataValue)18 ImportOptions (org.hisp.dhis.dxf2.common.ImportOptions)17 DhisTest (org.hisp.dhis.DhisTest)10 DataValueAudit (org.hisp.dhis.datavalue.DataValueAudit)5 DataValueSet (org.hisp.dhis.dxf2.datavalueset.DataValueSet)5 Period (org.hisp.dhis.period.Period)4 CompleteDataSetRegistration (org.hisp.dhis.dataset.CompleteDataSetRegistration)3 DataSet (org.hisp.dhis.dataset.DataSet)3 DhisConvenienceTest (org.hisp.dhis.DhisConvenienceTest)2 IdScheme (org.hisp.dhis.common.IdScheme)2 IdSchemes (org.hisp.dhis.common.IdSchemes)2 CachingMap (org.hisp.dhis.commons.collection.CachingMap)2 DataElement (org.hisp.dhis.dataelement.DataElement)2 DataValue (org.hisp.dhis.dxf2.datavalue.DataValue)2 ImportConflict (org.hisp.dhis.dxf2.importsummary.ImportConflict)2 ImportCount (org.hisp.dhis.dxf2.importsummary.ImportCount)2