Search in sources :

Example 1 with XMLReader

use of org.hisp.staxwax.reader.XMLReader in project dhis2-core by dhis2.

the class DefaultAdxDataService method saveDataValueSetInternal.

private ImportSummary saveDataValueSetInternal(InputStream in, ImportOptions importOptions, TaskId id) {
    notifier.clear(id).notify(id, "ADX parsing process started");
    ImportOptions adxImportOptions = ObjectUtils.firstNonNull(importOptions, ImportOptions.getDefaultImportOptions()).instance().setNotificationLevel(NotificationLevel.OFF);
    // Get import options
    IdScheme dataSetIdScheme = importOptions.getIdSchemes().getDataSetIdScheme();
    IdScheme dataElementIdScheme = importOptions.getIdSchemes().getDataElementIdScheme();
    // Create meta-data maps
    CachingMap<String, DataSet> dataSetMap = new CachingMap<>();
    CachingMap<String, DataElement> dataElementMap = new CachingMap<>();
    // Get meta-data maps
    IdentifiableObjectCallable<DataSet> dataSetCallable = new IdentifiableObjectCallable<>(identifiableObjectManager, DataSet.class, dataSetIdScheme, null);
    IdentifiableObjectCallable<DataElement> dataElementCallable = new IdentifiableObjectCallable<>(identifiableObjectManager, DataElement.class, dataElementIdScheme, null);
    // Heat cache
    if (importOptions.isPreheatCacheDefaultFalse()) {
        dataSetMap.load(identifiableObjectManager.getAll(DataSet.class), o -> o.getPropertyValue(dataSetIdScheme));
        dataElementMap.load(identifiableObjectManager.getAll(DataElement.class), o -> o.getPropertyValue(dataElementIdScheme));
    }
    XMLReader adxReader = XMLFactory.getXMLReader(in);
    ImportSummary importSummary;
    adxReader.moveToStartElement(AdxDataService.ROOT, AdxDataService.NAMESPACE);
    ExecutorService executor = Executors.newSingleThreadExecutor();
    // Give the DXF import a different notification task ID so it doesn't conflict with notifications from this level.
    TaskId dxfTaskId = new TaskId(TaskCategory.DATAVALUE_IMPORT_INTERNAL, id.getUser());
    int groupCount = 0;
    try (PipedOutputStream pipeOut = new PipedOutputStream()) {
        Future<ImportSummary> futureImportSummary = executor.submit(new AdxPipedImporter(dataValueSetService, adxImportOptions, dxfTaskId, pipeOut, sessionFactory));
        XMLOutputFactory factory = XMLOutputFactory.newInstance();
        XMLStreamWriter dxfWriter = factory.createXMLStreamWriter(pipeOut);
        List<ImportConflict> adxConflicts = new LinkedList<>();
        dxfWriter.writeStartDocument("1.0");
        dxfWriter.writeStartElement("dataValueSet");
        dxfWriter.writeDefaultNamespace("http://dhis2.org/schema/dxf/2.0");
        notifier.notify(id, "Starting to import ADX data groups.");
        while (adxReader.moveToStartElement(AdxDataService.GROUP, AdxDataService.NAMESPACE)) {
            notifier.update(id, "Importing ADX data group: " + groupCount);
            // note this returns conflicts which are detected at ADX level
            adxConflicts.addAll(parseAdxGroupToDxf(adxReader, dxfWriter, adxImportOptions, dataSetMap, dataSetCallable, dataElementMap, dataElementCallable));
            groupCount++;
        }
        // end dataValueSet
        dxfWriter.writeEndElement();
        dxfWriter.writeEndDocument();
        pipeOut.flush();
        importSummary = futureImportSummary.get(TOTAL_MINUTES_TO_WAIT, TimeUnit.MINUTES);
        importSummary.getConflicts().addAll(adxConflicts);
        importSummary.getImportCount().incrementIgnored(adxConflicts.size());
    } catch (AdxException ex) {
        importSummary = new ImportSummary();
        importSummary.setStatus(ImportStatus.ERROR);
        importSummary.setDescription("Data set import failed within group number: " + groupCount);
        importSummary.getConflicts().add(ex.getImportConflict());
        notifier.update(id, NotificationLevel.ERROR, "ADX data import done", true);
        log.warn("Import failed: " + DebugUtils.getStackTrace(ex));
    } catch (IOException | XMLStreamException | InterruptedException | ExecutionException | TimeoutException ex) {
        importSummary = new ImportSummary();
        importSummary.setStatus(ImportStatus.ERROR);
        importSummary.setDescription("Data set import failed within group number: " + groupCount);
        notifier.update(id, NotificationLevel.ERROR, "ADX data import done", true);
        log.warn("Import failed: " + DebugUtils.getStackTrace(ex));
    }
    executor.shutdown();
    notifier.update(id, INFO, "ADX data import done", true).addTaskSummary(id, importSummary);
    ImportCount c = importSummary.getImportCount();
    log.info("ADX data import done, imported: " + c.getImported() + ", updated: " + c.getUpdated() + ", deleted: " + c.getDeleted() + ", ignored: " + c.getIgnored());
    return importSummary;
}
Also used : XMLOutputFactory(javax.xml.stream.XMLOutputFactory) TaskId(org.hisp.dhis.scheduling.TaskId) DataSet(org.hisp.dhis.dataset.DataSet) ImportSummary(org.hisp.dhis.dxf2.importsummary.ImportSummary) PipedOutputStream(java.io.PipedOutputStream) DataElement(org.hisp.dhis.dataelement.DataElement) CachingMap(org.hisp.dhis.commons.collection.CachingMap) XMLStreamWriter(javax.xml.stream.XMLStreamWriter) ExecutionException(java.util.concurrent.ExecutionException) XMLReader(org.hisp.staxwax.reader.XMLReader) ImportConflict(org.hisp.dhis.dxf2.importsummary.ImportConflict) TimeoutException(java.util.concurrent.TimeoutException) ImportCount(org.hisp.dhis.dxf2.importsummary.ImportCount) IdScheme(org.hisp.dhis.common.IdScheme) IOException(java.io.IOException) IdentifiableObjectCallable(org.hisp.dhis.system.callable.IdentifiableObjectCallable) LinkedList(java.util.LinkedList) XMLStreamException(javax.xml.stream.XMLStreamException) ExecutorService(java.util.concurrent.ExecutorService) ImportOptions(org.hisp.dhis.dxf2.common.ImportOptions)

Aggregations

IOException (java.io.IOException)1 PipedOutputStream (java.io.PipedOutputStream)1 LinkedList (java.util.LinkedList)1 ExecutionException (java.util.concurrent.ExecutionException)1 ExecutorService (java.util.concurrent.ExecutorService)1 TimeoutException (java.util.concurrent.TimeoutException)1 XMLOutputFactory (javax.xml.stream.XMLOutputFactory)1 XMLStreamException (javax.xml.stream.XMLStreamException)1 XMLStreamWriter (javax.xml.stream.XMLStreamWriter)1 IdScheme (org.hisp.dhis.common.IdScheme)1 CachingMap (org.hisp.dhis.commons.collection.CachingMap)1 DataElement (org.hisp.dhis.dataelement.DataElement)1 DataSet (org.hisp.dhis.dataset.DataSet)1 ImportOptions (org.hisp.dhis.dxf2.common.ImportOptions)1 ImportConflict (org.hisp.dhis.dxf2.importsummary.ImportConflict)1 ImportCount (org.hisp.dhis.dxf2.importsummary.ImportCount)1 ImportSummary (org.hisp.dhis.dxf2.importsummary.ImportSummary)1 TaskId (org.hisp.dhis.scheduling.TaskId)1 IdentifiableObjectCallable (org.hisp.dhis.system.callable.IdentifiableObjectCallable)1 XMLReader (org.hisp.staxwax.reader.XMLReader)1