Examples with ImportCount org.hisp.dhis.dxf2.importsummary.ImportCount used on opensource projects

Example 1 with ImportCount

use of org.hisp.dhis.dxf2.importsummary.ImportCount in project dhis2-core by dhis2.

the class DefaultCompleteDataSetRegistrationExchangeService method saveCompleteDataSetRegistrations.

private ImportSummary saveCompleteDataSetRegistrations(ImportOptions importOptions, TaskId id, CompleteDataSetRegistrations completeRegistrations) {
    Clock clock = new Clock(log).startClock().logTime("Starting complete data set registration import, options: " + importOptions);
    notifier.clear(id).notify(id, "Process started");
    // Start here so we can access any outer attributes for the configuration
    completeRegistrations.open();
    ImportSummary importSummary = new ImportSummary();
    // ---------------------------------------------------------------------
    // Set up import configuration
    // ---------------------------------------------------------------------
    importOptions = importOptions != null ? importOptions : ImportOptions.getDefaultImportOptions();
    log.info("Import options: " + importOptions);
    ImportConfig cfg = new ImportConfig(completeRegistrations, importOptions);
    // ---------------------------------------------------------------------
    // Set up meta-data
    // ---------------------------------------------------------------------
    MetaDataCaches caches = new MetaDataCaches();
    MetaDataCallables metaDataCallables = new MetaDataCallables(cfg);
    if (importOptions.isPreheatCacheDefaultFalse()) {
        caches.preheat(idObjManager, cfg);
    }
    // ---------------------------------------------------------------------
    // Perform import
    // ---------------------------------------------------------------------
    notifier.notify(id, "Importing complete data set registrations");
    int totalCount = batchImport(completeRegistrations, cfg, importSummary, metaDataCallables, caches);
    notifier.notify(id, NotificationLevel.INFO, "Import done", true).addTaskSummary(id, importSummary);
    ImportCount count = importSummary.getImportCount();
    clock.logTime(String.format("Complete data set registration import done, total: %d, imported: %d, updated: %d, deleted: %d", totalCount, count.getImported(), count.getUpdated(), count.getDeleted()));
    completeRegistrations.close();
    return importSummary;
}

Example 2 with ImportCount

use of org.hisp.dhis.dxf2.importsummary.ImportCount in project dhis2-core by dhis2.

the class DefaultAdxDataService method saveDataValueSetInternal.

private ImportSummary saveDataValueSetInternal(InputStream in, ImportOptions importOptions, TaskId id) {
    notifier.clear(id).notify(id, "ADX parsing process started");
    ImportOptions adxImportOptions = ObjectUtils.firstNonNull(importOptions, ImportOptions.getDefaultImportOptions()).instance().setNotificationLevel(NotificationLevel.OFF);
    // Get import options
    IdScheme dataSetIdScheme = importOptions.getIdSchemes().getDataSetIdScheme();
    IdScheme dataElementIdScheme = importOptions.getIdSchemes().getDataElementIdScheme();
    // Create meta-data maps
    CachingMap<String, DataSet> dataSetMap = new CachingMap<>();
    CachingMap<String, DataElement> dataElementMap = new CachingMap<>();
    // Get meta-data maps
    IdentifiableObjectCallable<DataSet> dataSetCallable = new IdentifiableObjectCallable<>(identifiableObjectManager, DataSet.class, dataSetIdScheme, null);
    IdentifiableObjectCallable<DataElement> dataElementCallable = new IdentifiableObjectCallable<>(identifiableObjectManager, DataElement.class, dataElementIdScheme, null);
    // Heat cache
    if (importOptions.isPreheatCacheDefaultFalse()) {
        dataSetMap.load(identifiableObjectManager.getAll(DataSet.class), o -> o.getPropertyValue(dataSetIdScheme));
        dataElementMap.load(identifiableObjectManager.getAll(DataElement.class), o -> o.getPropertyValue(dataElementIdScheme));
    }
    XMLReader adxReader = XMLFactory.getXMLReader(in);
    ImportSummary importSummary;
    adxReader.moveToStartElement(AdxDataService.ROOT, AdxDataService.NAMESPACE);
    ExecutorService executor = Executors.newSingleThreadExecutor();
    // Give the DXF import a different notification task ID so it doesn't conflict with notifications from this level.
    TaskId dxfTaskId = new TaskId(TaskCategory.DATAVALUE_IMPORT_INTERNAL, id.getUser());
    int groupCount = 0;
    try (PipedOutputStream pipeOut = new PipedOutputStream()) {
        Future<ImportSummary> futureImportSummary = executor.submit(new AdxPipedImporter(dataValueSetService, adxImportOptions, dxfTaskId, pipeOut, sessionFactory));
        XMLOutputFactory factory = XMLOutputFactory.newInstance();
        XMLStreamWriter dxfWriter = factory.createXMLStreamWriter(pipeOut);
        List<ImportConflict> adxConflicts = new LinkedList<>();
        dxfWriter.writeStartDocument("1.0");
        dxfWriter.writeStartElement("dataValueSet");
        dxfWriter.writeDefaultNamespace("http://dhis2.org/schema/dxf/2.0");
        notifier.notify(id, "Starting to import ADX data groups.");
        while (adxReader.moveToStartElement(AdxDataService.GROUP, AdxDataService.NAMESPACE)) {
            notifier.update(id, "Importing ADX data group: " + groupCount);
            // note this returns conflicts which are detected at ADX level
            adxConflicts.addAll(parseAdxGroupToDxf(adxReader, dxfWriter, adxImportOptions, dataSetMap, dataSetCallable, dataElementMap, dataElementCallable));
            groupCount++;
        }
        // end dataValueSet
        dxfWriter.writeEndElement();
        dxfWriter.writeEndDocument();
        pipeOut.flush();
        importSummary = futureImportSummary.get(TOTAL_MINUTES_TO_WAIT, TimeUnit.MINUTES);
        importSummary.getConflicts().addAll(adxConflicts);
        importSummary.getImportCount().incrementIgnored(adxConflicts.size());
    } catch (AdxException ex) {
        importSummary = new ImportSummary();
        importSummary.setStatus(ImportStatus.ERROR);
        importSummary.setDescription("Data set import failed within group number: " + groupCount);
        importSummary.getConflicts().add(ex.getImportConflict());
        notifier.update(id, NotificationLevel.ERROR, "ADX data import done", true);
        log.warn("Import failed: " + DebugUtils.getStackTrace(ex));
    } catch (IOException | XMLStreamException | InterruptedException | ExecutionException | TimeoutException ex) {
        importSummary = new ImportSummary();
        importSummary.setStatus(ImportStatus.ERROR);
        importSummary.setDescription("Data set import failed within group number: " + groupCount);
        notifier.update(id, NotificationLevel.ERROR, "ADX data import done", true);
        log.warn("Import failed: " + DebugUtils.getStackTrace(ex));
    }
    executor.shutdown();
    notifier.update(id, INFO, "ADX data import done", true).addTaskSummary(id, importSummary);
    ImportCount c = importSummary.getImportCount();
    log.info("ADX data import done, imported: " + c.getImported() + ", updated: " + c.getUpdated() + ", deleted: " + c.getDeleted() + ", ignored: " + c.getIgnored());
    return importSummary;
}

Example 3 with ImportCount

use of org.hisp.dhis.dxf2.importsummary.ImportCount in project dhis2-core by dhis2.

the class DefaultAdxDataService method saveDataValueSetInternal.

private ImportSummary saveDataValueSetInternal(InputStream in, ImportOptions importOptions, JobConfiguration id) {
    notifier.clear(id).notify(id, "ADX parsing process started");
    ImportOptions adxImportOptions = firstNonNull(importOptions, ImportOptions.getDefaultImportOptions()).instance().setNotificationLevel(NotificationLevel.OFF);
    // Get import options
    IdScheme dsScheme = importOptions.getIdSchemes().getDataSetIdScheme();
    IdScheme deScheme = importOptions.getIdSchemes().getDataElementIdScheme();
    // Create meta-data maps
    CachingMap<String, DataSet> dataSetMap = new CachingMap<>();
    CachingMap<String, DataElement> dataElementMap = new CachingMap<>();
    // Get meta-data maps
    IdentifiableObjectCallable<DataSet> dataSetCallable = new IdentifiableObjectCallable<>(identifiableObjectManager, DataSet.class, dsScheme, null);
    IdentifiableObjectCallable<DataElement> dataElementCallable = new IdentifiableObjectCallable<>(identifiableObjectManager, DataElement.class, deScheme, null);
    // Heat cache
    if (importOptions.isPreheatCacheDefaultFalse()) {
        dataSetMap.load(identifiableObjectManager.getAll(DataSet.class), o -> o.getPropertyValue(dsScheme));
        dataElementMap.load(identifiableObjectManager.getAll(DataElement.class), o -> o.getPropertyValue(deScheme));
    }
    XMLReader adxReader = XMLFactory.getXMLReader(in);
    ImportSummary importSummary;
    adxReader.moveToStartElement(AdxDataService.ROOT, AdxDataService.NAMESPACE);
    ExecutorService executor = Executors.newSingleThreadExecutor();
    // For Async runs, give the DXF import a different notification task ID
    // so it doesn't conflict with notifications from this level.
    JobConfiguration dxfJobId = (id == null) ? null : new JobConfiguration("dxfJob", JobType.DATAVALUE_IMPORT_INTERNAL, id.getUserUid(), true);
    int groupCount = 0;
    try (PipedOutputStream pipeOut = new PipedOutputStream()) {
        Future<ImportSummary> futureImportSummary = executor.submit(new AdxPipedImporter(dataValueSetService, adxImportOptions, dxfJobId, pipeOut, sessionFactory));
        XMLOutputFactory factory = XMLOutputFactory.newInstance();
        XMLStreamWriter dxfWriter = factory.createXMLStreamWriter(pipeOut);
        List<ImportConflict> adxConflicts = new LinkedList<>();
        dxfWriter.writeStartDocument("1.0");
        dxfWriter.writeStartElement("dataValueSet");
        dxfWriter.writeDefaultNamespace("http://dhis2.org/schema/dxf/2.0");
        notifier.notify(id, "Starting to import ADX data groups.");
        while (adxReader.moveToStartElement(AdxDataService.GROUP, AdxDataService.NAMESPACE)) {
            notifier.update(id, "Importing ADX data group: " + groupCount);
            // note this returns conflicts which are detected at ADX level
            adxConflicts.addAll(parseAdxGroupToDxf(adxReader, dxfWriter, adxImportOptions, dataSetMap, dataSetCallable, dataElementMap, dataElementCallable));
            groupCount++;
        }
        // end dataValueSet
        dxfWriter.writeEndElement();
        dxfWriter.writeEndDocument();
        pipeOut.flush();
        importSummary = futureImportSummary.get(TOTAL_MINUTES_TO_WAIT, TimeUnit.MINUTES);
        ImportSummary summary = importSummary;
        adxConflicts.forEach(conflict -> summary.addConflict(conflict.getObject(), conflict.getValue()));
        importSummary.getImportCount().incrementIgnored(adxConflicts.size());
    } catch (AdxException ex) {
        importSummary = new ImportSummary();
        importSummary.setStatus(ImportStatus.ERROR);
        importSummary.setDescription("Data set import failed within group number: " + groupCount);
        importSummary.addConflict(ex.getObject(), ex.getMessage());
        notifier.update(id, NotificationLevel.ERROR, "ADX data import done", true);
        log.warn("Import failed: " + DebugUtils.getStackTrace(ex));
    } catch (IOException | XMLStreamException | InterruptedException | ExecutionException | TimeoutException ex) {
        importSummary = new ImportSummary();
        importSummary.setStatus(ImportStatus.ERROR);
        importSummary.setDescription("Data set import failed within group number: " + groupCount);
        notifier.update(id, NotificationLevel.ERROR, "ADX data import done", true);
        log.warn("Import failed: " + DebugUtils.getStackTrace(ex));
    }
    executor.shutdown();
    notifier.update(id, INFO, "ADX data import done", true).addJobSummary(id, importSummary, ImportSummary.class);
    ImportCount c = importSummary.getImportCount();
    log.info("ADX data import done, imported: " + c.getImported() + ", updated: " + c.getUpdated() + ", deleted: " + c.getDeleted() + ", ignored: " + c.getIgnored());
    return importSummary;
}

Example 4 with ImportCount

use of org.hisp.dhis.dxf2.importsummary.ImportCount in project dhis2-core by dhis2.

the class DefaultCompleteDataSetRegistrationExchangeService method saveCompleteDataSetRegistrations.

private ImportSummary saveCompleteDataSetRegistrations(ImportOptions importOptions, JobConfiguration id, CompleteDataSetRegistrations completeRegistrations) {
    Clock clock = new Clock(log).startClock().logTime("Starting complete data set registration import, options: " + importOptions);
    notifier.clear(id).notify(id, "Process started");
    // Start here so we can access any outer attributes for the
    // configuration
    completeRegistrations.open();
    ImportSummary importSummary = new ImportSummary();
    // ---------------------------------------------------------------------
    // Set up import configuration
    // ---------------------------------------------------------------------
    importOptions = importOptions != null ? importOptions : ImportOptions.getDefaultImportOptions();
    log.info("Import options: " + importOptions);
    ImportConfig cfg = new ImportConfig(this.systemSettingManager, this.categoryService, completeRegistrations, importOptions);
    // ---------------------------------------------------------------------
    // Set up meta-data
    // ---------------------------------------------------------------------
    MetadataCaches caches = new MetadataCaches();
    MetadataCallables metaDataCallables = new MetadataCallables(cfg, this.idObjManager, this.periodService, this.categoryService);
    if (importOptions.isPreheatCacheDefaultFalse()) {
        caches.preheat(idObjManager, cfg);
    }
    // ---------------------------------------------------------------------
    // Perform import
    // ---------------------------------------------------------------------
    notifier.notify(id, "Importing complete data set registrations");
    int totalCount = batchImport(completeRegistrations, cfg, importSummary, metaDataCallables, caches);
    notifier.notify(id, NotificationLevel.INFO, "Import done", true).addJobSummary(id, importSummary, ImportSummary.class);
    ImportCount count = importSummary.getImportCount();
    clock.logTime(String.format("Complete data set registration import done, total: %d, imported: %d, updated: %d, deleted: %d", totalCount, count.getImported(), count.getUpdated(), count.getDeleted()));
    completeRegistrations.close();
    return importSummary;
}

Example 5 with ImportCount

use of org.hisp.dhis.dxf2.importsummary.ImportCount in project dhis2-core by dhis2.

the class DefaultDataValueSetService method importDataValueSet.

/**
 * There are specific id schemes for data elements and organisation units
 * and a generic id scheme for all objects. The specific id schemes will
 * take precedence over the generic id scheme. The generic id scheme also
 * applies to data set and category option combo.
 * <p>
 * The id schemes uses the following order of precedence:
 * <p>
 * <ul>
 * <li>Id scheme from the data value set</li>
 * <li>Id scheme from the import options</li>
 * <li>Default id scheme which is UID</li>
 * <ul>
 * <p>
 * If id scheme is specific in the data value set, any id schemes in the
 * import options will be ignored.
 */
private ImportSummary importDataValueSet(ImportOptions options, JobConfiguration id, DataValueSetReader reader) {
    DataValueSet dataValueSet = reader.readHeader();
    final ImportContext context = createDataValueSetImportContext(options, dataValueSet);
    logDataValueSetImportContextInfo(context);
    Clock clock = new Clock(log).startClock().logTime("Starting data value import, options: " + context.getImportOptions());
    NotificationLevel notificationLevel = context.getImportOptions().getNotificationLevel(INFO);
    notifier.clear(id).notify(id, notificationLevel, "Process started");
    // ---------------------------------------------------------------------
    // Heat caches
    // ---------------------------------------------------------------------
    preheatCaches(context);
    // ---------------------------------------------------------------------
    // Get outer meta-data
    // ---------------------------------------------------------------------
    ImportContext.DataSetContext dataSetContext = createDataSetContext(context, dataValueSet);
    if (importValidator.abortDataSetImport(dataValueSet, context, dataSetContext)) {
        context.getSummary().setDescription("Import process was aborted");
        notifier.notify(id, WARN, "Import process aborted", true).addJobSummary(id, context.getSummary(), ImportSummary.class);
        return context.getSummary();
    }
    Date completeDate = parseDate(dataValueSet.getCompleteDate());
    if (dataSetContext.getDataSet() != null && completeDate != null) {
        notifier.notify(id, notificationLevel, "Completing data set");
        handleComplete(dataSetContext.getDataSet(), completeDate, dataSetContext.getOuterPeriod(), dataSetContext.getOuterOrgUnit(), dataSetContext.getFallbackCategoryOptionCombo(), context.getCurrentUserName(), context.getSummary());
    } else {
        context.getSummary().setDataSetComplete(Boolean.FALSE.toString());
    }
    final ImportCount importCount = new ImportCount();
    // ---------------------------------------------------------------------
    // Data values
    // ---------------------------------------------------------------------
    Date now = new Date();
    clock.logTime("Validated outer meta-data");
    notifier.notify(id, notificationLevel, "Importing data values");
    List<? extends DataValueEntry> values = dataValueSet.getDataValues();
    int index = 0;
    if (values != null && !values.isEmpty()) {
        for (DataValueEntry dataValue : values) {
            importDataValue(context, dataSetContext, importCount, now, index++, dataValue);
        }
    }
    DataValueEntry dataValue = reader.readNext();
    while (dataValue != null) {
        importDataValue(context, dataSetContext, importCount, now, index++, dataValue);
        dataValue = reader.readNext();
    }
    context.getDataValueBatchHandler().flush();
    if (!context.isSkipAudit()) {
        context.getAuditBatchHandler().flush();
    }
    context.getSummary().setImportCount(importCount).setStatus(!context.getSummary().hasConflicts() ? ImportStatus.SUCCESS : ImportStatus.WARNING).setDescription("Import process completed successfully");
    clock.logTime("Data value import done, total: " + importCount.getTotalCount() + ", import: " + importCount.getImported() + ", update: " + importCount.getUpdated() + ", delete: " + importCount.getDeleted());
    notifier.notify(id, notificationLevel, "Import done", true).addJobSummary(id, notificationLevel, context.getSummary(), ImportSummary.class);
    return context.getSummary();
}