Examples with ImportStrategy org.hisp.dhis.importexport.ImportStrategy used on opensource projects

Example 1 with ImportStrategy

use of org.hisp.dhis.importexport.ImportStrategy in project dhis2-core by dhis2.

the class DefaultMetadataImportService method getParamsFromMap.

@Override
public MetadataImportParams getParamsFromMap(Map<String, List<String>> parameters) {
    MetadataImportParams params = new MetadataImportParams();
    if (params.getUser() == null) {
        params.setUser(currentUserService.getCurrentUser());
    }
    params.setSkipSharing(getBooleanWithDefault(parameters, "skipSharing", false));
    params.setSkipValidation(getBooleanWithDefault(parameters, "skipValidation", false));
    params.setImportMode(getEnumWithDefault(ObjectBundleMode.class, parameters, "importMode", ObjectBundleMode.COMMIT));
    params.setPreheatMode(getEnumWithDefault(PreheatMode.class, parameters, "preheatMode", PreheatMode.REFERENCE));
    params.setIdentifier(getEnumWithDefault(PreheatIdentifier.class, parameters, "identifier", PreheatIdentifier.UID));
    params.setImportStrategy(getEnumWithDefault(ImportStrategy.class, parameters, "importStrategy", ImportStrategy.CREATE_AND_UPDATE));
    params.setAtomicMode(getEnumWithDefault(AtomicMode.class, parameters, "atomicMode", AtomicMode.ALL));
    params.setMergeMode(getEnumWithDefault(MergeMode.class, parameters, "mergeMode", MergeMode.REPLACE));
    params.setFlushMode(getEnumWithDefault(FlushMode.class, parameters, "flushMode", FlushMode.AUTO));
    params.setImportReportMode(getEnumWithDefault(ImportReportMode.class, parameters, "importReportMode", ImportReportMode.ERRORS));
    if (getBooleanWithDefault(parameters, "async", false)) {
        TaskId taskId = new TaskId(TaskCategory.METADATA_IMPORT, params.getUser());
        notifier.clear(taskId);
        params.setTaskId(taskId);
    }
    return params;
}

Example 2 with ImportStrategy

use of org.hisp.dhis.importexport.ImportStrategy in project dhis2-core by dhis2.

the class EventController method postJsonEvent.

@RequestMapping(method = RequestMethod.POST, consumes = "application/json")
@PreAuthorize("hasRole('ALL') or hasRole('F_TRACKED_ENTITY_DATAVALUE_ADD')")
public void postJsonEvent(@RequestParam(defaultValue = "CREATE") ImportStrategy strategy, HttpServletResponse response, HttpServletRequest request, ImportOptions importOptions) throws Exception {
    importOptions.setImportStrategy(strategy);
    InputStream inputStream = StreamUtils.wrapAndCheckCompressionFormat(request.getInputStream());
    importOptions.setIdSchemes(getIdSchemesFromParameters(importOptions.getIdSchemes(), contextService.getParameterValuesMap()));
    if (!importOptions.isAsync()) {
        ImportSummaries importSummaries = eventService.addEventsJson(inputStream, importOptions);
        importSummaries.setImportOptions(importOptions);
        importSummaries.getImportSummaries().stream().filter(importSummary -> !importOptions.isDryRun() && !importSummary.getStatus().equals(ImportStatus.ERROR) && !importOptions.getImportStrategy().isDelete()).forEach(importSummary -> importSummary.setHref(ContextUtils.getRootPath(request) + RESOURCE_PATH + "/" + importSummary.getReference()));
        if (importSummaries.getImportSummaries().size() == 1) {
            ImportSummary importSummary = importSummaries.getImportSummaries().get(0);
            importSummary.setImportOptions(importOptions);
            if (!importOptions.isDryRun()) {
                if (!importSummary.getStatus().equals(ImportStatus.ERROR)) {
                    response.setHeader("Location", ContextUtils.getRootPath(request) + RESOURCE_PATH + "/" + importSummary.getReference());
                }
            }
        }
        webMessageService.send(WebMessageUtils.importSummaries(importSummaries), response, request);
    } else {
        TaskId taskId = new TaskId(TaskCategory.EVENT_IMPORT, currentUserService.getCurrentUser());
        List<Event> events = eventService.getEventsJson(inputStream);
        scheduler.executeTask(new ImportEventTask(events, eventService, importOptions, taskId));
        response.setHeader("Location", ContextUtils.getRootPath(request) + "/system/tasks/" + TaskCategory.EVENT_IMPORT);
        response.setStatus(HttpServletResponse.SC_NO_CONTENT);
    }
}

Example 3 with ImportStrategy

use of org.hisp.dhis.importexport.ImportStrategy in project dhis2-core by dhis2.

the class DefaultDataValueSetService method saveDataValueSet.

/**
     * There are specific id schemes for data elements and organisation units and
     * a generic id scheme for all objects. The specific id schemes will take
     * precedence over the generic id scheme. The generic id scheme also applies
     * to data set and category option combo.
     * <p>
     * The id schemes uses the following order of precedence:
     * <p>
     * <ul>
     * <li>Id scheme from the data value set</li>
     * <li>Id scheme from the import options</li>
     * <li>Default id scheme which is UID</li>
     * <ul>
     * <p>
     * If id scheme is specific in the data value set, any id schemes in the import
     * options will be ignored.
     *
     * @param importOptions
     * @param id
     * @param dataValueSet
     * @return
     */
private ImportSummary saveDataValueSet(ImportOptions importOptions, TaskId id, DataValueSet dataValueSet) {
    importOptions = ObjectUtils.firstNonNull(importOptions, ImportOptions.getDefaultImportOptions());
    Clock clock = new Clock(log).startClock().logTime("Starting data value import, options: " + importOptions);
    NotificationLevel notificationLevel = importOptions.getNotificationLevel(INFO);
    notifier.clear(id).notify(id, notificationLevel, "Process started");
    ImportSummary summary = new ImportSummary().setImportOptions(importOptions);
    boolean isIso8601 = calendarService.getSystemCalendar().isIso8601();
    boolean skipLockExceptionCheck = !lockExceptionStore.anyExists();
    log.info(String.format("Is ISO calendar: %b, skip lock exception check: %b", isIso8601, skipLockExceptionCheck));
    I18n i18n = i18nManager.getI18n();
    // ---------------------------------------------------------------------
    // Get import options
    // ---------------------------------------------------------------------
    log.info("Import options: " + importOptions);
    IdScheme dvSetIdScheme = IdScheme.from(dataValueSet.getIdSchemeProperty());
    IdScheme dvSetDataElementIdScheme = IdScheme.from(dataValueSet.getDataElementIdSchemeProperty());
    IdScheme dvSetOrgUnitIdScheme = IdScheme.from(dataValueSet.getOrgUnitIdSchemeProperty());
    IdScheme dvSetCategoryOptComboIdScheme = IdScheme.from(dataValueSet.getCategoryOptionComboIdSchemeProperty());
    IdScheme dvSetDataSetIdScheme = IdScheme.from(dataValueSet.getDataSetIdSchemeProperty());
    log.info("Data value set identifier scheme: " + dvSetIdScheme + ", data element: " + dvSetDataElementIdScheme + ", org unit: " + dvSetOrgUnitIdScheme + ", category option combo: " + dvSetCategoryOptComboIdScheme + ", data set: " + dvSetDataSetIdScheme);
    IdScheme idScheme = dvSetIdScheme.isNotNull() ? dvSetIdScheme : importOptions.getIdSchemes().getIdScheme();
    IdScheme dataElementIdScheme = dvSetDataElementIdScheme.isNotNull() ? dvSetDataElementIdScheme : importOptions.getIdSchemes().getDataElementIdScheme();
    IdScheme orgUnitIdScheme = dvSetOrgUnitIdScheme.isNotNull() ? dvSetOrgUnitIdScheme : importOptions.getIdSchemes().getOrgUnitIdScheme();
    IdScheme categoryOptComboIdScheme = dvSetCategoryOptComboIdScheme.isNotNull() ? dvSetCategoryOptComboIdScheme : importOptions.getIdSchemes().getCategoryOptionComboIdScheme();
    IdScheme dataSetIdScheme = dvSetDataSetIdScheme.isNotNull() ? dvSetDataSetIdScheme : importOptions.getIdSchemes().getDataSetIdScheme();
    log.info("Identifier scheme: " + idScheme + ", data element: " + dataElementIdScheme + ", org unit: " + orgUnitIdScheme + ", category option combo: " + categoryOptComboIdScheme + ", data set: " + dataSetIdScheme);
    ImportStrategy strategy = dataValueSet.getStrategy() != null ? ImportStrategy.valueOf(dataValueSet.getStrategy()) : importOptions.getImportStrategy();
    boolean dryRun = dataValueSet.getDryRun() != null ? dataValueSet.getDryRun() : importOptions.isDryRun();
    boolean skipExistingCheck = importOptions.isSkipExistingCheck();
    boolean strictPeriods = importOptions.isStrictPeriods() || (Boolean) systemSettingManager.getSystemSetting(SettingKey.DATA_IMPORT_STRICT_PERIODS);
    boolean strictCategoryOptionCombos = importOptions.isStrictCategoryOptionCombos() || (Boolean) systemSettingManager.getSystemSetting(SettingKey.DATA_IMPORT_STRICT_CATEGORY_OPTION_COMBOS);
    boolean strictAttrOptionCombos = importOptions.isStrictAttributeOptionCombos() || (Boolean) systemSettingManager.getSystemSetting(SettingKey.DATA_IMPORT_STRICT_ATTRIBUTE_OPTION_COMBOS);
    boolean strictOrgUnits = importOptions.isStrictOrganisationUnits() || (Boolean) systemSettingManager.getSystemSetting(SettingKey.DATA_IMPORT_STRICT_ORGANISATION_UNITS);
    boolean requireCategoryOptionCombo = importOptions.isRequireCategoryOptionCombo() || (Boolean) systemSettingManager.getSystemSetting(SettingKey.DATA_IMPORT_REQUIRE_CATEGORY_OPTION_COMBO);
    boolean requireAttrOptionCombo = importOptions.isRequireAttributeOptionCombo() || (Boolean) systemSettingManager.getSystemSetting(SettingKey.DATA_IMPORT_REQUIRE_ATTRIBUTE_OPTION_COMBO);
    // ---------------------------------------------------------------------
    // Create meta-data maps
    // ---------------------------------------------------------------------
    CachingMap<String, DataElement> dataElementMap = new CachingMap<>();
    CachingMap<String, OrganisationUnit> orgUnitMap = new CachingMap<>();
    CachingMap<String, DataElementCategoryOptionCombo> optionComboMap = new CachingMap<>();
    CachingMap<String, DataSet> dataElementDataSetMap = new CachingMap<>();
    CachingMap<String, Period> periodMap = new CachingMap<>();
    CachingMap<String, Set<PeriodType>> dataElementPeriodTypesMap = new CachingMap<>();
    CachingMap<String, Set<DataElementCategoryOptionCombo>> dataElementCategoryOptionComboMap = new CachingMap<>();
    CachingMap<String, Set<DataElementCategoryOptionCombo>> dataElementAttrOptionComboMap = new CachingMap<>();
    CachingMap<String, Boolean> dataElementOrgUnitMap = new CachingMap<>();
    CachingMap<String, Boolean> dataSetLockedMap = new CachingMap<>();
    CachingMap<String, Period> dataElementLatestFuturePeriodMap = new CachingMap<>();
    CachingMap<String, Boolean> orgUnitInHierarchyMap = new CachingMap<>();
    CachingMap<String, DateRange> attrOptionComboDateRangeMap = new CachingMap<>();
    CachingMap<String, Boolean> attrOptionComboOrgUnitMap = new CachingMap<>();
    CachingMap<String, Optional<Set<String>>> dataElementOptionsMap = new CachingMap<>();
    CachingMap<String, Boolean> approvalMap = new CachingMap<>();
    CachingMap<String, Boolean> lowestApprovalLevelMap = new CachingMap<>();
    CachingMap<String, Boolean> periodOpenForDataElement = new CachingMap<>();
    // ---------------------------------------------------------------------
    // Get meta-data maps
    // ---------------------------------------------------------------------
    IdentifiableObjectCallable<DataElement> dataElementCallable = new IdentifiableObjectCallable<>(identifiableObjectManager, DataElement.class, dataElementIdScheme, null);
    IdentifiableObjectCallable<OrganisationUnit> orgUnitCallable = new IdentifiableObjectCallable<>(identifiableObjectManager, OrganisationUnit.class, orgUnitIdScheme, trimToNull(dataValueSet.getOrgUnit()));
    IdentifiableObjectCallable<DataElementCategoryOptionCombo> categoryOptionComboCallable = new CategoryOptionComboAclCallable(categoryService, categoryOptComboIdScheme, null);
    IdentifiableObjectCallable<DataElementCategoryOptionCombo> attributeOptionComboCallable = new CategoryOptionComboAclCallable(categoryService, categoryOptComboIdScheme, null);
    IdentifiableObjectCallable<Period> periodCallable = new PeriodCallable(periodService, null, trimToNull(dataValueSet.getPeriod()));
    if (importOptions.isPreheatCacheDefaultFalse()) {
        dataElementMap.load(identifiableObjectManager.getAll(DataElement.class), o -> o.getPropertyValue(dataElementIdScheme));
        orgUnitMap.load(identifiableObjectManager.getAll(OrganisationUnit.class), o -> o.getPropertyValue(orgUnitIdScheme));
        optionComboMap.load(identifiableObjectManager.getAll(DataElementCategoryOptionCombo.class), o -> o.getPropertyValue(categoryOptComboIdScheme));
    }
    // ---------------------------------------------------------------------
    // Get outer meta-data
    // ---------------------------------------------------------------------
    DataSet dataSet = dataValueSet.getDataSet() != null ? identifiableObjectManager.getObject(DataSet.class, dataSetIdScheme, dataValueSet.getDataSet()) : null;
    Date completeDate = parseDate(dataValueSet.getCompleteDate());
    Period outerPeriod = periodMap.get(trimToNull(dataValueSet.getPeriod()), periodCallable);
    OrganisationUnit outerOrgUnit = orgUnitMap.get(trimToNull(dataValueSet.getOrgUnit()), orgUnitCallable);
    DataElementCategoryOptionCombo fallbackCategoryOptionCombo = categoryService.getDefaultDataElementCategoryOptionCombo();
    DataElementCategoryOptionCombo outerAttrOptionCombo = null;
    if (dataValueSet.getAttributeOptionCombo() != null) {
        outerAttrOptionCombo = optionComboMap.get(trimToNull(dataValueSet.getAttributeOptionCombo()), attributeOptionComboCallable.setId(trimToNull(dataValueSet.getAttributeOptionCombo())));
    } else if (dataValueSet.getAttributeCategoryOptions() != null) {
        outerAttrOptionCombo = inputUtils.getAttributeOptionCombo(dataSet.getCategoryCombo(), new HashSet<String>(dataValueSet.getAttributeCategoryOptions()), idScheme);
    }
    if (dataSet == null && trimToNull(dataValueSet.getDataSet()) != null) {
        summary.getConflicts().add(new ImportConflict(dataValueSet.getDataSet(), "Data set not found or not accessible"));
        summary.setStatus(ImportStatus.ERROR);
    }
    if (outerOrgUnit == null && trimToNull(dataValueSet.getOrgUnit()) != null) {
        summary.getConflicts().add(new ImportConflict(dataValueSet.getDataSet(), "Org unit not found or not accessible"));
        summary.setStatus(ImportStatus.ERROR);
    }
    if (outerAttrOptionCombo == null && trimToNull(dataValueSet.getAttributeOptionCombo()) != null) {
        summary.getConflicts().add(new ImportConflict(dataValueSet.getDataSet(), "Attribute option combo not found or not accessible"));
        summary.setStatus(ImportStatus.ERROR);
    }
    if (ImportStatus.ERROR.equals(summary.getStatus())) {
        summary.setDescription("Import process was aborted");
        notifier.notify(id, WARN, "Import process aborted", true).addTaskSummary(id, summary);
        dataValueSet.close();
        return summary;
    }
    if (dataSet != null && completeDate != null) {
        notifier.notify(id, notificationLevel, "Completing data set");
        //TODO
        handleComplete(dataSet, completeDate, outerPeriod, outerOrgUnit, fallbackCategoryOptionCombo, summary);
    } else {
        summary.setDataSetComplete(Boolean.FALSE.toString());
    }
    final String currentUser = currentUserService.getCurrentUsername();
    final Set<OrganisationUnit> currentOrgUnits = currentUserService.getCurrentUserOrganisationUnits();
    BatchHandler<DataValue> dataValueBatchHandler = batchHandlerFactory.createBatchHandler(DataValueBatchHandler.class).init();
    BatchHandler<DataValueAudit> auditBatchHandler = batchHandlerFactory.createBatchHandler(DataValueAuditBatchHandler.class).init();
    int importCount = 0;
    int updateCount = 0;
    int deleteCount = 0;
    int totalCount = 0;
    // ---------------------------------------------------------------------
    // Data values
    // ---------------------------------------------------------------------
    Date now = new Date();
    clock.logTime("Validated outer meta-data");
    notifier.notify(id, notificationLevel, "Importing data values");
    while (dataValueSet.hasNextDataValue()) {
        org.hisp.dhis.dxf2.datavalue.DataValue dataValue = dataValueSet.getNextDataValue();
        totalCount++;
        final DataElement dataElement = dataElementMap.get(trimToNull(dataValue.getDataElement()), dataElementCallable.setId(trimToNull(dataValue.getDataElement())));
        final Period period = outerPeriod != null ? outerPeriod : periodMap.get(trimToNull(dataValue.getPeriod()), periodCallable.setId(trimToNull(dataValue.getPeriod())));
        final OrganisationUnit orgUnit = outerOrgUnit != null ? outerOrgUnit : orgUnitMap.get(trimToNull(dataValue.getOrgUnit()), orgUnitCallable.setId(trimToNull(dataValue.getOrgUnit())));
        DataElementCategoryOptionCombo categoryOptionCombo = optionComboMap.get(trimToNull(dataValue.getCategoryOptionCombo()), categoryOptionComboCallable.setId(trimToNull(dataValue.getCategoryOptionCombo())));
        DataElementCategoryOptionCombo attrOptionCombo = outerAttrOptionCombo != null ? outerAttrOptionCombo : optionComboMap.get(trimToNull(dataValue.getAttributeOptionCombo()), attributeOptionComboCallable.setId(trimToNull(dataValue.getAttributeOptionCombo())));
        if (!dataElementMap.isCacheLoaded() && dataElementMap.getCacheMissCount() > CACHE_MISS_THRESHOLD) {
            dataElementMap.load(identifiableObjectManager.getAll(DataElement.class), o -> o.getPropertyValue(dataElementIdScheme));
            log.info("Data element cache heated after cache miss threshold reached");
        }
        if (!orgUnitMap.isCacheLoaded() && orgUnitMap.getCacheMissCount() > CACHE_MISS_THRESHOLD) {
            orgUnitMap.load(identifiableObjectManager.getAll(OrganisationUnit.class), o -> o.getPropertyValue(orgUnitIdScheme));
            log.info("Org unit cache heated after cache miss threshold reached");
        }
        if (dataElement == null) {
            summary.getConflicts().add(new ImportConflict(dataValue.getDataElement(), "Data element not found or not accessible"));
            continue;
        }
        if (period == null) {
            summary.getConflicts().add(new ImportConflict(dataValue.getPeriod(), "Period not valid"));
            continue;
        }
        if (orgUnit == null) {
            summary.getConflicts().add(new ImportConflict(dataValue.getOrgUnit(), "Organisation unit not found or not accessible"));
            continue;
        }
        if (categoryOptionCombo == null && trimToNull(dataValue.getCategoryOptionCombo()) != null) {
            summary.getConflicts().add(new ImportConflict(dataValue.getCategoryOptionCombo(), "Category option combo not found or not accessible"));
            continue;
        }
        if (attrOptionCombo == null && trimToNull(dataValue.getAttributeOptionCombo()) != null) {
            summary.getConflicts().add(new ImportConflict(dataValue.getAttributeOptionCombo(), "Attribute option combo not found or not accessible"));
            continue;
        }
        boolean inUserHierarchy = orgUnitInHierarchyMap.get(orgUnit.getUid(), () -> orgUnit.isDescendant(currentOrgUnits));
        if (!inUserHierarchy) {
            summary.getConflicts().add(new ImportConflict(orgUnit.getUid(), "Organisation unit not in hierarchy of current user: " + currentUser));
            continue;
        }
        if (dataValue.isNullValue() && !dataValue.isDeletedValue()) {
            summary.getConflicts().add(new ImportConflict("Value", "Data value or comment not specified for data element: " + dataElement.getUid()));
            continue;
        }
        dataValue.setValueForced(ValidationUtils.normalizeBoolean(dataValue.getValue(), dataElement.getValueType()));
        String valueValid = ValidationUtils.dataValueIsValid(dataValue.getValue(), dataElement);
        if (valueValid != null) {
            summary.getConflicts().add(new ImportConflict(dataValue.getValue(), i18n.getString(valueValid) + ", must match data element type: " + dataElement.getUid()));
            continue;
        }
        String commentValid = ValidationUtils.commentIsValid(dataValue.getComment());
        if (commentValid != null) {
            summary.getConflicts().add(new ImportConflict("Comment", i18n.getString(commentValid)));
            continue;
        }
        Optional<Set<String>> optionCodes = dataElementOptionsMap.get(dataElement.getUid(), () -> dataElement.hasOptionSet() ? Optional.of(dataElement.getOptionSet().getOptionCodesAsSet()) : Optional.empty());
        if (optionCodes.isPresent() && !optionCodes.get().contains(dataValue.getValue())) {
            summary.getConflicts().add(new ImportConflict(dataValue.getValue(), "Data value is not a valid option of the data element option set: " + dataElement.getUid()));
            continue;
        }
        if (categoryOptionCombo == null) {
            if (requireCategoryOptionCombo) {
                summary.getConflicts().add(new ImportConflict(dataValue.getValue(), "Category option combo is required but is not specified"));
                continue;
            } else {
                categoryOptionCombo = fallbackCategoryOptionCombo;
            }
        }
        if (attrOptionCombo == null) {
            if (requireAttrOptionCombo) {
                summary.getConflicts().add(new ImportConflict(dataValue.getValue(), "Attribute option combo is required but is not specified"));
                continue;
            } else {
                attrOptionCombo = fallbackCategoryOptionCombo;
            }
        }
        if (strictPeriods && !dataElementPeriodTypesMap.get(dataElement.getUid(), () -> dataElement.getPeriodTypes()).contains(period.getPeriodType())) {
            summary.getConflicts().add(new ImportConflict(dataValue.getPeriod(), "Period type of period: " + period.getIsoDate() + " not valid for data element: " + dataElement.getUid()));
            continue;
        }
        if (strictCategoryOptionCombos && !dataElementCategoryOptionComboMap.get(dataElement.getUid(), () -> dataElement.getCategoryOptionCombos()).contains(categoryOptionCombo)) {
            summary.getConflicts().add(new ImportConflict(categoryOptionCombo.getUid(), "Category option combo: " + categoryOptionCombo.getUid() + " must be part of category combo of data element: " + dataElement.getUid()));
            continue;
        }
        if (strictAttrOptionCombos && !dataElementAttrOptionComboMap.get(dataElement.getUid(), () -> dataElement.getDataSetCategoryOptionCombos()).contains(attrOptionCombo)) {
            summary.getConflicts().add(new ImportConflict(attrOptionCombo.getUid(), "Attribute option combo: " + attrOptionCombo.getUid() + " must be part of category combo of data sets of data element: " + dataElement.getUid()));
            continue;
        }
        if (strictOrgUnits && BooleanUtils.isFalse(dataElementOrgUnitMap.get(dataElement.getUid() + orgUnit.getUid(), () -> orgUnit.hasDataElement(dataElement)))) {
            summary.getConflicts().add(new ImportConflict(orgUnit.getUid(), "Data element: " + dataElement.getUid() + " must be assigned through data sets to organisation unit: " + orgUnit.getUid()));
            continue;
        }
        boolean zeroInsignificant = ValidationUtils.dataValueIsZeroAndInsignificant(dataValue.getValue(), dataElement);
        if (zeroInsignificant) {
            summary.getConflicts().add(new ImportConflict(dataValue.getValue(), "Value is zero and not significant, must match data element: " + dataElement.getUid()));
            continue;
        }
        String storedByValid = ValidationUtils.storedByIsValid(dataValue.getStoredBy());
        if (storedByValid != null) {
            summary.getConflicts().add(new ImportConflict(dataValue.getStoredBy(), i18n.getString(storedByValid)));
            continue;
        }
        String storedBy = dataValue.getStoredBy() == null || dataValue.getStoredBy().trim().isEmpty() ? currentUser : dataValue.getStoredBy();
        final DataElementCategoryOptionCombo aoc = attrOptionCombo;
        DateRange aocDateRange = attrOptionComboDateRangeMap.get(attrOptionCombo.getUid(), () -> aoc.getDateRange());
        if ((aocDateRange.getStartDate() != null && aocDateRange.getStartDate().compareTo(period.getStartDate()) > 0) || (aocDateRange.getEndDate() != null && aocDateRange.getEndDate().compareTo(period.getEndDate()) < 0)) {
            summary.getConflicts().add(new ImportConflict(orgUnit.getUid(), "Period: " + period.getIsoDate() + " is not within date range of attribute option combo: " + attrOptionCombo.getUid()));
            continue;
        }
        if (!attrOptionComboOrgUnitMap.get(attrOptionCombo.getUid() + orgUnit.getUid(), () -> {
            Set<OrganisationUnit> aocOrgUnits = aoc.getOrganisationUnits();
            return aocOrgUnits == null || orgUnit.isDescendant(aocOrgUnits);
        })) {
            summary.getConflicts().add(new ImportConflict(orgUnit.getUid(), "Organisation unit: " + orgUnit.getUid() + " is not valid for attribute option combo: " + attrOptionCombo.getUid()));
            continue;
        }
        final DataSet approvalDataSet = dataSet != null ? dataSet : dataElementDataSetMap.get(dataElement.getUid(), () -> dataElement.getApprovalDataSet());
        if (// Data element is assigned to at least one data set
        approvalDataSet != null) {
            if (dataSetLockedMap.get(approvalDataSet.getUid() + period.getUid() + orgUnit.getUid(), () -> isLocked(approvalDataSet, period, orgUnit, skipLockExceptionCheck))) {
                summary.getConflicts().add(new ImportConflict(period.getIsoDate(), "Current date is past expiry days for period " + period.getIsoDate() + " and data set: " + approvalDataSet.getUid()));
                continue;
            }
            Period latestFuturePeriod = dataElementLatestFuturePeriodMap.get(dataElement.getUid(), () -> dataElement.getLatestOpenFuturePeriod());
            if (period.isAfter(latestFuturePeriod) && isIso8601) {
                summary.getConflicts().add(new ImportConflict(period.getIsoDate(), "Period: " + period.getIsoDate() + " is after latest open future period: " + latestFuturePeriod.getIsoDate() + " for data element: " + dataElement.getUid()));
                continue;
            }
            DataApprovalWorkflow workflow = approvalDataSet.getWorkflow();
            if (workflow != null) {
                final String workflowPeriodAoc = workflow.getUid() + period.getUid() + attrOptionCombo.getUid();
                if (approvalMap.get(orgUnit.getUid() + workflowPeriodAoc, () -> {
                    DataApproval lowestApproval = DataApproval.getLowestApproval(new DataApproval(null, workflow, period, orgUnit, aoc));
                    return lowestApprovalLevelMap.get(lowestApproval.getDataApprovalLevel().getUid() + lowestApproval.getOrganisationUnit().getUid() + workflowPeriodAoc, () -> approvalService.getDataApproval(lowestApproval) != null);
                })) {
                    summary.getConflicts().add(new ImportConflict(orgUnit.getUid(), "Data is already approved for data set: " + approvalDataSet.getUid() + " period: " + period.getIsoDate() + " organisation unit: " + orgUnit.getUid() + " attribute option combo: " + attrOptionCombo.getUid()));
                    continue;
                }
            }
        }
        if (approvalDataSet != null && !approvalDataSet.isDataInputPeriodAndDateAllowed(period, new Date())) {
            summary.getConflicts().add(new ImportConflict(orgUnit.getUid(), "Period: " + period.getIsoDate() + " is not open for this data set at this time: " + approvalDataSet.getUid()));
            continue;
        }
        if (!periodOpenForDataElement.get(dataElement.getUid() + period.getIsoDate(), () -> dataElement.isDataInputAllowedForPeriodAndDate(period, new Date()))) {
            summary.getConflicts().add(new ImportConflict(orgUnit.getUid(), "Period " + period.getName() + " does not conform to the open periods of associated data sets"));
            continue;
        }
        // -----------------------------------------------------------------
        // Create data value
        // -----------------------------------------------------------------
        DataValue internalValue = new DataValue();
        internalValue.setDataElement(dataElement);
        internalValue.setPeriod(period);
        internalValue.setSource(orgUnit);
        internalValue.setCategoryOptionCombo(categoryOptionCombo);
        internalValue.setAttributeOptionCombo(attrOptionCombo);
        internalValue.setValue(trimToNull(dataValue.getValue()));
        internalValue.setStoredBy(storedBy);
        internalValue.setCreated(dataValue.hasCreated() ? parseDate(dataValue.getCreated()) : now);
        internalValue.setLastUpdated(dataValue.hasLastUpdated() ? parseDate(dataValue.getLastUpdated()) : now);
        internalValue.setComment(trimToNull(dataValue.getComment()));
        internalValue.setFollowup(dataValue.getFollowup());
        internalValue.setDeleted(BooleanUtils.isTrue(dataValue.getDeleted()));
        // -----------------------------------------------------------------
        // Save, update or delete data value
        // -----------------------------------------------------------------
        DataValue existingValue = !skipExistingCheck ? dataValueBatchHandler.findObject(internalValue) : null;
        if (!skipExistingCheck && existingValue != null && !existingValue.isDeleted()) {
            if (strategy.isCreateAndUpdate() || strategy.isUpdate()) {
                DataValueAudit auditValue = new DataValueAudit(internalValue, existingValue.getValue(), storedBy, AuditType.UPDATE);
                if (internalValue.isNullValue() || internalValue.isDeleted()) {
                    internalValue.setDeleted(true);
                    auditValue.setAuditType(AuditType.DELETE);
                    deleteCount++;
                } else {
                    updateCount++;
                }
                if (!dryRun) {
                    dataValueBatchHandler.updateObject(internalValue);
                    auditBatchHandler.addObject(auditValue);
                }
            } else if (strategy.isDelete()) {
                DataValueAudit auditValue = new DataValueAudit(internalValue, existingValue.getValue(), storedBy, AuditType.DELETE);
                internalValue.setDeleted(true);
                deleteCount++;
                if (!dryRun) {
                    dataValueBatchHandler.updateObject(internalValue);
                    auditBatchHandler.addObject(auditValue);
                }
            }
        } else {
            if (strategy.isCreateAndUpdate() || strategy.isCreate()) {
                if (// Ignore null values
                !internalValue.isNullValue()) {
                    if (existingValue != null && existingValue.isDeleted()) {
                        importCount++;
                        if (!dryRun) {
                            dataValueBatchHandler.updateObject(internalValue);
                        }
                    } else {
                        boolean added = false;
                        if (!dryRun) {
                            added = dataValueBatchHandler.addObject(internalValue);
                        }
                        if (dryRun || added) {
                            importCount++;
                        }
                    }
                }
            }
        }
    }
    dataValueBatchHandler.flush();
    auditBatchHandler.flush();
    int ignores = totalCount - importCount - updateCount - deleteCount;
    summary.setImportCount(new ImportCount(importCount, updateCount, ignores, deleteCount));
    summary.setStatus(summary.getConflicts().isEmpty() ? ImportStatus.SUCCESS : ImportStatus.WARNING);
    summary.setDescription("Import process completed successfully");
    clock.logTime("Data value import done, total: " + totalCount + ", import: " + importCount + ", update: " + updateCount + ", delete: " + deleteCount);
    notifier.notify(id, notificationLevel, "Import done", true).addTaskSummary(id, notificationLevel, summary);
    dataValueSet.close();
    return summary;
}

Example 4 with ImportStrategy

use of org.hisp.dhis.importexport.ImportStrategy in project dhis2-core by dhis2.

the class DefaultCompleteDataSetRegistrationExchangeService method batchImport.

/**
     * @return total number of processed CompleteDataSetRegistration objects
     */
private int batchImport(CompleteDataSetRegistrations completeRegistrations, ImportConfig config, ImportSummary summary, MetaDataCallables mdCallables, MetaDataCaches mdCaches) {
    final String currentUser = currentUserService.getCurrentUsername();
    final Set<OrganisationUnit> userOrgUnits = currentUserService.getCurrentUserOrganisationUnits();
    final I18n i18n = i18nManager.getI18n();
    BatchHandler<CompleteDataSetRegistration> batchHandler = batchHandlerFactory.createBatchHandler(CompleteDataSetRegistrationBatchHandler.class).init();
    int importCount = 0, updateCount = 0, deleteCount = 0, totalCount = 0;
    Date now = new Date();
    while (completeRegistrations.hasNextCompleteDataSetRegistration()) {
        org.hisp.dhis.dxf2.dataset.CompleteDataSetRegistration cdsr = completeRegistrations.getNextCompleteDataSetRegistration();
        totalCount++;
        // ---------------------------------------------------------------------
        // Init meta-data properties against meta-data cache
        // ---------------------------------------------------------------------
        MetaDataProperties mdProps = initMetaDataProperties(cdsr, mdCallables, mdCaches);
        heatCaches(mdCaches, config);
        // ---------------------------------------------------------------------
        // Meta-data validation
        // ---------------------------------------------------------------------
        String storedBy;
        try {
            // Validate CDSR meta-data properties
            mdProps.validate(cdsr, config);
            validateOrgUnitInUserHierarchy(mdCaches, mdProps, userOrgUnits, currentUser);
            if (config.strictAttrOptionCombos) {
                validateAocMatchesDataSetCc(mdProps);
            }
            validateAttrOptCombo(mdProps, mdCaches, config);
            if (config.strictPeriods) {
                validateHasMatchingPeriodTypes(mdProps);
            }
            if (config.strictOrgUnits) {
                validateDataSetIsAssignedToOrgUnit(mdProps);
            }
            storedBy = cdsr.getStoredBy();
            validateStoredBy(storedBy, i18n);
            storedBy = StringUtils.isBlank(storedBy) ? currentUser : storedBy;
        // TODO Check if Period is within range of data set?
        } catch (ImportConflictException ic) {
            summary.getConflicts().add(ic.getImportConflict());
            continue;
        }
        // -----------------------------------------------------------------
        // Create complete data set registration
        // -----------------------------------------------------------------
        CompleteDataSetRegistration internalCdsr = createCompleteDataSetRegistration(cdsr, mdProps, now, storedBy);
        CompleteDataSetRegistration existingCdsr = config.skipExistingCheck ? null : batchHandler.findObject(internalCdsr);
        ImportStrategy strategy = config.strategy;
        boolean isDryRun = config.dryRun;
        if (!config.skipExistingCheck && existingCdsr != null) {
            if (strategy.isCreateAndUpdate() || strategy.isUpdate()) {
                // Update existing CDSR
                updateCount++;
                if (!isDryRun) {
                    batchHandler.updateObject(internalCdsr);
                }
            } else if (strategy.isDelete()) {
                // TODO Does 'delete' even make sense for CDSR?
                // Replace existing CDSR
                deleteCount++;
                if (!isDryRun) {
                    batchHandler.deleteObject(internalCdsr);
                }
            }
        } else {
            if (strategy.isCreateAndUpdate() || strategy.isCreate()) {
                if (existingCdsr != null) {
                    // Already exists -> update
                    importCount++;
                    if (!isDryRun) {
                        batchHandler.updateObject(internalCdsr);
                    }
                } else {
                    // Does not exist -> add new CDSR
                    boolean added = false;
                    if (!isDryRun) {
                        added = batchHandler.addObject(internalCdsr);
                    }
                    if (isDryRun || added) {
                        importCount++;
                    }
                }
            }
        }
    }
    batchHandler.flush();
    finalizeSummary(summary, totalCount, importCount, updateCount, deleteCount);
    return totalCount;
}

Example 5 with ImportStrategy

use of org.hisp.dhis.importexport.ImportStrategy in project dhis2-core by dhis2.

the class EventController method postXmlEvent.

// -------------------------------------------------------------------------
// CREATE
// -------------------------------------------------------------------------
@RequestMapping(method = RequestMethod.POST, consumes = "application/xml")
@PreAuthorize("hasRole('ALL') or hasRole('F_TRACKED_ENTITY_DATAVALUE_ADD')")
public void postXmlEvent(@RequestParam(defaultValue = "CREATE") ImportStrategy strategy, HttpServletResponse response, HttpServletRequest request, ImportOptions importOptions) throws Exception {
    importOptions.setImportStrategy(strategy);
    InputStream inputStream = StreamUtils.wrapAndCheckCompressionFormat(request.getInputStream());
    importOptions.setIdSchemes(getIdSchemesFromParameters(importOptions.getIdSchemes(), contextService.getParameterValuesMap()));
    if (!importOptions.isAsync()) {
        ImportSummaries importSummaries = eventService.addEventsXml(inputStream, importOptions);
        importSummaries.setImportOptions(importOptions);
        importSummaries.getImportSummaries().stream().filter(importSummary -> !importOptions.isDryRun() && !importSummary.getStatus().equals(ImportStatus.ERROR) && !importOptions.getImportStrategy().isDelete()).forEach(importSummary -> importSummary.setHref(ContextUtils.getRootPath(request) + RESOURCE_PATH + "/" + importSummary.getReference()));
        if (importSummaries.getImportSummaries().size() == 1) {
            ImportSummary importSummary = importSummaries.getImportSummaries().get(0);
            importSummary.setImportOptions(importOptions);
            if (!importOptions.isDryRun()) {
                if (!importSummary.getStatus().equals(ImportStatus.ERROR)) {
                    response.setHeader("Location", ContextUtils.getRootPath(request) + RESOURCE_PATH + "/" + importSummary.getReference());
                }
            }
        }
        webMessageService.send(WebMessageUtils.importSummaries(importSummaries), response, request);
    } else {
        TaskId taskId = new TaskId(TaskCategory.EVENT_IMPORT, currentUserService.getCurrentUser());
        List<Event> events = eventService.getEventsXml(inputStream);
        scheduler.executeTask(new ImportEventTask(events, eventService, importOptions, taskId));
        response.setHeader("Location", ContextUtils.getRootPath(request) + "/system/tasks/" + TaskCategory.EVENT_IMPORT);
        response.setStatus(HttpServletResponse.SC_NO_CONTENT);
    }
}