use of org.hisp.dhis.dxf2.importsummary.ImportConflict in project dhis2-core by dhis2.
the class DefaultAdxDataService method saveDataValueSetInternal.
private ImportSummary saveDataValueSetInternal(InputStream in, ImportOptions importOptions, TaskId id) {
notifier.clear(id).notify(id, "ADX parsing process started");
ImportOptions adxImportOptions = ObjectUtils.firstNonNull(importOptions, ImportOptions.getDefaultImportOptions()).instance().setNotificationLevel(NotificationLevel.OFF);
// Get import options
IdScheme dataSetIdScheme = importOptions.getIdSchemes().getDataSetIdScheme();
IdScheme dataElementIdScheme = importOptions.getIdSchemes().getDataElementIdScheme();
// Create meta-data maps
CachingMap<String, DataSet> dataSetMap = new CachingMap<>();
CachingMap<String, DataElement> dataElementMap = new CachingMap<>();
// Get meta-data maps
IdentifiableObjectCallable<DataSet> dataSetCallable = new IdentifiableObjectCallable<>(identifiableObjectManager, DataSet.class, dataSetIdScheme, null);
IdentifiableObjectCallable<DataElement> dataElementCallable = new IdentifiableObjectCallable<>(identifiableObjectManager, DataElement.class, dataElementIdScheme, null);
// Heat cache
if (importOptions.isPreheatCacheDefaultFalse()) {
dataSetMap.load(identifiableObjectManager.getAll(DataSet.class), o -> o.getPropertyValue(dataSetIdScheme));
dataElementMap.load(identifiableObjectManager.getAll(DataElement.class), o -> o.getPropertyValue(dataElementIdScheme));
}
XMLReader adxReader = XMLFactory.getXMLReader(in);
ImportSummary importSummary;
adxReader.moveToStartElement(AdxDataService.ROOT, AdxDataService.NAMESPACE);
ExecutorService executor = Executors.newSingleThreadExecutor();
// Give the DXF import a different notification task ID so it doesn't conflict with notifications from this level.
TaskId dxfTaskId = new TaskId(TaskCategory.DATAVALUE_IMPORT_INTERNAL, id.getUser());
int groupCount = 0;
try (PipedOutputStream pipeOut = new PipedOutputStream()) {
Future<ImportSummary> futureImportSummary = executor.submit(new AdxPipedImporter(dataValueSetService, adxImportOptions, dxfTaskId, pipeOut, sessionFactory));
XMLOutputFactory factory = XMLOutputFactory.newInstance();
XMLStreamWriter dxfWriter = factory.createXMLStreamWriter(pipeOut);
List<ImportConflict> adxConflicts = new LinkedList<>();
dxfWriter.writeStartDocument("1.0");
dxfWriter.writeStartElement("dataValueSet");
dxfWriter.writeDefaultNamespace("http://dhis2.org/schema/dxf/2.0");
notifier.notify(id, "Starting to import ADX data groups.");
while (adxReader.moveToStartElement(AdxDataService.GROUP, AdxDataService.NAMESPACE)) {
notifier.update(id, "Importing ADX data group: " + groupCount);
// note this returns conflicts which are detected at ADX level
adxConflicts.addAll(parseAdxGroupToDxf(adxReader, dxfWriter, adxImportOptions, dataSetMap, dataSetCallable, dataElementMap, dataElementCallable));
groupCount++;
}
// end dataValueSet
dxfWriter.writeEndElement();
dxfWriter.writeEndDocument();
pipeOut.flush();
importSummary = futureImportSummary.get(TOTAL_MINUTES_TO_WAIT, TimeUnit.MINUTES);
importSummary.getConflicts().addAll(adxConflicts);
importSummary.getImportCount().incrementIgnored(adxConflicts.size());
} catch (AdxException ex) {
importSummary = new ImportSummary();
importSummary.setStatus(ImportStatus.ERROR);
importSummary.setDescription("Data set import failed within group number: " + groupCount);
importSummary.getConflicts().add(ex.getImportConflict());
notifier.update(id, NotificationLevel.ERROR, "ADX data import done", true);
log.warn("Import failed: " + DebugUtils.getStackTrace(ex));
} catch (IOException | XMLStreamException | InterruptedException | ExecutionException | TimeoutException ex) {
importSummary = new ImportSummary();
importSummary.setStatus(ImportStatus.ERROR);
importSummary.setDescription("Data set import failed within group number: " + groupCount);
notifier.update(id, NotificationLevel.ERROR, "ADX data import done", true);
log.warn("Import failed: " + DebugUtils.getStackTrace(ex));
}
executor.shutdown();
notifier.update(id, INFO, "ADX data import done", true).addTaskSummary(id, importSummary);
ImportCount c = importSummary.getImportCount();
log.info("ADX data import done, imported: " + c.getImported() + ", updated: " + c.getUpdated() + ", deleted: " + c.getDeleted() + ", ignored: " + c.getIgnored());
return importSummary;
}
use of org.hisp.dhis.dxf2.importsummary.ImportConflict in project dhis2-core by dhis2.
the class DefaultDataValueSetService method saveDataValueSet.
/**
* There are specific id schemes for data elements and organisation units and
* a generic id scheme for all objects. The specific id schemes will take
* precedence over the generic id scheme. The generic id scheme also applies
* to data set and category option combo.
* <p>
* The id schemes uses the following order of precedence:
* <p>
* <ul>
* <li>Id scheme from the data value set</li>
* <li>Id scheme from the import options</li>
* <li>Default id scheme which is UID</li>
* <ul>
* <p>
* If id scheme is specific in the data value set, any id schemes in the import
* options will be ignored.
*
* @param importOptions
* @param id
* @param dataValueSet
* @return
*/
private ImportSummary saveDataValueSet(ImportOptions importOptions, TaskId id, DataValueSet dataValueSet) {
importOptions = ObjectUtils.firstNonNull(importOptions, ImportOptions.getDefaultImportOptions());
Clock clock = new Clock(log).startClock().logTime("Starting data value import, options: " + importOptions);
NotificationLevel notificationLevel = importOptions.getNotificationLevel(INFO);
notifier.clear(id).notify(id, notificationLevel, "Process started");
ImportSummary summary = new ImportSummary().setImportOptions(importOptions);
boolean isIso8601 = calendarService.getSystemCalendar().isIso8601();
boolean skipLockExceptionCheck = !lockExceptionStore.anyExists();
log.info(String.format("Is ISO calendar: %b, skip lock exception check: %b", isIso8601, skipLockExceptionCheck));
I18n i18n = i18nManager.getI18n();
// ---------------------------------------------------------------------
// Get import options
// ---------------------------------------------------------------------
log.info("Import options: " + importOptions);
IdScheme dvSetIdScheme = IdScheme.from(dataValueSet.getIdSchemeProperty());
IdScheme dvSetDataElementIdScheme = IdScheme.from(dataValueSet.getDataElementIdSchemeProperty());
IdScheme dvSetOrgUnitIdScheme = IdScheme.from(dataValueSet.getOrgUnitIdSchemeProperty());
IdScheme dvSetCategoryOptComboIdScheme = IdScheme.from(dataValueSet.getCategoryOptionComboIdSchemeProperty());
IdScheme dvSetDataSetIdScheme = IdScheme.from(dataValueSet.getDataSetIdSchemeProperty());
log.info("Data value set identifier scheme: " + dvSetIdScheme + ", data element: " + dvSetDataElementIdScheme + ", org unit: " + dvSetOrgUnitIdScheme + ", category option combo: " + dvSetCategoryOptComboIdScheme + ", data set: " + dvSetDataSetIdScheme);
IdScheme idScheme = dvSetIdScheme.isNotNull() ? dvSetIdScheme : importOptions.getIdSchemes().getIdScheme();
IdScheme dataElementIdScheme = dvSetDataElementIdScheme.isNotNull() ? dvSetDataElementIdScheme : importOptions.getIdSchemes().getDataElementIdScheme();
IdScheme orgUnitIdScheme = dvSetOrgUnitIdScheme.isNotNull() ? dvSetOrgUnitIdScheme : importOptions.getIdSchemes().getOrgUnitIdScheme();
IdScheme categoryOptComboIdScheme = dvSetCategoryOptComboIdScheme.isNotNull() ? dvSetCategoryOptComboIdScheme : importOptions.getIdSchemes().getCategoryOptionComboIdScheme();
IdScheme dataSetIdScheme = dvSetDataSetIdScheme.isNotNull() ? dvSetDataSetIdScheme : importOptions.getIdSchemes().getDataSetIdScheme();
log.info("Identifier scheme: " + idScheme + ", data element: " + dataElementIdScheme + ", org unit: " + orgUnitIdScheme + ", category option combo: " + categoryOptComboIdScheme + ", data set: " + dataSetIdScheme);
ImportStrategy strategy = dataValueSet.getStrategy() != null ? ImportStrategy.valueOf(dataValueSet.getStrategy()) : importOptions.getImportStrategy();
boolean dryRun = dataValueSet.getDryRun() != null ? dataValueSet.getDryRun() : importOptions.isDryRun();
boolean skipExistingCheck = importOptions.isSkipExistingCheck();
boolean strictPeriods = importOptions.isStrictPeriods() || (Boolean) systemSettingManager.getSystemSetting(SettingKey.DATA_IMPORT_STRICT_PERIODS);
boolean strictCategoryOptionCombos = importOptions.isStrictCategoryOptionCombos() || (Boolean) systemSettingManager.getSystemSetting(SettingKey.DATA_IMPORT_STRICT_CATEGORY_OPTION_COMBOS);
boolean strictAttrOptionCombos = importOptions.isStrictAttributeOptionCombos() || (Boolean) systemSettingManager.getSystemSetting(SettingKey.DATA_IMPORT_STRICT_ATTRIBUTE_OPTION_COMBOS);
boolean strictOrgUnits = importOptions.isStrictOrganisationUnits() || (Boolean) systemSettingManager.getSystemSetting(SettingKey.DATA_IMPORT_STRICT_ORGANISATION_UNITS);
boolean requireCategoryOptionCombo = importOptions.isRequireCategoryOptionCombo() || (Boolean) systemSettingManager.getSystemSetting(SettingKey.DATA_IMPORT_REQUIRE_CATEGORY_OPTION_COMBO);
boolean requireAttrOptionCombo = importOptions.isRequireAttributeOptionCombo() || (Boolean) systemSettingManager.getSystemSetting(SettingKey.DATA_IMPORT_REQUIRE_ATTRIBUTE_OPTION_COMBO);
// ---------------------------------------------------------------------
// Create meta-data maps
// ---------------------------------------------------------------------
CachingMap<String, DataElement> dataElementMap = new CachingMap<>();
CachingMap<String, OrganisationUnit> orgUnitMap = new CachingMap<>();
CachingMap<String, DataElementCategoryOptionCombo> optionComboMap = new CachingMap<>();
CachingMap<String, DataSet> dataElementDataSetMap = new CachingMap<>();
CachingMap<String, Period> periodMap = new CachingMap<>();
CachingMap<String, Set<PeriodType>> dataElementPeriodTypesMap = new CachingMap<>();
CachingMap<String, Set<DataElementCategoryOptionCombo>> dataElementCategoryOptionComboMap = new CachingMap<>();
CachingMap<String, Set<DataElementCategoryOptionCombo>> dataElementAttrOptionComboMap = new CachingMap<>();
CachingMap<String, Boolean> dataElementOrgUnitMap = new CachingMap<>();
CachingMap<String, Boolean> dataSetLockedMap = new CachingMap<>();
CachingMap<String, Period> dataElementLatestFuturePeriodMap = new CachingMap<>();
CachingMap<String, Boolean> orgUnitInHierarchyMap = new CachingMap<>();
CachingMap<String, DateRange> attrOptionComboDateRangeMap = new CachingMap<>();
CachingMap<String, Boolean> attrOptionComboOrgUnitMap = new CachingMap<>();
CachingMap<String, Optional<Set<String>>> dataElementOptionsMap = new CachingMap<>();
CachingMap<String, Boolean> approvalMap = new CachingMap<>();
CachingMap<String, Boolean> lowestApprovalLevelMap = new CachingMap<>();
CachingMap<String, Boolean> periodOpenForDataElement = new CachingMap<>();
// ---------------------------------------------------------------------
// Get meta-data maps
// ---------------------------------------------------------------------
IdentifiableObjectCallable<DataElement> dataElementCallable = new IdentifiableObjectCallable<>(identifiableObjectManager, DataElement.class, dataElementIdScheme, null);
IdentifiableObjectCallable<OrganisationUnit> orgUnitCallable = new IdentifiableObjectCallable<>(identifiableObjectManager, OrganisationUnit.class, orgUnitIdScheme, trimToNull(dataValueSet.getOrgUnit()));
IdentifiableObjectCallable<DataElementCategoryOptionCombo> categoryOptionComboCallable = new CategoryOptionComboAclCallable(categoryService, categoryOptComboIdScheme, null);
IdentifiableObjectCallable<DataElementCategoryOptionCombo> attributeOptionComboCallable = new CategoryOptionComboAclCallable(categoryService, categoryOptComboIdScheme, null);
IdentifiableObjectCallable<Period> periodCallable = new PeriodCallable(periodService, null, trimToNull(dataValueSet.getPeriod()));
if (importOptions.isPreheatCacheDefaultFalse()) {
dataElementMap.load(identifiableObjectManager.getAll(DataElement.class), o -> o.getPropertyValue(dataElementIdScheme));
orgUnitMap.load(identifiableObjectManager.getAll(OrganisationUnit.class), o -> o.getPropertyValue(orgUnitIdScheme));
optionComboMap.load(identifiableObjectManager.getAll(DataElementCategoryOptionCombo.class), o -> o.getPropertyValue(categoryOptComboIdScheme));
}
// ---------------------------------------------------------------------
// Get outer meta-data
// ---------------------------------------------------------------------
DataSet dataSet = dataValueSet.getDataSet() != null ? identifiableObjectManager.getObject(DataSet.class, dataSetIdScheme, dataValueSet.getDataSet()) : null;
Date completeDate = parseDate(dataValueSet.getCompleteDate());
Period outerPeriod = periodMap.get(trimToNull(dataValueSet.getPeriod()), periodCallable);
OrganisationUnit outerOrgUnit = orgUnitMap.get(trimToNull(dataValueSet.getOrgUnit()), orgUnitCallable);
DataElementCategoryOptionCombo fallbackCategoryOptionCombo = categoryService.getDefaultDataElementCategoryOptionCombo();
DataElementCategoryOptionCombo outerAttrOptionCombo = null;
if (dataValueSet.getAttributeOptionCombo() != null) {
outerAttrOptionCombo = optionComboMap.get(trimToNull(dataValueSet.getAttributeOptionCombo()), attributeOptionComboCallable.setId(trimToNull(dataValueSet.getAttributeOptionCombo())));
} else if (dataValueSet.getAttributeCategoryOptions() != null) {
outerAttrOptionCombo = inputUtils.getAttributeOptionCombo(dataSet.getCategoryCombo(), new HashSet<String>(dataValueSet.getAttributeCategoryOptions()), idScheme);
}
if (dataSet == null && trimToNull(dataValueSet.getDataSet()) != null) {
summary.getConflicts().add(new ImportConflict(dataValueSet.getDataSet(), "Data set not found or not accessible"));
summary.setStatus(ImportStatus.ERROR);
}
if (outerOrgUnit == null && trimToNull(dataValueSet.getOrgUnit()) != null) {
summary.getConflicts().add(new ImportConflict(dataValueSet.getDataSet(), "Org unit not found or not accessible"));
summary.setStatus(ImportStatus.ERROR);
}
if (outerAttrOptionCombo == null && trimToNull(dataValueSet.getAttributeOptionCombo()) != null) {
summary.getConflicts().add(new ImportConflict(dataValueSet.getDataSet(), "Attribute option combo not found or not accessible"));
summary.setStatus(ImportStatus.ERROR);
}
if (ImportStatus.ERROR.equals(summary.getStatus())) {
summary.setDescription("Import process was aborted");
notifier.notify(id, WARN, "Import process aborted", true).addTaskSummary(id, summary);
dataValueSet.close();
return summary;
}
if (dataSet != null && completeDate != null) {
notifier.notify(id, notificationLevel, "Completing data set");
//TODO
handleComplete(dataSet, completeDate, outerPeriod, outerOrgUnit, fallbackCategoryOptionCombo, summary);
} else {
summary.setDataSetComplete(Boolean.FALSE.toString());
}
final String currentUser = currentUserService.getCurrentUsername();
final Set<OrganisationUnit> currentOrgUnits = currentUserService.getCurrentUserOrganisationUnits();
BatchHandler<DataValue> dataValueBatchHandler = batchHandlerFactory.createBatchHandler(DataValueBatchHandler.class).init();
BatchHandler<DataValueAudit> auditBatchHandler = batchHandlerFactory.createBatchHandler(DataValueAuditBatchHandler.class).init();
int importCount = 0;
int updateCount = 0;
int deleteCount = 0;
int totalCount = 0;
// ---------------------------------------------------------------------
// Data values
// ---------------------------------------------------------------------
Date now = new Date();
clock.logTime("Validated outer meta-data");
notifier.notify(id, notificationLevel, "Importing data values");
while (dataValueSet.hasNextDataValue()) {
org.hisp.dhis.dxf2.datavalue.DataValue dataValue = dataValueSet.getNextDataValue();
totalCount++;
final DataElement dataElement = dataElementMap.get(trimToNull(dataValue.getDataElement()), dataElementCallable.setId(trimToNull(dataValue.getDataElement())));
final Period period = outerPeriod != null ? outerPeriod : periodMap.get(trimToNull(dataValue.getPeriod()), periodCallable.setId(trimToNull(dataValue.getPeriod())));
final OrganisationUnit orgUnit = outerOrgUnit != null ? outerOrgUnit : orgUnitMap.get(trimToNull(dataValue.getOrgUnit()), orgUnitCallable.setId(trimToNull(dataValue.getOrgUnit())));
DataElementCategoryOptionCombo categoryOptionCombo = optionComboMap.get(trimToNull(dataValue.getCategoryOptionCombo()), categoryOptionComboCallable.setId(trimToNull(dataValue.getCategoryOptionCombo())));
DataElementCategoryOptionCombo attrOptionCombo = outerAttrOptionCombo != null ? outerAttrOptionCombo : optionComboMap.get(trimToNull(dataValue.getAttributeOptionCombo()), attributeOptionComboCallable.setId(trimToNull(dataValue.getAttributeOptionCombo())));
if (!dataElementMap.isCacheLoaded() && dataElementMap.getCacheMissCount() > CACHE_MISS_THRESHOLD) {
dataElementMap.load(identifiableObjectManager.getAll(DataElement.class), o -> o.getPropertyValue(dataElementIdScheme));
log.info("Data element cache heated after cache miss threshold reached");
}
if (!orgUnitMap.isCacheLoaded() && orgUnitMap.getCacheMissCount() > CACHE_MISS_THRESHOLD) {
orgUnitMap.load(identifiableObjectManager.getAll(OrganisationUnit.class), o -> o.getPropertyValue(orgUnitIdScheme));
log.info("Org unit cache heated after cache miss threshold reached");
}
if (dataElement == null) {
summary.getConflicts().add(new ImportConflict(dataValue.getDataElement(), "Data element not found or not accessible"));
continue;
}
if (period == null) {
summary.getConflicts().add(new ImportConflict(dataValue.getPeriod(), "Period not valid"));
continue;
}
if (orgUnit == null) {
summary.getConflicts().add(new ImportConflict(dataValue.getOrgUnit(), "Organisation unit not found or not accessible"));
continue;
}
if (categoryOptionCombo == null && trimToNull(dataValue.getCategoryOptionCombo()) != null) {
summary.getConflicts().add(new ImportConflict(dataValue.getCategoryOptionCombo(), "Category option combo not found or not accessible"));
continue;
}
if (attrOptionCombo == null && trimToNull(dataValue.getAttributeOptionCombo()) != null) {
summary.getConflicts().add(new ImportConflict(dataValue.getAttributeOptionCombo(), "Attribute option combo not found or not accessible"));
continue;
}
boolean inUserHierarchy = orgUnitInHierarchyMap.get(orgUnit.getUid(), () -> orgUnit.isDescendant(currentOrgUnits));
if (!inUserHierarchy) {
summary.getConflicts().add(new ImportConflict(orgUnit.getUid(), "Organisation unit not in hierarchy of current user: " + currentUser));
continue;
}
if (dataValue.isNullValue() && !dataValue.isDeletedValue()) {
summary.getConflicts().add(new ImportConflict("Value", "Data value or comment not specified for data element: " + dataElement.getUid()));
continue;
}
dataValue.setValueForced(ValidationUtils.normalizeBoolean(dataValue.getValue(), dataElement.getValueType()));
String valueValid = ValidationUtils.dataValueIsValid(dataValue.getValue(), dataElement);
if (valueValid != null) {
summary.getConflicts().add(new ImportConflict(dataValue.getValue(), i18n.getString(valueValid) + ", must match data element type: " + dataElement.getUid()));
continue;
}
String commentValid = ValidationUtils.commentIsValid(dataValue.getComment());
if (commentValid != null) {
summary.getConflicts().add(new ImportConflict("Comment", i18n.getString(commentValid)));
continue;
}
Optional<Set<String>> optionCodes = dataElementOptionsMap.get(dataElement.getUid(), () -> dataElement.hasOptionSet() ? Optional.of(dataElement.getOptionSet().getOptionCodesAsSet()) : Optional.empty());
if (optionCodes.isPresent() && !optionCodes.get().contains(dataValue.getValue())) {
summary.getConflicts().add(new ImportConflict(dataValue.getValue(), "Data value is not a valid option of the data element option set: " + dataElement.getUid()));
continue;
}
if (categoryOptionCombo == null) {
if (requireCategoryOptionCombo) {
summary.getConflicts().add(new ImportConflict(dataValue.getValue(), "Category option combo is required but is not specified"));
continue;
} else {
categoryOptionCombo = fallbackCategoryOptionCombo;
}
}
if (attrOptionCombo == null) {
if (requireAttrOptionCombo) {
summary.getConflicts().add(new ImportConflict(dataValue.getValue(), "Attribute option combo is required but is not specified"));
continue;
} else {
attrOptionCombo = fallbackCategoryOptionCombo;
}
}
if (strictPeriods && !dataElementPeriodTypesMap.get(dataElement.getUid(), () -> dataElement.getPeriodTypes()).contains(period.getPeriodType())) {
summary.getConflicts().add(new ImportConflict(dataValue.getPeriod(), "Period type of period: " + period.getIsoDate() + " not valid for data element: " + dataElement.getUid()));
continue;
}
if (strictCategoryOptionCombos && !dataElementCategoryOptionComboMap.get(dataElement.getUid(), () -> dataElement.getCategoryOptionCombos()).contains(categoryOptionCombo)) {
summary.getConflicts().add(new ImportConflict(categoryOptionCombo.getUid(), "Category option combo: " + categoryOptionCombo.getUid() + " must be part of category combo of data element: " + dataElement.getUid()));
continue;
}
if (strictAttrOptionCombos && !dataElementAttrOptionComboMap.get(dataElement.getUid(), () -> dataElement.getDataSetCategoryOptionCombos()).contains(attrOptionCombo)) {
summary.getConflicts().add(new ImportConflict(attrOptionCombo.getUid(), "Attribute option combo: " + attrOptionCombo.getUid() + " must be part of category combo of data sets of data element: " + dataElement.getUid()));
continue;
}
if (strictOrgUnits && BooleanUtils.isFalse(dataElementOrgUnitMap.get(dataElement.getUid() + orgUnit.getUid(), () -> orgUnit.hasDataElement(dataElement)))) {
summary.getConflicts().add(new ImportConflict(orgUnit.getUid(), "Data element: " + dataElement.getUid() + " must be assigned through data sets to organisation unit: " + orgUnit.getUid()));
continue;
}
boolean zeroInsignificant = ValidationUtils.dataValueIsZeroAndInsignificant(dataValue.getValue(), dataElement);
if (zeroInsignificant) {
summary.getConflicts().add(new ImportConflict(dataValue.getValue(), "Value is zero and not significant, must match data element: " + dataElement.getUid()));
continue;
}
String storedByValid = ValidationUtils.storedByIsValid(dataValue.getStoredBy());
if (storedByValid != null) {
summary.getConflicts().add(new ImportConflict(dataValue.getStoredBy(), i18n.getString(storedByValid)));
continue;
}
String storedBy = dataValue.getStoredBy() == null || dataValue.getStoredBy().trim().isEmpty() ? currentUser : dataValue.getStoredBy();
final DataElementCategoryOptionCombo aoc = attrOptionCombo;
DateRange aocDateRange = attrOptionComboDateRangeMap.get(attrOptionCombo.getUid(), () -> aoc.getDateRange());
if ((aocDateRange.getStartDate() != null && aocDateRange.getStartDate().compareTo(period.getStartDate()) > 0) || (aocDateRange.getEndDate() != null && aocDateRange.getEndDate().compareTo(period.getEndDate()) < 0)) {
summary.getConflicts().add(new ImportConflict(orgUnit.getUid(), "Period: " + period.getIsoDate() + " is not within date range of attribute option combo: " + attrOptionCombo.getUid()));
continue;
}
if (!attrOptionComboOrgUnitMap.get(attrOptionCombo.getUid() + orgUnit.getUid(), () -> {
Set<OrganisationUnit> aocOrgUnits = aoc.getOrganisationUnits();
return aocOrgUnits == null || orgUnit.isDescendant(aocOrgUnits);
})) {
summary.getConflicts().add(new ImportConflict(orgUnit.getUid(), "Organisation unit: " + orgUnit.getUid() + " is not valid for attribute option combo: " + attrOptionCombo.getUid()));
continue;
}
final DataSet approvalDataSet = dataSet != null ? dataSet : dataElementDataSetMap.get(dataElement.getUid(), () -> dataElement.getApprovalDataSet());
if (// Data element is assigned to at least one data set
approvalDataSet != null) {
if (dataSetLockedMap.get(approvalDataSet.getUid() + period.getUid() + orgUnit.getUid(), () -> isLocked(approvalDataSet, period, orgUnit, skipLockExceptionCheck))) {
summary.getConflicts().add(new ImportConflict(period.getIsoDate(), "Current date is past expiry days for period " + period.getIsoDate() + " and data set: " + approvalDataSet.getUid()));
continue;
}
Period latestFuturePeriod = dataElementLatestFuturePeriodMap.get(dataElement.getUid(), () -> dataElement.getLatestOpenFuturePeriod());
if (period.isAfter(latestFuturePeriod) && isIso8601) {
summary.getConflicts().add(new ImportConflict(period.getIsoDate(), "Period: " + period.getIsoDate() + " is after latest open future period: " + latestFuturePeriod.getIsoDate() + " for data element: " + dataElement.getUid()));
continue;
}
DataApprovalWorkflow workflow = approvalDataSet.getWorkflow();
if (workflow != null) {
final String workflowPeriodAoc = workflow.getUid() + period.getUid() + attrOptionCombo.getUid();
if (approvalMap.get(orgUnit.getUid() + workflowPeriodAoc, () -> {
DataApproval lowestApproval = DataApproval.getLowestApproval(new DataApproval(null, workflow, period, orgUnit, aoc));
return lowestApprovalLevelMap.get(lowestApproval.getDataApprovalLevel().getUid() + lowestApproval.getOrganisationUnit().getUid() + workflowPeriodAoc, () -> approvalService.getDataApproval(lowestApproval) != null);
})) {
summary.getConflicts().add(new ImportConflict(orgUnit.getUid(), "Data is already approved for data set: " + approvalDataSet.getUid() + " period: " + period.getIsoDate() + " organisation unit: " + orgUnit.getUid() + " attribute option combo: " + attrOptionCombo.getUid()));
continue;
}
}
}
if (approvalDataSet != null && !approvalDataSet.isDataInputPeriodAndDateAllowed(period, new Date())) {
summary.getConflicts().add(new ImportConflict(orgUnit.getUid(), "Period: " + period.getIsoDate() + " is not open for this data set at this time: " + approvalDataSet.getUid()));
continue;
}
if (!periodOpenForDataElement.get(dataElement.getUid() + period.getIsoDate(), () -> dataElement.isDataInputAllowedForPeriodAndDate(period, new Date()))) {
summary.getConflicts().add(new ImportConflict(orgUnit.getUid(), "Period " + period.getName() + " does not conform to the open periods of associated data sets"));
continue;
}
// -----------------------------------------------------------------
// Create data value
// -----------------------------------------------------------------
DataValue internalValue = new DataValue();
internalValue.setDataElement(dataElement);
internalValue.setPeriod(period);
internalValue.setSource(orgUnit);
internalValue.setCategoryOptionCombo(categoryOptionCombo);
internalValue.setAttributeOptionCombo(attrOptionCombo);
internalValue.setValue(trimToNull(dataValue.getValue()));
internalValue.setStoredBy(storedBy);
internalValue.setCreated(dataValue.hasCreated() ? parseDate(dataValue.getCreated()) : now);
internalValue.setLastUpdated(dataValue.hasLastUpdated() ? parseDate(dataValue.getLastUpdated()) : now);
internalValue.setComment(trimToNull(dataValue.getComment()));
internalValue.setFollowup(dataValue.getFollowup());
internalValue.setDeleted(BooleanUtils.isTrue(dataValue.getDeleted()));
// -----------------------------------------------------------------
// Save, update or delete data value
// -----------------------------------------------------------------
DataValue existingValue = !skipExistingCheck ? dataValueBatchHandler.findObject(internalValue) : null;
if (!skipExistingCheck && existingValue != null && !existingValue.isDeleted()) {
if (strategy.isCreateAndUpdate() || strategy.isUpdate()) {
DataValueAudit auditValue = new DataValueAudit(internalValue, existingValue.getValue(), storedBy, AuditType.UPDATE);
if (internalValue.isNullValue() || internalValue.isDeleted()) {
internalValue.setDeleted(true);
auditValue.setAuditType(AuditType.DELETE);
deleteCount++;
} else {
updateCount++;
}
if (!dryRun) {
dataValueBatchHandler.updateObject(internalValue);
auditBatchHandler.addObject(auditValue);
}
} else if (strategy.isDelete()) {
DataValueAudit auditValue = new DataValueAudit(internalValue, existingValue.getValue(), storedBy, AuditType.DELETE);
internalValue.setDeleted(true);
deleteCount++;
if (!dryRun) {
dataValueBatchHandler.updateObject(internalValue);
auditBatchHandler.addObject(auditValue);
}
}
} else {
if (strategy.isCreateAndUpdate() || strategy.isCreate()) {
if (// Ignore null values
!internalValue.isNullValue()) {
if (existingValue != null && existingValue.isDeleted()) {
importCount++;
if (!dryRun) {
dataValueBatchHandler.updateObject(internalValue);
}
} else {
boolean added = false;
if (!dryRun) {
added = dataValueBatchHandler.addObject(internalValue);
}
if (dryRun || added) {
importCount++;
}
}
}
}
}
}
dataValueBatchHandler.flush();
auditBatchHandler.flush();
int ignores = totalCount - importCount - updateCount - deleteCount;
summary.setImportCount(new ImportCount(importCount, updateCount, ignores, deleteCount));
summary.setStatus(summary.getConflicts().isEmpty() ? ImportStatus.SUCCESS : ImportStatus.WARNING);
summary.setDescription("Import process completed successfully");
clock.logTime("Data value import done, total: " + totalCount + ", import: " + importCount + ", update: " + updateCount + ", delete: " + deleteCount);
notifier.notify(id, notificationLevel, "Import done", true).addTaskSummary(id, notificationLevel, summary);
dataValueSet.close();
return summary;
}
use of org.hisp.dhis.dxf2.importsummary.ImportConflict in project dhis2-core by dhis2.
the class DefaultDataValueSetService method handleComplete.
// -------------------------------------------------------------------------
// Supportive methods
// -------------------------------------------------------------------------
private void handleComplete(DataSet dataSet, Date completeDate, Period period, OrganisationUnit orgUnit, DataElementCategoryOptionCombo attributeOptionCombo, ImportSummary summary) {
if (orgUnit == null) {
summary.getConflicts().add(new ImportConflict(OrganisationUnit.class.getSimpleName(), ERROR_OBJECT_NEEDED_TO_COMPLETE));
return;
}
if (period == null) {
summary.getConflicts().add(new ImportConflict(Period.class.getSimpleName(), ERROR_OBJECT_NEEDED_TO_COMPLETE));
return;
}
period = periodService.reloadPeriod(period);
CompleteDataSetRegistration completeAlready = registrationService.getCompleteDataSetRegistration(dataSet, period, orgUnit, attributeOptionCombo);
String username = currentUserService.getCurrentUsername();
if (completeAlready != null) {
completeAlready.setStoredBy(username);
completeAlready.setDate(completeDate);
registrationService.updateCompleteDataSetRegistration(completeAlready);
} else {
CompleteDataSetRegistration registration = new CompleteDataSetRegistration(dataSet, period, orgUnit, attributeOptionCombo, completeDate, username);
registrationService.saveCompleteDataSetRegistration(registration);
}
summary.setDataSetComplete(DateUtils.getMediumDateString(completeDate));
}
use of org.hisp.dhis.dxf2.importsummary.ImportConflict in project dhis2-core by dhis2.
the class DefaultCompleteDataSetRegistrationExchangeService method validateAocMatchesDataSetCc.
private static void validateAocMatchesDataSetCc(MetaDataProperties mdProps) throws ImportConflictException {
// TODO MdCache?
DataElementCategoryCombo aocCC = mdProps.attrOptCombo.getCategoryCombo();
DataElementCategoryCombo dsCc = mdProps.dataSet.getCategoryCombo();
if (!aocCC.equals(dsCc)) {
throw new ImportConflictException(new ImportConflict(aocCC.getUid(), String.format("Attribute option combo: %s must have category combo: %s", aocCC.getUid(), dsCc.getUid())));
}
}
use of org.hisp.dhis.dxf2.importsummary.ImportConflict in project dhis2-core by dhis2.
the class DefaultAdxDataService method parseAdxGroupToDxf.
// -------------------------------------------------------------------------
// Utility methods
// -------------------------------------------------------------------------
private List<ImportConflict> parseAdxGroupToDxf(XMLReader adxReader, XMLStreamWriter dxfWriter, ImportOptions importOptions, CachingMap<String, DataSet> dataSetMap, IdentifiableObjectCallable<DataSet> dataSetCallable, CachingMap<String, DataElement> dataElementMap, IdentifiableObjectCallable<DataElement> dataElementCallable) throws XMLStreamException, AdxException {
List<ImportConflict> adxConflicts = new LinkedList<>();
IdentifiableProperty dataElementIdScheme = importOptions.getIdSchemes().getDataElementIdScheme().getIdentifiableProperty();
Map<String, String> groupAttributes = adxReader.readAttributes();
if (!groupAttributes.containsKey(AdxDataService.PERIOD)) {
throw new AdxException(AdxDataService.PERIOD + " attribute is required on 'group'");
}
if (!groupAttributes.containsKey(AdxDataService.ORGUNIT)) {
throw new AdxException(AdxDataService.ORGUNIT + " attribute is required on 'group'");
}
// translate ADX period to DXF
String periodStr = groupAttributes.get(AdxDataService.PERIOD);
groupAttributes.remove(AdxDataService.PERIOD);
Period period = AdxPeriod.parse(periodStr);
groupAttributes.put(AdxDataService.PERIOD, period.getIsoDate());
// process ADX group attributes
if (!groupAttributes.containsKey(AdxDataService.ATTOPTCOMBO) && groupAttributes.containsKey(AdxDataService.DATASET)) {
log.debug("No attribute option combo present, check data set for attribute category combo");
String dataSetStr = trimToNull(groupAttributes.get(AdxDataService.DATASET));
final DataSet dataSet = dataSetMap.get(dataSetStr, dataSetCallable.setId(dataSetStr));
if (dataSet == null) {
throw new AdxException("No data set matching identifier: " + groupAttributes.get(AdxDataService.DATASET));
}
groupAttributes.put(AdxDataService.DATASET, dataSet.getUid());
DataElementCategoryCombo attributeCombo = dataSet.getCategoryCombo();
convertAttributesToDxf(groupAttributes, AdxDataService.ATTOPTCOMBO, attributeCombo, dataElementIdScheme);
}
// process the dataValues
while (adxReader.moveToStartElement(AdxDataService.DATAVALUE, AdxDataService.GROUP)) {
try {
parseADXDataValueToDxf(adxReader, dxfWriter, groupAttributes, importOptions, dataElementMap, dataElementCallable);
} catch (AdxException ex) {
adxConflicts.add(ex.getImportConflict());
log.info("ADX data value conflict: " + ex.getImportConflict());
}
}
return adxConflicts;
}
Aggregations