use of org.hisp.dhis.datavalue.DeflatedDataValue in project dhis2-core by dhis2.
the class DataValidationTaskTest method verifySimpleValidation_oneRule_noErrors.
/**
* Verify that a single rule passes against a Data Element
*/
@Test
void verifySimpleValidation_oneRule_noErrors() {
Expression leftExpression = createExpression2('A', "#{FUrCpcvMAmC.OrDRjJL9bTS}");
Expression rightExpression = createExpression2('B', "-10");
ValidationRuleExtended vre = createValidationRuleExtended(leftExpression, rightExpression, Operator.not_equal_to);
List<PeriodTypeExtended> periodTypes = new ArrayList<>();
PeriodTypeExtended periodType = createPeriodTypeExtended(vre);
periodType.addDataElement(deA);
periodTypes.add(periodType);
CategoryOptionCombo categoryOptionCombo = createCategoryOptionCombo('A', 'B');
ValidationRunContext ctx = ValidationRunContext.newBuilder().withOrgUnits(organisationUnits).withItemMap(new HashMap<>()).withOrgUnitGroupMap(new HashMap<>()).withDefaultAttributeCombo(categoryOptionCombo).withPeriodTypeXs(periodTypes).withMaxResults(500).build();
List<DeflatedDataValue> deflatedDataValues = new ArrayList<>();
DataValue dv = createDataValue(deA, createPeriod("201901"), ouA, "12.4", createCategoryOptionCombo('B', 'C'));
DeflatedDataValue ddv = new DeflatedDataValue(dv);
deflatedDataValues.add(ddv);
when(dataValueService.getDeflatedDataValues(any(DataExportParams.class))).thenReturn(deflatedDataValues);
Map<DimensionalItemObject, Object> vals = new HashMap<>();
vals.put(deA, 12.4);
mockExpressionService(leftExpression, vals, 8.4);
mockExpressionService(rightExpression, vals, -10.0);
when(expressionService.getExpressionValue(ExpressionParams.builder().expression("8.4!=-10.0").parseType(SIMPLE_TEST).build())).thenReturn(true);
subject.init(organisationUnits, ctx, analyticsService);
subject.run();
assertThat(ctx.getValidationResults().size(), is(0));
}
use of org.hisp.dhis.datavalue.DeflatedDataValue in project dhis2-core by dhis2.
the class MinMaxOutlierAnalysisService method analyse.
// -------------------------------------------------------------------------
// DataAnalysisService implementation
// -------------------------------------------------------------------------
@Override
public List<DeflatedDataValue> analyse(Collection<OrganisationUnit> parents, Collection<DataElement> dataElements, Collection<Period> periods, Double stdDevFactor, Date from) {
Set<DataElement> elements = dataElements.stream().filter(de -> ValueType.NUMERIC_TYPES.contains(de.getValueType())).collect(Collectors.toSet());
Set<CategoryOptionCombo> categoryOptionCombos = new HashSet<>();
for (DataElement dataElement : elements) {
categoryOptionCombos.addAll(dataElement.getCategoryOptionCombos());
}
log.debug("Starting min-max analysis, no of data elements: " + elements.size() + ", no of parent org units: " + parents.size());
return dataAnalysisStore.getMinMaxViolations(elements, categoryOptionCombos, periods, parents, MAX_OUTLIERS);
}
use of org.hisp.dhis.datavalue.DeflatedDataValue in project dhis2-core by dhis2.
the class JdbcDataAnalysisStore method getDeflatedDataValues.
private List<DeflatedDataValue> getDeflatedDataValues(DataElement dataElement, CategoryOptionCombo categoryOptionCombo, Collection<Period> periods, List<Long> organisationUnits, Map<Long, Integer> lowerBoundMap, Map<Long, Integer> upperBoundMap) {
String periodIds = TextUtils.getCommaDelimitedString(getIdentifiers(periods));
// @formatter:off
String sql = "select dv.dataelementid, dv.periodid, dv.sourceid, " + "dv.categoryoptioncomboid, dv.attributeoptioncomboid, dv.value, dv.storedby, dv.lastupdated, " + "dv.created, dv.comment, dv.followup, ou.name as sourcename, " + "? as dataelementname, pt.name as periodtypename, pe.startdate, pe.enddate, " + "? as categoryoptioncomboname " + "from datavalue dv " + "inner join period pe on dv.periodid = pe.periodid " + "inner join periodtype pt on pe.periodtypeid = pt.periodtypeid " + "inner join organisationunit ou on dv.sourceid = ou.organisationunitid " + "where dv.dataelementid = " + dataElement.getId() + " " + "and dv.categoryoptioncomboid = " + categoryOptionCombo.getId() + " " + "and dv.periodid in (" + periodIds + ") and (";
for (Long orgUnitUid : organisationUnits) {
sql += "(dv.sourceid = " + orgUnitUid + " " + "and (cast( dv.value as " + statementBuilder.getDoubleColumnType() + " ) < " + lowerBoundMap.get(orgUnitUid) + " " + "or cast(dv.value as " + statementBuilder.getDoubleColumnType() + ") > " + upperBoundMap.get(orgUnitUid) + ")) or ";
}
sql = TextUtils.removeLastOr(sql) + ") ";
sql += "and dv.deleted is false ";
PreparedStatementSetter pss = (ps) -> {
ps.setString(1, dataElement.getName());
ps.setString(2, categoryOptionCombo.getName());
};
return jdbcTemplate.query(sql, pss, new DeflatedDataValueNameMinMaxRowMapper(lowerBoundMap, upperBoundMap));
}
use of org.hisp.dhis.datavalue.DeflatedDataValue in project dhis2-core by dhis2.
the class MinMaxOutlierAnalysisServiceTest method testGetFindOutliers.
// ----------------------------------------------------------------------
// Business logic tests
// ----------------------------------------------------------------------
@Test
public void testGetFindOutliers() {
dataValueA = createDataValue(dataElementA, periodI, organisationUnitA, "41", categoryOptionCombo);
dataValueB = createDataValue(dataElementA, periodJ, organisationUnitA, "-41", categoryOptionCombo);
dataValueService.addDataValue(createDataValue(dataElementA, periodA, organisationUnitA, "5", categoryOptionCombo));
dataValueService.addDataValue(createDataValue(dataElementA, periodB, organisationUnitA, "-5", categoryOptionCombo));
dataValueService.addDataValue(createDataValue(dataElementA, periodC, organisationUnitA, "5", categoryOptionCombo));
dataValueService.addDataValue(createDataValue(dataElementA, periodD, organisationUnitA, "-5", categoryOptionCombo));
dataValueService.addDataValue(createDataValue(dataElementA, periodE, organisationUnitA, "10", categoryOptionCombo));
dataValueService.addDataValue(createDataValue(dataElementA, periodF, organisationUnitA, "-10", categoryOptionCombo));
dataValueService.addDataValue(createDataValue(dataElementA, periodG, organisationUnitA, "13", categoryOptionCombo));
dataValueService.addDataValue(createDataValue(dataElementA, periodH, organisationUnitA, "-13", categoryOptionCombo));
dataValueService.addDataValue(dataValueA);
dataValueService.addDataValue(dataValueB);
minMaxDataElement = new MinMaxDataElement(organisationUnitA, dataElementA, categoryOptionCombo, -40, 40, false);
minMaxDataElementService.addMinMaxDataElement(minMaxDataElement);
List<Period> periods = new ArrayList<>();
periods.add(periodI);
periods.add(periodJ);
periods.add(periodA);
periods.add(periodE);
List<DeflatedDataValue> result = minMaxOutlierAnalysisService.analyse(Lists.newArrayList(organisationUnitA), dataElementsA, periods, null, from);
assertEquals(2, result.size());
}
use of org.hisp.dhis.datavalue.DeflatedDataValue in project dhis2-core by dhis2.
the class FormUtilsImpl method getValidationViolations.
// -------------------------------------------------------------------------
// Utils
// -------------------------------------------------------------------------
@Override
@SuppressWarnings("unchecked")
public Map<String, DeflatedDataValue> getValidationViolations(OrganisationUnit organisationUnit, Collection<DataElement> dataElements, Period period) {
Map<String, DeflatedDataValue> validationErrorMap = new HashMap<>();
Double factor = (Double) systemSettingManager.getSystemSetting(SettingKey.FACTOR_OF_DEVIATION);
Date from = new DateTime(period.getStartDate()).minusYears(2).toDate();
Collection<DeflatedDataValue> stdDevs = stdDevOutlierAnalysisService.analyse(Sets.newHashSet(organisationUnit), dataElements, Sets.newHashSet(period), factor, from);
Collection<DeflatedDataValue> minMaxs = minMaxOutlierAnalysisService.analyse(Sets.newHashSet(organisationUnit), dataElements, Sets.newHashSet(period), null, from);
Collection<DeflatedDataValue> deflatedDataValues = CollectionUtils.union(stdDevs, minMaxs);
for (DeflatedDataValue deflatedDataValue : deflatedDataValues) {
String key = String.format("DE%dOC%d", deflatedDataValue.getDataElementId(), deflatedDataValue.getCategoryOptionComboId());
validationErrorMap.put(key, deflatedDataValue);
}
return validationErrorMap;
}
Aggregations