Search in sources :

Example 11 with DeflatedDataValue

use of org.hisp.dhis.datavalue.DeflatedDataValue in project dhis2-core by dhis2.

the class DataValidationTaskTest method verifySimpleValidation_oneRule_noErrors.

/**
 * Verify that a single rule passes against a Data Element
 */
@Test
void verifySimpleValidation_oneRule_noErrors() {
    Expression leftExpression = createExpression2('A', "#{FUrCpcvMAmC.OrDRjJL9bTS}");
    Expression rightExpression = createExpression2('B', "-10");
    ValidationRuleExtended vre = createValidationRuleExtended(leftExpression, rightExpression, Operator.not_equal_to);
    List<PeriodTypeExtended> periodTypes = new ArrayList<>();
    PeriodTypeExtended periodType = createPeriodTypeExtended(vre);
    periodType.addDataElement(deA);
    periodTypes.add(periodType);
    CategoryOptionCombo categoryOptionCombo = createCategoryOptionCombo('A', 'B');
    ValidationRunContext ctx = ValidationRunContext.newBuilder().withOrgUnits(organisationUnits).withItemMap(new HashMap<>()).withOrgUnitGroupMap(new HashMap<>()).withDefaultAttributeCombo(categoryOptionCombo).withPeriodTypeXs(periodTypes).withMaxResults(500).build();
    List<DeflatedDataValue> deflatedDataValues = new ArrayList<>();
    DataValue dv = createDataValue(deA, createPeriod("201901"), ouA, "12.4", createCategoryOptionCombo('B', 'C'));
    DeflatedDataValue ddv = new DeflatedDataValue(dv);
    deflatedDataValues.add(ddv);
    when(dataValueService.getDeflatedDataValues(any(DataExportParams.class))).thenReturn(deflatedDataValues);
    Map<DimensionalItemObject, Object> vals = new HashMap<>();
    vals.put(deA, 12.4);
    mockExpressionService(leftExpression, vals, 8.4);
    mockExpressionService(rightExpression, vals, -10.0);
    when(expressionService.getExpressionValue(ExpressionParams.builder().expression("8.4!=-10.0").parseType(SIMPLE_TEST).build())).thenReturn(true);
    subject.init(organisationUnits, ctx, analyticsService);
    subject.run();
    assertThat(ctx.getValidationResults().size(), is(0));
}
Also used : HashMap(java.util.HashMap) DeflatedDataValue(org.hisp.dhis.datavalue.DeflatedDataValue) DhisConvenienceTest.createDataValue(org.hisp.dhis.DhisConvenienceTest.createDataValue) DataValue(org.hisp.dhis.datavalue.DataValue) ArrayList(java.util.ArrayList) DeflatedDataValue(org.hisp.dhis.datavalue.DeflatedDataValue) DimensionalItemObject(org.hisp.dhis.common.DimensionalItemObject) Expression(org.hisp.dhis.expression.Expression) DataExportParams(org.hisp.dhis.datavalue.DataExportParams) DimensionalItemObject(org.hisp.dhis.common.DimensionalItemObject) DhisConvenienceTest.createCategoryOptionCombo(org.hisp.dhis.DhisConvenienceTest.createCategoryOptionCombo) CategoryOptionCombo(org.hisp.dhis.category.CategoryOptionCombo) Test(org.junit.jupiter.api.Test)

Example 12 with DeflatedDataValue

use of org.hisp.dhis.datavalue.DeflatedDataValue in project dhis2-core by dhis2.

the class MinMaxOutlierAnalysisService method analyse.

// -------------------------------------------------------------------------
// DataAnalysisService implementation
// -------------------------------------------------------------------------
@Override
public List<DeflatedDataValue> analyse(Collection<OrganisationUnit> parents, Collection<DataElement> dataElements, Collection<Period> periods, Double stdDevFactor, Date from) {
    Set<DataElement> elements = dataElements.stream().filter(de -> ValueType.NUMERIC_TYPES.contains(de.getValueType())).collect(Collectors.toSet());
    Set<CategoryOptionCombo> categoryOptionCombos = new HashSet<>();
    for (DataElement dataElement : elements) {
        categoryOptionCombos.addAll(dataElement.getCategoryOptionCombos());
    }
    log.debug("Starting min-max analysis, no of data elements: " + elements.size() + ", no of parent org units: " + parents.size());
    return dataAnalysisStore.getMinMaxViolations(elements, categoryOptionCombos, periods, parents, MAX_OUTLIERS);
}
Also used : BatchHandlerFactory(org.hisp.quick.BatchHandlerFactory) java.util(java.util) MinMaxDataElementService(org.hisp.dhis.minmax.MinMaxDataElementService) ValueType(org.hisp.dhis.common.ValueType) Preconditions.checkNotNull(com.google.common.base.Preconditions.checkNotNull) DateTime(org.joda.time.DateTime) DeflatedDataValue(org.hisp.dhis.datavalue.DeflatedDataValue) Collectors(java.util.stream.Collectors) DataElement(org.hisp.dhis.dataelement.DataElement) OrganisationUnit(org.hisp.dhis.organisationunit.OrganisationUnit) Slf4j(lombok.extern.slf4j.Slf4j) Lists(com.google.common.collect.Lists) CategoryOptionCombo(org.hisp.dhis.category.CategoryOptionCombo) Service(org.springframework.stereotype.Service) MinMaxDataElementBatchHandler(org.hisp.dhis.jdbc.batchhandler.MinMaxDataElementBatchHandler) MathUtils(org.hisp.dhis.system.util.MathUtils) MinMaxDataElement(org.hisp.dhis.minmax.MinMaxDataElement) BatchHandler(org.hisp.quick.BatchHandler) Period(org.hisp.dhis.period.Period) DataElement(org.hisp.dhis.dataelement.DataElement) MinMaxDataElement(org.hisp.dhis.minmax.MinMaxDataElement) CategoryOptionCombo(org.hisp.dhis.category.CategoryOptionCombo)

Example 13 with DeflatedDataValue

use of org.hisp.dhis.datavalue.DeflatedDataValue in project dhis2-core by dhis2.

the class JdbcDataAnalysisStore method getDeflatedDataValues.

private List<DeflatedDataValue> getDeflatedDataValues(DataElement dataElement, CategoryOptionCombo categoryOptionCombo, Collection<Period> periods, List<Long> organisationUnits, Map<Long, Integer> lowerBoundMap, Map<Long, Integer> upperBoundMap) {
    String periodIds = TextUtils.getCommaDelimitedString(getIdentifiers(periods));
    // @formatter:off
    String sql = "select dv.dataelementid, dv.periodid, dv.sourceid, " + "dv.categoryoptioncomboid, dv.attributeoptioncomboid, dv.value, dv.storedby, dv.lastupdated, " + "dv.created, dv.comment, dv.followup, ou.name as sourcename, " + "? as dataelementname, pt.name as periodtypename, pe.startdate, pe.enddate, " + "? as categoryoptioncomboname " + "from datavalue dv " + "inner join period pe on dv.periodid = pe.periodid " + "inner join periodtype pt on pe.periodtypeid = pt.periodtypeid " + "inner join organisationunit ou on dv.sourceid = ou.organisationunitid " + "where dv.dataelementid = " + dataElement.getId() + " " + "and dv.categoryoptioncomboid = " + categoryOptionCombo.getId() + " " + "and dv.periodid in (" + periodIds + ") and (";
    for (Long orgUnitUid : organisationUnits) {
        sql += "(dv.sourceid = " + orgUnitUid + " " + "and (cast( dv.value as " + statementBuilder.getDoubleColumnType() + " ) < " + lowerBoundMap.get(orgUnitUid) + " " + "or cast(dv.value as " + statementBuilder.getDoubleColumnType() + ") > " + upperBoundMap.get(orgUnitUid) + ")) or ";
    }
    sql = TextUtils.removeLastOr(sql) + ") ";
    sql += "and dv.deleted is false ";
    PreparedStatementSetter pss = (ps) -> {
        ps.setString(1, dataElement.getName());
        ps.setString(2, categoryOptionCombo.getName());
    };
    return jdbcTemplate.query(sql, pss, new DeflatedDataValueNameMinMaxRowMapper(lowerBoundMap, upperBoundMap));
}
Also used : IdentifiableObjectUtils.getIdentifiers(org.hisp.dhis.common.IdentifiableObjectUtils.getIdentifiers) Date(java.util.Date) PreparedStatementSetter(org.springframework.jdbc.core.PreparedStatementSetter) DataAnalysisStore(org.hisp.dhis.dataanalysis.DataAnalysisStore) DeflatedDataValue(org.hisp.dhis.datavalue.DeflatedDataValue) ArrayList(java.util.ArrayList) JdbcTemplate(org.springframework.jdbc.core.JdbcTemplate) DataElement(org.hisp.dhis.dataelement.DataElement) PaginatedList(org.hisp.dhis.commons.collection.PaginatedList) Map(java.util.Map) Qualifier(org.springframework.beans.factory.annotation.Qualifier) SqlRowSet(org.springframework.jdbc.support.rowset.SqlRowSet) Repository(org.springframework.stereotype.Repository) TextUtils.getCommaDelimitedString(org.hisp.dhis.commons.util.TextUtils.getCommaDelimitedString) Period(org.hisp.dhis.period.Period) Collection(java.util.Collection) Preconditions.checkNotNull(com.google.common.base.Preconditions.checkNotNull) StatementBuilder(org.hisp.dhis.jdbc.StatementBuilder) OrganisationUnit(org.hisp.dhis.organisationunit.OrganisationUnit) List(java.util.List) Slf4j(lombok.extern.slf4j.Slf4j) DataAnalysisMeasures(org.hisp.dhis.dataanalysis.DataAnalysisMeasures) CategoryOptionCombo(org.hisp.dhis.category.CategoryOptionCombo) DeflatedDataValueNameMinMaxRowMapper(org.hisp.dhis.system.objectmapper.DeflatedDataValueNameMinMaxRowMapper) DateUtils(org.hisp.dhis.util.DateUtils) TextUtils(org.hisp.dhis.commons.util.TextUtils) PreparedStatementSetter(org.springframework.jdbc.core.PreparedStatementSetter) TextUtils.getCommaDelimitedString(org.hisp.dhis.commons.util.TextUtils.getCommaDelimitedString) DeflatedDataValueNameMinMaxRowMapper(org.hisp.dhis.system.objectmapper.DeflatedDataValueNameMinMaxRowMapper)

Example 14 with DeflatedDataValue

use of org.hisp.dhis.datavalue.DeflatedDataValue in project dhis2-core by dhis2.

the class MinMaxOutlierAnalysisServiceTest method testGetFindOutliers.

// ----------------------------------------------------------------------
// Business logic tests
// ----------------------------------------------------------------------
@Test
public void testGetFindOutliers() {
    dataValueA = createDataValue(dataElementA, periodI, organisationUnitA, "41", categoryOptionCombo);
    dataValueB = createDataValue(dataElementA, periodJ, organisationUnitA, "-41", categoryOptionCombo);
    dataValueService.addDataValue(createDataValue(dataElementA, periodA, organisationUnitA, "5", categoryOptionCombo));
    dataValueService.addDataValue(createDataValue(dataElementA, periodB, organisationUnitA, "-5", categoryOptionCombo));
    dataValueService.addDataValue(createDataValue(dataElementA, periodC, organisationUnitA, "5", categoryOptionCombo));
    dataValueService.addDataValue(createDataValue(dataElementA, periodD, organisationUnitA, "-5", categoryOptionCombo));
    dataValueService.addDataValue(createDataValue(dataElementA, periodE, organisationUnitA, "10", categoryOptionCombo));
    dataValueService.addDataValue(createDataValue(dataElementA, periodF, organisationUnitA, "-10", categoryOptionCombo));
    dataValueService.addDataValue(createDataValue(dataElementA, periodG, organisationUnitA, "13", categoryOptionCombo));
    dataValueService.addDataValue(createDataValue(dataElementA, periodH, organisationUnitA, "-13", categoryOptionCombo));
    dataValueService.addDataValue(dataValueA);
    dataValueService.addDataValue(dataValueB);
    minMaxDataElement = new MinMaxDataElement(organisationUnitA, dataElementA, categoryOptionCombo, -40, 40, false);
    minMaxDataElementService.addMinMaxDataElement(minMaxDataElement);
    List<Period> periods = new ArrayList<>();
    periods.add(periodI);
    periods.add(periodJ);
    periods.add(periodA);
    periods.add(periodE);
    List<DeflatedDataValue> result = minMaxOutlierAnalysisService.analyse(Lists.newArrayList(organisationUnitA), dataElementsA, periods, null, from);
    assertEquals(2, result.size());
}
Also used : DeflatedDataValue(org.hisp.dhis.datavalue.DeflatedDataValue) MinMaxDataElement(org.hisp.dhis.minmax.MinMaxDataElement) ArrayList(java.util.ArrayList) Period(org.hisp.dhis.period.Period) Test(org.junit.Test) DhisSpringTest(org.hisp.dhis.DhisSpringTest)

Example 15 with DeflatedDataValue

use of org.hisp.dhis.datavalue.DeflatedDataValue in project dhis2-core by dhis2.

the class FormUtilsImpl method getValidationViolations.

// -------------------------------------------------------------------------
// Utils
// -------------------------------------------------------------------------
@Override
@SuppressWarnings("unchecked")
public Map<String, DeflatedDataValue> getValidationViolations(OrganisationUnit organisationUnit, Collection<DataElement> dataElements, Period period) {
    Map<String, DeflatedDataValue> validationErrorMap = new HashMap<>();
    Double factor = (Double) systemSettingManager.getSystemSetting(SettingKey.FACTOR_OF_DEVIATION);
    Date from = new DateTime(period.getStartDate()).minusYears(2).toDate();
    Collection<DeflatedDataValue> stdDevs = stdDevOutlierAnalysisService.analyse(Sets.newHashSet(organisationUnit), dataElements, Sets.newHashSet(period), factor, from);
    Collection<DeflatedDataValue> minMaxs = minMaxOutlierAnalysisService.analyse(Sets.newHashSet(organisationUnit), dataElements, Sets.newHashSet(period), null, from);
    Collection<DeflatedDataValue> deflatedDataValues = CollectionUtils.union(stdDevs, minMaxs);
    for (DeflatedDataValue deflatedDataValue : deflatedDataValues) {
        String key = String.format("DE%dOC%d", deflatedDataValue.getDataElementId(), deflatedDataValue.getCategoryOptionComboId());
        validationErrorMap.put(key, deflatedDataValue);
    }
    return validationErrorMap;
}
Also used : DeflatedDataValue(org.hisp.dhis.datavalue.DeflatedDataValue) DateTime(org.joda.time.DateTime)

Aggregations

DeflatedDataValue (org.hisp.dhis.datavalue.DeflatedDataValue)28 ArrayList (java.util.ArrayList)16 Period (org.hisp.dhis.period.Period)16 OrganisationUnit (org.hisp.dhis.organisationunit.OrganisationUnit)10 CategoryOptionCombo (org.hisp.dhis.category.CategoryOptionCombo)9 DataElement (org.hisp.dhis.dataelement.DataElement)9 List (java.util.List)7 DataElementOperand (org.hisp.dhis.dataelement.DataElementOperand)6 DataExportParams (org.hisp.dhis.datavalue.DataExportParams)6 Grid (org.hisp.dhis.common.Grid)5 DataValue (org.hisp.dhis.datavalue.DataValue)5 I18nFormat (org.hisp.dhis.i18n.I18nFormat)5 ListGrid (org.hisp.dhis.system.grid.ListGrid)5 DateTime (org.joda.time.DateTime)5 Date (java.util.Date)4 HashSet (java.util.HashSet)4 Slf4j (lombok.extern.slf4j.Slf4j)4 Collectors (java.util.stream.Collectors)3 WebMessageException (org.hisp.dhis.dxf2.webmessage.WebMessageException)3 MinMaxDataElement (org.hisp.dhis.minmax.MinMaxDataElement)3