Search in sources :

Example 1 with BatchHandlerFactory

use of org.hisp.quick.BatchHandlerFactory in project dhis2-core by dhis2.

the class DefaultPredictionService method predict.

@Override
public void predict(Predictor predictor, Date startDate, Date endDate, PredictionSummary predictionSummary) {
    Expression generator = predictor.getGenerator();
    Expression skipTest = predictor.getSampleSkipTest();
    DataElement outputDataElement = predictor.getOutput();
    DataType expressionDataType = DataType.fromValueType(outputDataElement.getValueType());
    Map<DimensionalItemId, DimensionalItemObject> outputPeriodItemMap = new HashMap<>();
    Map<DimensionalItemId, DimensionalItemObject> sampledItemMap = new HashMap<>();
    expressionService.getExpressionDimensionalItemMaps(generator.getExpression(), PREDICTOR_EXPRESSION, expressionDataType, outputPeriodItemMap, sampledItemMap);
    Set<String> orgUnitGroupIds = expressionService.getExpressionOrgUnitGroupIds(generator.getExpression(), PREDICTOR_EXPRESSION);
    if (skipTest != null) {
        expressionService.getExpressionDimensionalItemMaps(skipTest.getExpression(), PREDICTOR_SKIP_TEST, DataType.BOOLEAN, sampledItemMap, sampledItemMap);
        orgUnitGroupIds.addAll(expressionService.getExpressionOrgUnitGroupIds(skipTest.getExpression(), PREDICTOR_SKIP_TEST));
    }
    Map<String, OrganisationUnitGroup> orgUnitGroupMap = orgUnitGroupIds.stream().map(organisationUnitGroupService::getOrganisationUnitGroup).filter(Objects::nonNull).collect(Collectors.toMap(OrganisationUnitGroup::getUid, g -> g));
    Map<DimensionalItemId, DimensionalItemObject> itemMap = new HashMap<>(outputPeriodItemMap);
    itemMap.putAll(sampledItemMap);
    Set<DimensionalItemObject> outputPeriodItems = new HashSet<>(outputPeriodItemMap.values());
    Set<DimensionalItemObject> sampledItems = new HashSet<>(sampledItemMap.values());
    Set<DimensionalItemObject> items = new HashSet<>(itemMap.values());
    List<Period> outputPeriods = getPeriodsBetweenDates(predictor.getPeriodType(), startDate, endDate);
    Set<Period> existingOutputPeriods = getExistingPeriods(outputPeriods);
    ListMap<Period, Period> samplePeriodsMap = getSamplePeriodsMap(outputPeriods, predictor);
    Set<Period> allSamplePeriods = samplePeriodsMap.uniqueValues();
    Set<Period> analyticsQueryPeriods = getAnalyticsQueryPeriods(sampledItems, allSamplePeriods, outputPeriodItems, existingOutputPeriods);
    Set<Period> dataValueQueryPeriods = getDataValueQueryPeriods(analyticsQueryPeriods, existingOutputPeriods);
    outputPeriods = periodService.reloadPeriods(outputPeriods);
    CategoryOptionCombo defaultCategoryOptionCombo = categoryService.getDefaultCategoryOptionCombo();
    CategoryOptionCombo outputOptionCombo = predictor.getOutputCombo() == null ? defaultCategoryOptionCombo : predictor.getOutputCombo();
    DataElementOperand outputDataElementOperand = new DataElementOperand(outputDataElement, outputOptionCombo);
    Date now = new Date();
    boolean requireData = generator.getMissingValueStrategy() != NEVER_SKIP && (!items.isEmpty());
    DimensionalItemObject forwardReference = addOuputToItems(outputDataElementOperand, items);
    Set<OrganisationUnit> currentUserOrgUnits = new HashSet<>();
    String storedBy = "system-process";
    User currentUser = currentUserService.getCurrentUser();
    if (currentUser != null) {
        currentUserOrgUnits = currentUser.getOrganisationUnits();
        storedBy = currentUser.getUsername();
    }
    PredictionDataConsolidator consolidator = new PredictionDataConsolidator(items, predictor.getOrganisationUnitDescendants().equals(DESCENDANTS), new PredictionDataValueFetcher(dataValueService, categoryService), new PredictionAnalyticsDataFetcher(analyticsService, categoryService));
    PredictionWriter predictionWriter = new PredictionWriter(dataValueService, batchHandlerFactory);
    predictionWriter.init(existingOutputPeriods, predictionSummary);
    predictionSummary.incrementPredictors();
    for (OrganisationUnitLevel orgUnitLevel : predictor.getOrganisationUnitLevels()) {
        List<OrganisationUnit> orgUnits = organisationUnitService.getOrganisationUnitsAtOrgUnitLevels(Lists.newArrayList(orgUnitLevel), currentUserOrgUnits);
        consolidator.init(currentUserOrgUnits, orgUnitLevel.getLevel(), orgUnits, dataValueQueryPeriods, analyticsQueryPeriods, existingOutputPeriods, outputDataElementOperand);
        PredictionData data;
        while ((data = consolidator.getData()) != null) {
            List<DataValue> predictions = new ArrayList<>();
            List<PredictionContext> contexts = PredictionContextGenerator.getContexts(outputPeriods, data.getValues(), defaultCategoryOptionCombo);
            for (PredictionContext c : contexts) {
                List<Period> samplePeriods = new ArrayList<>(samplePeriodsMap.get(c.getOutputPeriod()));
                samplePeriods.removeAll(getSkippedPeriods(allSamplePeriods, itemMap, c.getPeriodValueMap(), skipTest, orgUnitGroupMap, data.getOrgUnit()));
                if (requireData && !dataIsPresent(outputPeriodItems, c.getValueMap(), sampledItems, samplePeriods, c.getPeriodValueMap())) {
                    continue;
                }
                Object value = expressionService.getExpressionValue(ExpressionParams.builder().expression(predictor.getGenerator().getExpression()).parseType(PREDICTOR_EXPRESSION).dataType(expressionDataType).itemMap(itemMap).valueMap(c.getValueMap()).orgUnitGroupMap(orgUnitGroupMap).days(c.getOutputPeriod().getDaysInPeriod()).missingValueStrategy(generator.getMissingValueStrategy()).orgUnit(data.getOrgUnit()).samplePeriods(samplePeriods).periodValueMap(c.getPeriodValueMap()).build());
                if (value != null || generator.getMissingValueStrategy() == NEVER_SKIP) {
                    String valueString = formatPrediction(value, outputDataElement);
                    if (valueString != null) {
                        DataValue prediction = new DataValue(outputDataElement, c.getOutputPeriod(), data.getOrgUnit(), outputOptionCombo, c.getAttributeOptionCombo(), valueString, storedBy, now, null);
                        carryPredictionForward(prediction, contexts, forwardReference);
                        predictions.add(prediction);
                    }
                }
            }
            predictionWriter.write(predictions, data.getOldPredictions());
        }
    }
    predictionWriter.flush();
}
Also used : CategoryService(org.hisp.dhis.category.CategoryService) Date(java.util.Date) DataType(org.hisp.dhis.analytics.DataType) PeriodService(org.hisp.dhis.period.PeriodService) PREDICTOR_SKIP_TEST(org.hisp.dhis.expression.ParseType.PREDICTOR_SKIP_TEST) OrganisationUnitService(org.hisp.dhis.organisationunit.OrganisationUnitService) StringUtils(org.apache.commons.lang3.StringUtils) PredictorJobParameters(org.hisp.dhis.scheduling.parameters.PredictorJobParameters) CurrentUserServiceTarget(org.hisp.dhis.user.CurrentUserServiceTarget) Map(java.util.Map) ExpressionParams(org.hisp.dhis.expression.ExpressionParams) JobConfiguration(org.hisp.dhis.scheduling.JobConfiguration) Period(org.hisp.dhis.period.Period) OrganisationUnitGroupService(org.hisp.dhis.organisationunit.OrganisationUnitGroupService) DimensionalItemId(org.hisp.dhis.common.DimensionalItemId) OrganisationUnitGroup(org.hisp.dhis.organisationunit.OrganisationUnitGroup) AnalyticsServiceTarget(org.hisp.dhis.analytics.AnalyticsServiceTarget) Set(java.util.Set) PREDICTOR_EXPRESSION(org.hisp.dhis.expression.ParseType.PREDICTOR_EXPRESSION) Collectors(java.util.stream.Collectors) Sets(com.google.common.collect.Sets) Objects(java.util.Objects) List(java.util.List) Slf4j(lombok.extern.slf4j.Slf4j) CategoryOptionCombo(org.hisp.dhis.category.CategoryOptionCombo) MoreObjects.firstNonNull(com.google.common.base.MoreObjects.firstNonNull) DESCENDANTS(org.hisp.dhis.common.OrganisationUnitDescendants.DESCENDANTS) BatchHandlerFactory(org.hisp.quick.BatchHandlerFactory) ListMap(org.hisp.dhis.common.ListMap) DimensionalItemObject(org.hisp.dhis.common.DimensionalItemObject) HashMap(java.util.HashMap) CollectionUtils(org.apache.commons.collections4.CollectionUtils) ArrayList(java.util.ArrayList) HashSet(java.util.HashSet) DataElement(org.hisp.dhis.dataelement.DataElement) Notifier(org.hisp.dhis.system.notification.Notifier) NEVER_SKIP(org.hisp.dhis.expression.MissingValueStrategy.NEVER_SKIP) ExpressionService(org.hisp.dhis.expression.ExpressionService) PredictionFormatter.formatPrediction(org.hisp.dhis.predictor.PredictionFormatter.formatPrediction) DataValueService(org.hisp.dhis.datavalue.DataValueService) Lists(com.google.common.collect.Lists) IdentifiableObjectManager(org.hisp.dhis.common.IdentifiableObjectManager) Service(org.springframework.stereotype.Service) User(org.hisp.dhis.user.User) DataElementOperand(org.hisp.dhis.dataelement.DataElementOperand) DebugUtils(org.hisp.dhis.commons.util.DebugUtils) OrganisationUnit(org.hisp.dhis.organisationunit.OrganisationUnit) NotificationLevel(org.hisp.dhis.system.notification.NotificationLevel) CurrentUserService(org.hisp.dhis.user.CurrentUserService) ERROR(org.hisp.dhis.system.notification.NotificationLevel.ERROR) DataValue(org.hisp.dhis.datavalue.DataValue) PeriodType(org.hisp.dhis.period.PeriodType) Expression(org.hisp.dhis.expression.Expression) AllArgsConstructor(lombok.AllArgsConstructor) OrganisationUnitLevel(org.hisp.dhis.organisationunit.OrganisationUnitLevel) AnalyticsService(org.hisp.dhis.analytics.AnalyticsService) DateUtils(org.hisp.dhis.util.DateUtils) MapMap(org.hisp.dhis.common.MapMap) Transactional(org.springframework.transaction.annotation.Transactional) DimensionalItemId(org.hisp.dhis.common.DimensionalItemId) User(org.hisp.dhis.user.User) HashMap(java.util.HashMap) DataValue(org.hisp.dhis.datavalue.DataValue) ArrayList(java.util.ArrayList) DataElement(org.hisp.dhis.dataelement.DataElement) DimensionalItemObject(org.hisp.dhis.common.DimensionalItemObject) DataType(org.hisp.dhis.analytics.DataType) HashSet(java.util.HashSet) DataElementOperand(org.hisp.dhis.dataelement.DataElementOperand) OrganisationUnit(org.hisp.dhis.organisationunit.OrganisationUnit) Period(org.hisp.dhis.period.Period) Date(java.util.Date) OrganisationUnitGroup(org.hisp.dhis.organisationunit.OrganisationUnitGroup) Expression(org.hisp.dhis.expression.Expression) OrganisationUnitLevel(org.hisp.dhis.organisationunit.OrganisationUnitLevel) DimensionalItemObject(org.hisp.dhis.common.DimensionalItemObject) CategoryOptionCombo(org.hisp.dhis.category.CategoryOptionCombo)

Aggregations

MoreObjects.firstNonNull (com.google.common.base.MoreObjects.firstNonNull)1 Lists (com.google.common.collect.Lists)1 Sets (com.google.common.collect.Sets)1 ArrayList (java.util.ArrayList)1 Date (java.util.Date)1 HashMap (java.util.HashMap)1 HashSet (java.util.HashSet)1 List (java.util.List)1 Map (java.util.Map)1 Objects (java.util.Objects)1 Set (java.util.Set)1 Collectors (java.util.stream.Collectors)1 AllArgsConstructor (lombok.AllArgsConstructor)1 Slf4j (lombok.extern.slf4j.Slf4j)1 CollectionUtils (org.apache.commons.collections4.CollectionUtils)1 StringUtils (org.apache.commons.lang3.StringUtils)1 AnalyticsService (org.hisp.dhis.analytics.AnalyticsService)1 AnalyticsServiceTarget (org.hisp.dhis.analytics.AnalyticsServiceTarget)1 DataType (org.hisp.dhis.analytics.DataType)1 CategoryOptionCombo (org.hisp.dhis.category.CategoryOptionCombo)1