Examples with DataElementOperand org.hisp.dhis.dataelement.DataElementOperand used on opensource projects

Example 56 with DataElementOperand

use of org.hisp.dhis.dataelement.DataElementOperand in project dhis2-core by dhis2.

the class DefaultCompleteDataSetRegistrationExchangeService method batchImport.

/**
 * @return total number of processed CompleteDataSetRegistration objects
 */
private int batchImport(CompleteDataSetRegistrations completeRegistrations, ImportConfig config, ImportSummary summary, MetadataCallables mdCallables, MetadataCaches mdCaches) {
    final User currentUser = currentUserService.getCurrentUser();
    final String currentUserName = currentUser.getUsername();
    final Set<OrganisationUnit> userOrgUnits = currentUserService.getCurrentUserOrganisationUnits();
    final I18n i18n = i18nManager.getI18n();
    BatchHandler<CompleteDataSetRegistration> batchHandler = batchHandlerFactory.createBatchHandler(CompleteDataSetRegistrationBatchHandler.class).init();
    int importCount = 0, updateCount = 0, deleteCount = 0, totalCount = 0;
    Date now = new Date();
    while (completeRegistrations.hasNextCompleteDataSetRegistration()) {
        org.hisp.dhis.dxf2.dataset.CompleteDataSetRegistration cdsr = completeRegistrations.getNextCompleteDataSetRegistration();
        totalCount++;
        // ---------------------------------------------------------------------
        // Init meta-data properties against meta-data cache
        // ---------------------------------------------------------------------
        MetadataProperties mdProps = initMetaDataProperties(cdsr, mdCallables, mdCaches);
        heatCaches(mdCaches, config);
        // ---------------------------------------------------------------------
        // Meta-data validation
        // ---------------------------------------------------------------------
        String storedBy;
        String lastUpdatedBy;
        Boolean isCompleted;
        try {
            // Validate CDSR meta-data properties
            mdProps.validate(cdsr, config);
            validateOrgUnitInUserHierarchy(mdCaches, mdProps, userOrgUnits, currentUserName);
            if (config.isStrictAttrOptionCombos()) {
                validateAocMatchesDataSetCc(mdProps);
            }
            validateAttrOptCombo(mdProps, mdCaches, config);
            if (config.isStrictPeriods()) {
                validateHasMatchingPeriodTypes(mdProps);
            }
            if (config.isStrictOrgUnits()) {
                validateDataSetIsAssignedToOrgUnit(mdProps);
            }
            storedBy = cdsr.getStoredBy();
            validateStoredBy(storedBy, i18n);
            storedBy = StringUtils.isBlank(storedBy) ? currentUserName : storedBy;
            lastUpdatedBy = cdsr.getLastUpdatedBy();
            validateStoredBy(lastUpdatedBy, i18n);
            lastUpdatedBy = StringUtils.isBlank(lastUpdatedBy) ? currentUserName : lastUpdatedBy;
            cdsr.setLastUpdatedBy(lastUpdatedBy);
            boolean DEFAULT_COMPLETENESS_STATUS = true;
            isCompleted = cdsr.getCompleted();
            isCompleted = (isCompleted == null) ? DEFAULT_COMPLETENESS_STATUS : isCompleted;
            cdsr.setCompleted(isCompleted);
        // TODO Check if Period is within range of data set?
        } catch (ImportConflictException ic) {
            summary.addConflict(ic.getImportConflict().getObject(), ic.getImportConflict().getValue());
            continue;
        }
        // ---------------------------------------------------------------------
        // Compulsory fields validation
        // ---------------------------------------------------------------------
        List<DataElementOperand> missingDataElementOperands = registrationService.getMissingCompulsoryFields(mdProps.dataSet, mdProps.period, mdProps.orgUnit, mdProps.attrOptCombo);
        if (!missingDataElementOperands.isEmpty()) {
            for (DataElementOperand dataElementOperand : missingDataElementOperands) {
                summary.addConflict("dataElementOperand", dataElementOperand.getDisplayName() + " needs to be filled. It is compulsory.");
            }
            if (mdProps.dataSet.isCompulsoryFieldsCompleteOnly()) {
                continue;
            }
        }
        // ---------------------------------------------------------------------
        // Data Sharing check
        // ---------------------------------------------------------------------
        List<String> errors = validateDataAccess(currentUser, mdProps);
        if (!errors.isEmpty()) {
            errors.forEach(error -> summary.addConflict("dataSet", error));
            continue;
        }
        // -----------------------------------------------------------------
        // Create complete data set registration
        // -----------------------------------------------------------------
        CompleteDataSetRegistration internalCdsr = createCompleteDataSetRegistration(cdsr, mdProps, now, storedBy);
        CompleteDataSetRegistration existingCdsr = config.isSkipExistingCheck() ? null : batchHandler.findObject(internalCdsr);
        ImportStrategy strategy = config.getStrategy();
        boolean isDryRun = config.isDryRun();
        if (!config.isSkipExistingCheck() && existingCdsr != null) {
            if (strategy.isCreateAndUpdate() || strategy.isUpdate()) {
                // Update existing CDSR
                updateCount++;
                if (!isDryRun) {
                    batchHandler.updateObject(internalCdsr);
                }
            } else if (strategy.isDelete()) {
                // TODO Does 'delete' even make sense for CDSR?
                // Replace existing CDSR
                deleteCount++;
                if (!isDryRun) {
                    batchHandler.deleteObject(internalCdsr);
                }
            }
        } else {
            if (strategy.isCreateAndUpdate() || strategy.isCreate()) {
                if (existingCdsr != null) {
                    // Already exists -> update
                    importCount++;
                    if (!isDryRun) {
                        batchHandler.updateObject(internalCdsr);
                    }
                } else {
                    // Does not exist -> add new CDSR
                    boolean added = false;
                    if (!isDryRun) {
                        added = batchHandler.addObject(internalCdsr);
                        if (added) {
                            sendNotifications(config, internalCdsr);
                        }
                    }
                    if (isDryRun || added) {
                        importCount++;
                    }
                }
            }
        }
    }
    batchHandler.flush();
    finalizeSummary(summary, totalCount, importCount, updateCount, deleteCount);
    return totalCount;
}

Example 57 with DataElementOperand

use of org.hisp.dhis.dataelement.DataElementOperand in project dhis2-core by dhis2.

the class SectionObjectBundleHook method preUpdate.

@Override
public void preUpdate(Section section, Section persistedObject, ObjectBundle bundle) {
    Set<DataElementOperand> returnGreyFields = new HashSet<>();
    for (DataElementOperand greyField : section.getGreyedFields()) {
        boolean exist = false;
        for (DataElement de : section.getDataElements()) {
            if (!ObjectUtils.allNonNull(de.getUid(), greyField.getDataElement())) {
                continue;
            }
            if (de.getUid().equals(greyField.getDataElement().getUid())) {
                exist = true;
                break;
            }
        }
        if (exist) {
            returnGreyFields.add(greyField);
        }
    }
    section.setGreyedFields(returnGreyFields);
}

Example 58 with DataElementOperand

use of org.hisp.dhis.dataelement.DataElementOperand in project dhis2-core by dhis2.

the class ObjectBundleServiceTest method testCreateDataSetWithSectionsAndGreyedFields.

@Test
void testCreateDataSetWithSectionsAndGreyedFields() throws IOException {
    Map<Class<? extends IdentifiableObject>, List<IdentifiableObject>> metadata = renderService.fromMetadata(new ClassPathResource("dxf2/dataset_with_sections_gf.json").getInputStream(), RenderFormat.JSON);
    ObjectBundleParams params = new ObjectBundleParams();
    params.setObjectBundleMode(ObjectBundleMode.COMMIT);
    params.setImportStrategy(ImportStrategy.CREATE);
    params.setObjects(metadata);
    ObjectBundle bundle = objectBundleService.create(params);
    ObjectBundleValidationReport validate = objectBundleValidationService.validate(bundle);
    assertFalse(validate.hasErrorReports());
    objectBundleService.commit(bundle);
    List<DataSet> dataSets = manager.getAll(DataSet.class);
    List<Section> sections = manager.getAll(Section.class);
    List<OrganisationUnit> organisationUnits = manager.getAll(OrganisationUnit.class);
    List<DataElement> dataElements = manager.getAll(DataElement.class);
    List<UserAuthorityGroup> userRoles = manager.getAll(UserAuthorityGroup.class);
    List<User> users = manager.getAll(User.class);
    List<DataElementOperand> dataElementOperands = manager.getAll(DataElementOperand.class);
    List<TrackedEntityType> trackedEntityTypes = manager.getAll(TrackedEntityType.class);
    List<OrganisationUnitLevel> organisationUnitLevels = manager.getAll(OrganisationUnitLevel.class);
    assertFalse(organisationUnits.isEmpty());
    assertEquals(1, organisationUnitLevels.size());
    assertEquals(1, trackedEntityTypes.size());
    assertFalse(dataElements.isEmpty());
    assertFalse(users.isEmpty());
    assertFalse(userRoles.isEmpty());
    assertEquals(1, dataSets.size());
    assertEquals(2, sections.size());
    assertEquals(1, dataElementOperands.size());
    DataSet dataSet = dataSets.get(0);
    assertEquals(2, dataSet.getSections().size());
    Section section1 = sections.get(0);
    Section section2 = sections.get(1);
    assertEquals(1, section1.getDataElements().size());
    assertEquals(1, section2.getDataElements().size());
    assertNotNull(section1.getDataSet());
    assertNotNull(section2.getDataSet());
    Section section = manager.get(Section.class, "C50M0WxaI7y");
    assertNotNull(section.getDataSet());
    assertEquals(1, section.getCategoryCombos().size());
    assertEquals(1, section.getGreyedFields().size());
    CategoryCombo categoryCombo = manager.get(CategoryCombo.class, "faV8QvLgIwB");
    assertNotNull(categoryCombo);
    Category category = manager.get(Category.class, "XJGLlMAMCcn");
    assertNotNull(category);
    CategoryOption categoryOption1 = manager.get(CategoryOption.class, "JYiFOMKa25J");
    CategoryOption categoryOption2 = manager.get(CategoryOption.class, "tdaMRD34m8o");
    assertNotNull(categoryOption1);
    assertNotNull(categoryOption2);
}

Example 59 with DataElementOperand

use of org.hisp.dhis.dataelement.DataElementOperand in project dhis2-core by dhis2.

the class ObjectBundleServiceTest method testUpdateDataSetWithSectionsAndGreyedFields.

@Test
void testUpdateDataSetWithSectionsAndGreyedFields() throws IOException {
    Map<Class<? extends IdentifiableObject>, List<IdentifiableObject>> metadata = renderService.fromMetadata(new ClassPathResource("dxf2/dataset_with_sections_gf.json").getInputStream(), RenderFormat.JSON);
    ObjectBundleParams params = new ObjectBundleParams();
    params.setObjectBundleMode(ObjectBundleMode.COMMIT);
    params.setImportStrategy(ImportStrategy.CREATE);
    params.setObjects(metadata);
    ObjectBundle bundle = objectBundleService.create(params);
    ObjectBundleValidationReport validate = objectBundleValidationService.validate(bundle);
    assertFalse(validate.hasErrorReports());
    objectBundleService.commit(bundle);
    dbmsManager.clearSession();
    Section section1 = manager.get(Section.class, "JwcV2ZifEQf");
    assertNotNull(section1.getDataSet());
    assertEquals(1, section1.getCategoryCombos().size());
    assertTrue(section1.getGreyedFields().isEmpty());
    assertEquals(1, section1.getDataElements().size());
    assertNotNull(section1.getDataSet());
    Section section2 = manager.get(Section.class, "C50M0WxaI7y");
    assertNotNull(section2.getDataSet());
    assertEquals(1, section2.getCategoryCombos().size());
    assertEquals(1, section2.getGreyedFields().size());
    assertEquals(1, section2.getDataElements().size());
    assertNotNull(section2.getDataSet());
    metadata = renderService.fromMetadata(new ClassPathResource("dxf2/dataset_with_sections_gf_update.json").getInputStream(), RenderFormat.JSON);
    params = new ObjectBundleParams();
    params.setObjectBundleMode(ObjectBundleMode.COMMIT);
    params.setImportStrategy(ImportStrategy.UPDATE);
    params.setObjects(metadata);
    bundle = objectBundleService.create(params);
    validate = objectBundleValidationService.validate(bundle);
    assertFalse(validate.hasErrorReports());
    objectBundleService.commit(bundle);
    manager.flush();
    List<DataSet> dataSets = manager.getAll(DataSet.class);
    List<Section> sections = manager.getAll(Section.class);
    List<OrganisationUnit> organisationUnits = manager.getAll(OrganisationUnit.class);
    List<DataElement> dataElements = manager.getAll(DataElement.class);
    List<UserAuthorityGroup> userRoles = manager.getAll(UserAuthorityGroup.class);
    List<User> users = manager.getAll(User.class);
    List<DataElementOperand> dataElementOperands = manager.getAll(DataElementOperand.class);
    assertFalse(organisationUnits.isEmpty());
    assertFalse(dataElements.isEmpty());
    assertFalse(users.isEmpty());
    assertFalse(userRoles.isEmpty());
    assertEquals(1, dataSets.size());
    assertEquals(2, sections.size());
    assertEquals(1, dataElementOperands.size());
    DataSet dataSet = dataSets.get(0);
    assertEquals("Updated Data Set", dataSet.getName());
    assertEquals(2, dataSet.getSections().size());
    assertNotNull(dataSet.getCreatedBy());
    section1 = manager.get(Section.class, "JwcV2ZifEQf");
    assertNotNull(section1.getDataSet());
    assertEquals(1, section1.getCategoryCombos().size());
    assertEquals(1, section1.getGreyedFields().size());
    assertEquals(1, section1.getDataElements().size());
    assertNotNull(section1.getDataSet());
    section2 = manager.get(Section.class, "C50M0WxaI7y");
    assertNotNull(section2.getDataSet());
    assertEquals(1, section2.getCategoryCombos().size());
    assertTrue(section2.getGreyedFields().isEmpty());
    assertEquals(1, section2.getDataElements().size());
    assertNotNull(section2.getDataSet());
}

Example 60 with DataElementOperand

use of org.hisp.dhis.dataelement.DataElementOperand in project dhis2-core by dhis2.

the class DefaultPredictionService method predict.

@Override
public void predict(Predictor predictor, Date startDate, Date endDate, PredictionSummary predictionSummary) {
    Expression generator = predictor.getGenerator();
    Expression skipTest = predictor.getSampleSkipTest();
    DataElement outputDataElement = predictor.getOutput();
    DataType expressionDataType = DataType.fromValueType(outputDataElement.getValueType());
    Map<DimensionalItemId, DimensionalItemObject> outputPeriodItemMap = new HashMap<>();
    Map<DimensionalItemId, DimensionalItemObject> sampledItemMap = new HashMap<>();
    expressionService.getExpressionDimensionalItemMaps(generator.getExpression(), PREDICTOR_EXPRESSION, expressionDataType, outputPeriodItemMap, sampledItemMap);
    Set<String> orgUnitGroupIds = expressionService.getExpressionOrgUnitGroupIds(generator.getExpression(), PREDICTOR_EXPRESSION);
    if (skipTest != null) {
        expressionService.getExpressionDimensionalItemMaps(skipTest.getExpression(), PREDICTOR_SKIP_TEST, DataType.BOOLEAN, sampledItemMap, sampledItemMap);
        orgUnitGroupIds.addAll(expressionService.getExpressionOrgUnitGroupIds(skipTest.getExpression(), PREDICTOR_SKIP_TEST));
    }
    Map<String, OrganisationUnitGroup> orgUnitGroupMap = orgUnitGroupIds.stream().map(organisationUnitGroupService::getOrganisationUnitGroup).filter(Objects::nonNull).collect(Collectors.toMap(OrganisationUnitGroup::getUid, g -> g));
    Map<DimensionalItemId, DimensionalItemObject> itemMap = new HashMap<>(outputPeriodItemMap);
    itemMap.putAll(sampledItemMap);
    Set<DimensionalItemObject> outputPeriodItems = new HashSet<>(outputPeriodItemMap.values());
    Set<DimensionalItemObject> sampledItems = new HashSet<>(sampledItemMap.values());
    Set<DimensionalItemObject> items = new HashSet<>(itemMap.values());
    List<Period> outputPeriods = getPeriodsBetweenDates(predictor.getPeriodType(), startDate, endDate);
    Set<Period> existingOutputPeriods = getExistingPeriods(outputPeriods);
    ListMap<Period, Period> samplePeriodsMap = getSamplePeriodsMap(outputPeriods, predictor);
    Set<Period> allSamplePeriods = samplePeriodsMap.uniqueValues();
    Set<Period> analyticsQueryPeriods = getAnalyticsQueryPeriods(sampledItems, allSamplePeriods, outputPeriodItems, existingOutputPeriods);
    Set<Period> dataValueQueryPeriods = getDataValueQueryPeriods(analyticsQueryPeriods, existingOutputPeriods);
    outputPeriods = periodService.reloadPeriods(outputPeriods);
    CategoryOptionCombo defaultCategoryOptionCombo = categoryService.getDefaultCategoryOptionCombo();
    CategoryOptionCombo outputOptionCombo = predictor.getOutputCombo() == null ? defaultCategoryOptionCombo : predictor.getOutputCombo();
    DataElementOperand outputDataElementOperand = new DataElementOperand(outputDataElement, outputOptionCombo);
    Date now = new Date();
    boolean requireData = generator.getMissingValueStrategy() != NEVER_SKIP && (!items.isEmpty());
    DimensionalItemObject forwardReference = addOuputToItems(outputDataElementOperand, items);
    Set<OrganisationUnit> currentUserOrgUnits = new HashSet<>();
    String storedBy = "system-process";
    User currentUser = currentUserService.getCurrentUser();
    if (currentUser != null) {
        currentUserOrgUnits = currentUser.getOrganisationUnits();
        storedBy = currentUser.getUsername();
    }
    PredictionDataConsolidator consolidator = new PredictionDataConsolidator(items, predictor.getOrganisationUnitDescendants().equals(DESCENDANTS), new PredictionDataValueFetcher(dataValueService, categoryService), new PredictionAnalyticsDataFetcher(analyticsService, categoryService));
    PredictionWriter predictionWriter = new PredictionWriter(dataValueService, batchHandlerFactory);
    predictionWriter.init(existingOutputPeriods, predictionSummary);
    predictionSummary.incrementPredictors();
    for (OrganisationUnitLevel orgUnitLevel : predictor.getOrganisationUnitLevels()) {
        List<OrganisationUnit> orgUnits = organisationUnitService.getOrganisationUnitsAtOrgUnitLevels(Lists.newArrayList(orgUnitLevel), currentUserOrgUnits);
        consolidator.init(currentUserOrgUnits, orgUnitLevel.getLevel(), orgUnits, dataValueQueryPeriods, analyticsQueryPeriods, existingOutputPeriods, outputDataElementOperand);
        PredictionData data;
        while ((data = consolidator.getData()) != null) {
            List<DataValue> predictions = new ArrayList<>();
            List<PredictionContext> contexts = PredictionContextGenerator.getContexts(outputPeriods, data.getValues(), defaultCategoryOptionCombo);
            for (PredictionContext c : contexts) {
                List<Period> samplePeriods = new ArrayList<>(samplePeriodsMap.get(c.getOutputPeriod()));
                samplePeriods.removeAll(getSkippedPeriods(allSamplePeriods, itemMap, c.getPeriodValueMap(), skipTest, orgUnitGroupMap, data.getOrgUnit()));
                if (requireData && !dataIsPresent(outputPeriodItems, c.getValueMap(), sampledItems, samplePeriods, c.getPeriodValueMap())) {
                    continue;
                }
                Object value = expressionService.getExpressionValue(ExpressionParams.builder().expression(predictor.getGenerator().getExpression()).parseType(PREDICTOR_EXPRESSION).dataType(expressionDataType).itemMap(itemMap).valueMap(c.getValueMap()).orgUnitGroupMap(orgUnitGroupMap).days(c.getOutputPeriod().getDaysInPeriod()).missingValueStrategy(generator.getMissingValueStrategy()).orgUnit(data.getOrgUnit()).samplePeriods(samplePeriods).periodValueMap(c.getPeriodValueMap()).build());
                if (value != null || generator.getMissingValueStrategy() == NEVER_SKIP) {
                    String valueString = formatPrediction(value, outputDataElement);
                    if (valueString != null) {
                        DataValue prediction = new DataValue(outputDataElement, c.getOutputPeriod(), data.getOrgUnit(), outputOptionCombo, c.getAttributeOptionCombo(), valueString, storedBy, now, null);
                        carryPredictionForward(prediction, contexts, forwardReference);
                        predictions.add(prediction);
                    }
                }
            }
            predictionWriter.write(predictions, data.getOldPredictions());
        }
    }
    predictionWriter.flush();
}