Examples with AnalyticsTableType org.hisp.dhis.analytics.AnalyticsTableType used on opensource projects

Example 1 with AnalyticsTableType

use of org.hisp.dhis.analytics.AnalyticsTableType in project dhis2-core by dhis2.

the class AbstractJdbcTableManager method invokeAnalyticsTableSqlHooks.

@Override
public int invokeAnalyticsTableSqlHooks() {
    AnalyticsTableType type = getAnalyticsTableType();
    List<AnalyticsTableHook> hooks = tableHookService.getByPhaseAndAnalyticsTableType(AnalyticsTablePhase.ANALYTICS_TABLE_POPULATED, type);
    tableHookService.executeAnalyticsTableSqlHooks(hooks);
    return hooks.size();
}

Example 2 with AnalyticsTableType

use of org.hisp.dhis.analytics.AnalyticsTableType in project dhis2-core by dhis2.

the class DefaultAnalyticsTableService method update.

@Override
public void update(AnalyticsTableUpdateParams params, JobProgress progress) {
    final int processNo = getProcessNo();
    int tableUpdates = 0;
    log.info(String.format("Analytics table update parameters: %s", params));
    AnalyticsTableType tableType = getAnalyticsTableType();
    Clock clock = new Clock(log).startClock().logTime(String.format("Starting update of type: %s, table name: '%s', processes: %d", tableType, tableType.getTableName(), processNo));
    progress.startingStage("Validating Analytics Table " + tableType);
    String validState = tableManager.validState();
    progress.completedStage(validState);
    if (validState != null || progress.isCancellationRequested()) {
        return;
    }
    final List<AnalyticsTable> tables = tableManager.getAnalyticsTables(params);
    if (tables.isEmpty()) {
        clock.logTime(String.format("Table update aborted, no table or partitions to be updated: '%s'", tableType.getTableName()));
        progress.startingStage("Table updates " + tableType);
        progress.completedStage("Table updated aborted, no table or partitions to be updated");
        return;
    }
    clock.logTime(String.format("Table update start: %s, earliest: %s, parameters: %s", tableType.getTableName(), getLongDateString(params.getFromDate()), params.toString()));
    progress.startingStage("Performing pre-create table work");
    progress.runStage(() -> tableManager.preCreateTables(params));
    clock.logTime("Performed pre-create table work " + tableType);
    progress.startingStage("Dropping temp tables " + tableType, tables.size());
    dropTempTables(tables, progress);
    clock.logTime("Dropped temp tables");
    progress.startingStage("Creating analytics tables " + tableType, tables.size());
    createTables(tables, progress);
    clock.logTime("Created analytics tables");
    List<AnalyticsTablePartition> partitions = PartitionUtils.getTablePartitions(tables);
    progress.startingStage("Populating analytics tables " + tableType, partitions.size());
    populateTables(params, partitions, progress);
    clock.logTime("Populated analytics tables");
    progress.startingStage("Invoking analytics table hooks " + tableType);
    tableUpdates += progress.runStage(0, tableManager::invokeAnalyticsTableSqlHooks);
    clock.logTime("Invoked analytics table hooks");
    tableUpdates += applyAggregationLevels(tableType, partitions, progress);
    clock.logTime("Applied aggregation levels");
    if (tableUpdates > 0) {
        progress.startingStage("Vacuuming tables " + tableType, partitions.size());
        vacuumTables(partitions, progress);
        clock.logTime("Tables vacuumed");
    }
    List<AnalyticsIndex> indexes = getIndexes(partitions);
    progress.startingStage("Creating indexes " + tableType, indexes.size());
    createIndexes(indexes, progress);
    clock.logTime("Created indexes");
    progress.startingStage("Analyzing analytics tables " + tableType, partitions.size());
    analyzeTables(partitions, progress);
    clock.logTime("Analyzed tables");
    if (params.isLatestUpdate()) {
        progress.startingStage("Removing updated and deleted data " + tableType);
        tableManager.removeUpdatedData(tables);
        clock.logTime("Removed updated and deleted data");
    }
    swapTables(params, tables, progress);
    clock.logTime("Table update done: " + tableType.getTableName());
}

Example 3 with AnalyticsTableType

use of org.hisp.dhis.analytics.AnalyticsTableType in project dhis2-core by dhis2.

the class DefaultAnalyticsTableGenerator method generateTables.

// TODO introduce last successful timestamps per table type
@Override
public void generateTables(AnalyticsTableUpdateParams params, JobProgress progress) {
    final Clock clock = new Clock(log).startClock();
    final Date lastSuccessfulUpdate = systemSettingManager.getDateSetting(SettingKey.LAST_SUCCESSFUL_ANALYTICS_TABLES_UPDATE);
    final Set<AnalyticsTableType> skipTypes = CollectionUtils.emptyIfNull(params.getSkipTableTypes());
    final Set<AnalyticsTableType> availableTypes = analyticsTableServices.stream().map(AnalyticsTableService::getAnalyticsTableType).collect(Collectors.toSet());
    params = AnalyticsTableUpdateParams.newBuilder(params).withLastSuccessfulUpdate(lastSuccessfulUpdate).build();
    log.info("Found {} analytics table types: {}", availableTypes.size(), availableTypes);
    log.info("Analytics table update: {}", params);
    log.info("Last successful analytics table update: '{}'", getLongDateString(lastSuccessfulUpdate));
    progress.startingProcess("Analytics table update process started");
    try {
        if (!params.isSkipResourceTables() && !params.isLatestUpdate()) {
            generateResourceTablesInternal(progress);
        }
        for (AnalyticsTableService service : analyticsTableServices) {
            AnalyticsTableType tableType = service.getAnalyticsTableType();
            if (!skipTypes.contains(tableType)) {
                service.update(params, progress);
            }
        }
        clock.logTime("Analytics tables updated");
        progress.completedProcess("Analytics tables updated: " + clock.time());
    } catch (Exception ex) {
        log.error("Analytics table process failed: " + DebugUtils.getStackTrace(ex), ex);
        progress.failedProcess(ex);
        messageService.sendSystemErrorNotification("Analytics table process failed", ex);
        throw ex;
    }
    if (params.isLatestUpdate()) {
        systemSettingManager.saveSystemSetting(SettingKey.LAST_SUCCESSFUL_LATEST_ANALYTICS_PARTITION_UPDATE, params.getStartTime());
        systemSettingManager.saveSystemSetting(SettingKey.LAST_SUCCESSFUL_LATEST_ANALYTICS_PARTITION_RUNTIME, clock.time());
    } else {
        systemSettingManager.saveSystemSetting(SettingKey.LAST_SUCCESSFUL_ANALYTICS_TABLES_UPDATE, params.getStartTime());
        systemSettingManager.saveSystemSetting(SettingKey.LAST_SUCCESSFUL_ANALYTICS_TABLES_RUNTIME, clock.time());
    }
    analyticsCache.invalidateAll();
}

Example 4 with AnalyticsTableType

use of org.hisp.dhis.analytics.AnalyticsTableType in project dhis2-core by dhis2.

the class DefaultAnalyticsTableService method createIndexes.

/**
 * Creates indexes on the given analytics tables.
 */
private void createIndexes(List<AnalyticsIndex> indexes, JobProgress progress) {
    AnalyticsTableType type = getAnalyticsTableType();
    log.info("No of analytics table indexes: " + indexes.size());
    progress.runStageInParallel(getProcessNo(), indexes, index -> getIndexName(index, type).replace("\"", ""), tableManager::createIndex);
}

Example 5 with AnalyticsTableType

use of org.hisp.dhis.analytics.AnalyticsTableType in project dhis2-core by dhis2.

the class ResourceTableController method analytics.

@RequestMapping(value = "/analytics", method = { RequestMethod.PUT, RequestMethod.POST })
@PreAuthorize("hasRole('ALL') or hasRole('F_PERFORM_MAINTENANCE')")
@ResponseBody
public WebMessage analytics(@RequestParam(required = false) boolean skipResourceTables, @RequestParam(required = false) boolean skipAggregate, @RequestParam(required = false) boolean skipEvents, @RequestParam(required = false) boolean skipEnrollment, @RequestParam(required = false) Integer lastYears) {
    Set<AnalyticsTableType> skipTableTypes = new HashSet<>();
    Set<String> skipPrograms = new HashSet<>();
    if (skipAggregate) {
        skipTableTypes.add(AnalyticsTableType.DATA_VALUE);
        skipTableTypes.add(AnalyticsTableType.COMPLETENESS);
        skipTableTypes.add(AnalyticsTableType.COMPLETENESS_TARGET);
    }
    if (skipEvents) {
        skipTableTypes.add(AnalyticsTableType.EVENT);
    }
    if (skipEnrollment) {
        skipTableTypes.add(AnalyticsTableType.ENROLLMENT);
    }
    AnalyticsJobParameters analyticsJobParameters = new AnalyticsJobParameters(lastYears, skipTableTypes, skipPrograms, skipResourceTables);
    JobConfiguration analyticsTableJob = new JobConfiguration("inMemoryAnalyticsJob", JobType.ANALYTICS_TABLE, "", analyticsJobParameters, true, true);
    analyticsTableJob.setUserUid(currentUserService.getCurrentUser().getUid());
    schedulingManager.executeNow(analyticsTableJob);
    return jobConfigurationReport(analyticsTableJob);
}