Examples with AnalyticsTablePartition org.hisp.dhis.analytics.AnalyticsTablePartition used on opensource projects

Example 1 with AnalyticsTablePartition

use of org.hisp.dhis.analytics.AnalyticsTablePartition in project dhis2-core by dhis2.

the class DefaultAnalyticsTableService method update.

@Override
public void update(AnalyticsTableUpdateParams params, JobProgress progress) {
    final int processNo = getProcessNo();
    int tableUpdates = 0;
    log.info(String.format("Analytics table update parameters: %s", params));
    AnalyticsTableType tableType = getAnalyticsTableType();
    Clock clock = new Clock(log).startClock().logTime(String.format("Starting update of type: %s, table name: '%s', processes: %d", tableType, tableType.getTableName(), processNo));
    progress.startingStage("Validating Analytics Table " + tableType);
    String validState = tableManager.validState();
    progress.completedStage(validState);
    if (validState != null || progress.isCancellationRequested()) {
        return;
    }
    final List<AnalyticsTable> tables = tableManager.getAnalyticsTables(params);
    if (tables.isEmpty()) {
        clock.logTime(String.format("Table update aborted, no table or partitions to be updated: '%s'", tableType.getTableName()));
        progress.startingStage("Table updates " + tableType);
        progress.completedStage("Table updated aborted, no table or partitions to be updated");
        return;
    }
    clock.logTime(String.format("Table update start: %s, earliest: %s, parameters: %s", tableType.getTableName(), getLongDateString(params.getFromDate()), params.toString()));
    progress.startingStage("Performing pre-create table work");
    progress.runStage(() -> tableManager.preCreateTables(params));
    clock.logTime("Performed pre-create table work " + tableType);
    progress.startingStage("Dropping temp tables " + tableType, tables.size());
    dropTempTables(tables, progress);
    clock.logTime("Dropped temp tables");
    progress.startingStage("Creating analytics tables " + tableType, tables.size());
    createTables(tables, progress);
    clock.logTime("Created analytics tables");
    List<AnalyticsTablePartition> partitions = PartitionUtils.getTablePartitions(tables);
    progress.startingStage("Populating analytics tables " + tableType, partitions.size());
    populateTables(params, partitions, progress);
    clock.logTime("Populated analytics tables");
    progress.startingStage("Invoking analytics table hooks " + tableType);
    tableUpdates += progress.runStage(0, tableManager::invokeAnalyticsTableSqlHooks);
    clock.logTime("Invoked analytics table hooks");
    tableUpdates += applyAggregationLevels(tableType, partitions, progress);
    clock.logTime("Applied aggregation levels");
    if (tableUpdates > 0) {
        progress.startingStage("Vacuuming tables " + tableType, partitions.size());
        vacuumTables(partitions, progress);
        clock.logTime("Tables vacuumed");
    }
    List<AnalyticsIndex> indexes = getIndexes(partitions);
    progress.startingStage("Creating indexes " + tableType, indexes.size());
    createIndexes(indexes, progress);
    clock.logTime("Created indexes");
    progress.startingStage("Analyzing analytics tables " + tableType, partitions.size());
    analyzeTables(partitions, progress);
    clock.logTime("Analyzed tables");
    if (params.isLatestUpdate()) {
        progress.startingStage("Removing updated and deleted data " + tableType);
        tableManager.removeUpdatedData(tables);
        clock.logTime("Removed updated and deleted data");
    }
    swapTables(params, tables, progress);
    clock.logTime("Table update done: " + tableType.getTableName());
}

Example 2 with AnalyticsTablePartition

use of org.hisp.dhis.analytics.AnalyticsTablePartition in project dhis2-core by dhis2.

the class JdbcAnalyticsTableManager method removeUpdatedData.

@Override
public void removeUpdatedData(List<AnalyticsTable> tables) {
    AnalyticsTablePartition partition = getLatestTablePartition(tables);
    String sql = "delete from " + quote(getAnalyticsTableType().getTableName()) + " ax " + "where ax.id in (" + "select (de.uid || '-' || ps.iso || '-' || ou.uid || '-' || co.uid || '-' || ao.uid) as id " + "from datavalue dv " + "inner join dataelement de on dv.dataelementid=de.dataelementid " + "inner join _periodstructure ps on dv.periodid=ps.periodid " + "inner join organisationunit ou on dv.sourceid=ou.organisationunitid " + "inner join categoryoptioncombo co on dv.categoryoptioncomboid=co.categoryoptioncomboid " + "inner join categoryoptioncombo ao on dv.attributeoptioncomboid=ao.categoryoptioncomboid " + "where dv.lastupdated >= '" + getLongDateString(partition.getStartDate()) + "' " + "and dv.lastupdated < '" + getLongDateString(partition.getEndDate()) + "')";
    invokeTimeAndLog(sql, "Remove updated data values");
}

Example 3 with AnalyticsTablePartition

use of org.hisp.dhis.analytics.AnalyticsTablePartition in project dhis2-core by dhis2.

the class JdbcEventAnalyticsTableManager method removeUpdatedData.

@Override
public void removeUpdatedData(List<AnalyticsTable> tables) {
    for (AnalyticsTable table : tables) {
        AnalyticsTablePartition partition = table.getLatestPartition();
        String sql = "delete from " + quote(table.getTableName()) + " ax " + "where ax.psi in (" + "select psi.uid " + "from programstageinstance psi " + "inner join programinstance pi on psi.programinstanceid=pi.programinstanceid " + "where pi.programid = " + table.getProgram().getId() + " " + "and psi.lastupdated >= '" + getLongDateString(partition.getStartDate()) + "' " + "and psi.lastupdated < '" + getLongDateString(partition.getEndDate()) + "')";
        invokeTimeAndLog(sql, String.format("Remove updated events for table: '%s'", table.getTableName()));
    }
}

Example 4 with AnalyticsTablePartition

use of org.hisp.dhis.analytics.AnalyticsTablePartition in project dhis2-core by dhis2.

the class JdbcEventAnalyticsTableManagerTest method verifyGetLatestAnalyticsTables.

@Test
void verifyGetLatestAnalyticsTables() {
    Program prA = createProgram('A');
    Program prB = createProgram('B');
    Program prC = createProgram('C');
    Program prD = createProgram('D');
    List<Program> programs = Lists.newArrayList(prA, prB, prC, prD);
    Date lastFullTableUpdate = new DateTime(2019, 3, 1, 2, 0).toDate();
    Date lastLatestPartitionUpdate = new DateTime(2019, 3, 1, 9, 0).toDate();
    Date startTime = new DateTime(2019, 3, 1, 10, 0).toDate();
    Set<String> skipPrograms = new HashSet<>();
    skipPrograms.add(prC.getUid());
    skipPrograms.add(prD.getUid());
    AnalyticsTableUpdateParams params = AnalyticsTableUpdateParams.newBuilder().withStartTime(startTime).withLatestPartition().withSkipPrograms(skipPrograms).build();
    List<Map<String, Object>> queryResp = Lists.newArrayList();
    queryResp.add(ImmutableMap.of("dataelementid", 1));
    when(systemSettingManager.getDateSetting(SettingKey.LAST_SUCCESSFUL_ANALYTICS_TABLES_UPDATE)).thenReturn(lastFullTableUpdate);
    when(systemSettingManager.getDateSetting(SettingKey.LAST_SUCCESSFUL_LATEST_ANALYTICS_PARTITION_UPDATE)).thenReturn(lastLatestPartitionUpdate);
    when(jdbcTemplate.queryForList(Mockito.anyString())).thenReturn(queryResp);
    when(idObjectManager.getAllNoAcl(Program.class)).thenReturn(programs);
    List<AnalyticsTable> tables = subject.getAnalyticsTables(params);
    assertThat(tables, hasSize(2));
    AnalyticsTable tableA = tables.get(0);
    AnalyticsTable tableB = tables.get(1);
    assertThat(tableA, notNullValue());
    assertThat(tableB, notNullValue());
    AnalyticsTablePartition partitionA = tableA.getLatestPartition();
    AnalyticsTablePartition partitionB = tableA.getLatestPartition();
    assertThat(partitionA, notNullValue());
    assertThat(partitionA.isLatestPartition(), equalTo(true));
    assertThat(partitionA.getStartDate(), equalTo(lastFullTableUpdate));
    assertThat(partitionA.getEndDate(), equalTo(startTime));
    assertThat(partitionB, notNullValue());
    assertThat(partitionB.isLatestPartition(), equalTo(true));
    assertThat(partitionB.getStartDate(), equalTo(lastFullTableUpdate));
    assertThat(partitionB.getEndDate(), equalTo(startTime));
}

Example 5 with AnalyticsTablePartition

use of org.hisp.dhis.analytics.AnalyticsTablePartition in project dhis2-core by dhis2.

the class AnalyticsTableTest method testGetTablePartitionName.

@Test
void testGetTablePartitionName() {
    Program program = new Program("ProgramA", "DescriptionA");
    program.setUid("UIDA");
    Period periodA = new YearlyPeriodType().createPeriod(new DateTime(2014, 1, 1, 0, 0).toDate());
    Period periodB = new YearlyPeriodType().createPeriod(new DateTime(2015, 1, 1, 0, 0).toDate());
    AnalyticsTable tableA = new AnalyticsTable(AnalyticsTableType.EVENT, Lists.newArrayList(), Lists.newArrayList(), program);
    tableA.addPartitionTable(2014, periodA.getStartDate(), periodA.getEndDate());
    tableA.addPartitionTable(2015, periodB.getStartDate(), periodB.getEndDate());
    AnalyticsTablePartition partitionA = tableA.getTablePartitions().get(0);
    AnalyticsTablePartition partitionB = tableA.getTablePartitions().get(1);
    assertNotNull(partitionA);
    assertNotNull(partitionB);
    assertEquals("analytics_event_uida_2014", partitionA.getTableName());
    assertEquals("analytics_event_uida_2015", partitionB.getTableName());
    assertEquals("analytics_event_temp_uida_2014", partitionA.getTempTableName());
    assertEquals("analytics_event_temp_uida_2015", partitionB.getTempTableName());
}