Examples with UpgradeDowngrade org.apache.hudi.table.upgrade.UpgradeDowngrade used on opensource projects

Example 1 with UpgradeDowngrade

use of org.apache.hudi.table.upgrade.UpgradeDowngrade in project hudi by apache.

the class HoodieFlinkWriteClient method upgradeDowngrade.

/**
 * Upgrade downgrade the Hoodie table.
 *
 * <p>This action should only be executed once for each commit.
 * The modification of the table properties is not thread safe.
 */
public void upgradeDowngrade(String instantTime) {
    HoodieTableMetaClient metaClient = createMetaClient(true);
    new UpgradeDowngrade(metaClient, config, context, FlinkUpgradeDowngradeHelper.getInstance()).run(HoodieTableVersion.current(), instantTime);
}

Example 2 with UpgradeDowngrade

use of org.apache.hudi.table.upgrade.UpgradeDowngrade in project hudi by apache.

the class BaseHoodieWriteClient method tryUpgrade.

private void tryUpgrade(HoodieTableMetaClient metaClient, Option<String> instantTime) {
    UpgradeDowngrade upgradeDowngrade = new UpgradeDowngrade(metaClient, config, context, upgradeDowngradeHelper);
    if (upgradeDowngrade.needsUpgradeOrDowngrade(HoodieTableVersion.current())) {
        // Ensure no inflight commits by setting EAGER policy and explicitly cleaning all failed commits
        List<String> instantsToRollback = getInstantsToRollback(metaClient, HoodieFailedWritesCleaningPolicy.EAGER, instantTime);
        Map<String, Option<HoodiePendingRollbackInfo>> pendingRollbacks = getPendingRollbackInfos(metaClient);
        instantsToRollback.forEach(entry -> pendingRollbacks.putIfAbsent(entry, Option.empty()));
        rollbackFailedWrites(pendingRollbacks, true);
        new UpgradeDowngrade(metaClient, config, context, upgradeDowngradeHelper).run(HoodieTableVersion.current(), instantTime.orElse(null));
        metaClient.reloadActiveTimeline();
    }
}

Example 3 with UpgradeDowngrade

use of org.apache.hudi.table.upgrade.UpgradeDowngrade in project hudi by apache.

the class TestHoodieBackedMetadata method testUpgradeDowngrade.

@Test
public void testUpgradeDowngrade() throws IOException {
    init(HoodieTableType.COPY_ON_WRITE, false);
    HoodieSparkEngineContext engineContext = new HoodieSparkEngineContext(jsc);
    // Perform a commit. This should bootstrap the metadata table with latest version.
    List<HoodieRecord> records;
    List<WriteStatus> writeStatuses;
    String commitTimestamp = HoodieActiveTimeline.createNewInstantTime();
    HoodieWriteConfig writeConfig = getWriteConfig(true, true);
    try (SparkRDDWriteClient client = new SparkRDDWriteClient(engineContext, writeConfig)) {
        records = dataGen.generateInserts(commitTimestamp, 5);
        client.startCommitWithTime(commitTimestamp);
        writeStatuses = client.bulkInsert(jsc.parallelize(records, 1), commitTimestamp).collect();
        assertNoWriteErrors(writeStatuses);
    }
    // Metadata table should have been bootstrapped
    assertTrue(fs.exists(new Path(metadataTableBasePath)), "Metadata table should exist");
    FileStatus oldStatus = fs.getFileStatus(new Path(metadataTableBasePath));
    // set hoodie.table.version to 2 in hoodie.properties file
    changeTableVersion(HoodieTableVersion.TWO);
    // With next commit the table should be deleted (as part of upgrade) and then re-bootstrapped automatically
    commitTimestamp = HoodieActiveTimeline.createNewInstantTime();
    metaClient.reloadActiveTimeline();
    FileStatus prevStatus = fs.getFileStatus(new Path(metadataTableBasePath));
    try (SparkRDDWriteClient client = new SparkRDDWriteClient(engineContext, getWriteConfig(true, true))) {
        records = dataGen.generateInserts(commitTimestamp, 5);
        client.startCommitWithTime(commitTimestamp);
        writeStatuses = client.bulkInsert(jsc.parallelize(records, 1), commitTimestamp).collect();
        assertNoWriteErrors(writeStatuses);
    }
    assertTrue(fs.exists(new Path(metadataTableBasePath)), "Metadata table should exist");
    FileStatus currentStatus = fs.getFileStatus(new Path(metadataTableBasePath));
    assertTrue(currentStatus.getModificationTime() > prevStatus.getModificationTime());
    initMetaClient();
    assertEquals(metaClient.getTableConfig().getTableVersion().versionCode(), HoodieTableVersion.FOUR.versionCode());
    assertTrue(fs.exists(new Path(metadataTableBasePath)), "Metadata table should exist");
    FileStatus newStatus = fs.getFileStatus(new Path(metadataTableBasePath));
    assertTrue(oldStatus.getModificationTime() < newStatus.getModificationTime());
    // Test downgrade by running the downgrader
    new UpgradeDowngrade(metaClient, writeConfig, context, SparkUpgradeDowngradeHelper.getInstance()).run(HoodieTableVersion.TWO, null);
    assertEquals(metaClient.getTableConfig().getTableVersion().versionCode(), HoodieTableVersion.TWO.versionCode());
    assertFalse(fs.exists(new Path(metadataTableBasePath)), "Metadata table should not exist");
}

Example 4 with UpgradeDowngrade

use of org.apache.hudi.table.upgrade.UpgradeDowngrade in project hudi by apache.

the class SparkMain method upgradeOrDowngradeTable.

/**
 * Upgrade or downgrade table.
 *
 * @param jsc instance of {@link JavaSparkContext} to use.
 * @param basePath base path of the dataset.
 * @param toVersion version to which upgrade/downgrade to be done.
 * @return 0 if success, else -1.
 * @throws Exception
 */
protected static int upgradeOrDowngradeTable(JavaSparkContext jsc, String basePath, String toVersion) {
    HoodieWriteConfig config = getWriteConfig(basePath, Boolean.parseBoolean(HoodieWriteConfig.ROLLBACK_USING_MARKERS_ENABLE.defaultValue()));
    HoodieTableMetaClient metaClient = HoodieTableMetaClient.builder().setConf(jsc.hadoopConfiguration()).setBasePath(config.getBasePath()).setLoadActiveTimelineOnLoad(false).setConsistencyGuardConfig(config.getConsistencyGuardConfig()).setLayoutVersion(Option.of(new TimelineLayoutVersion(config.getTimelineLayoutVersion()))).setFileSystemRetryConfig(config.getFileSystemRetryConfig()).build();
    try {
        new UpgradeDowngrade(metaClient, config, new HoodieSparkEngineContext(jsc), SparkUpgradeDowngradeHelper.getInstance()).run(HoodieTableVersion.valueOf(toVersion), null);
        LOG.info(String.format("Table at \"%s\" upgraded / downgraded to version \"%s\".", basePath, toVersion));
        return 0;
    } catch (Exception e) {
        LOG.warn(String.format("Failed: Could not upgrade/downgrade table at \"%s\" to version \"%s\".", basePath, toVersion), e);
        return -1;
    }
}