Examples with HiveSyncTool org.apache.hudi.hive.HiveSyncTool used on opensource projects

Example 1 with HiveSyncTool

use of org.apache.hudi.hive.HiveSyncTool in project hudi by apache.

the class BootstrapExecutor method syncHive.

/**
 * Sync to Hive.
 */
private void syncHive() {
    if (cfg.enableHiveSync || cfg.enableMetaSync) {
        HiveSyncConfig hiveSyncConfig = DataSourceUtils.buildHiveSyncConfig(props, cfg.targetBasePath, cfg.baseFileFormat);
        HiveConf hiveConf = new HiveConf(fs.getConf(), HiveConf.class);
        hiveConf.set(HiveConf.ConfVars.METASTOREURIS.varname, hiveSyncConfig.metastoreUris);
        LOG.info("Hive Conf => " + hiveConf.getAllProperties().toString());
        LOG.info("Hive Sync Conf => " + hiveSyncConfig);
        new HiveSyncTool(hiveSyncConfig, new HiveConf(configuration, HiveConf.class), fs).syncHoodieTable();
    }
}

Example 2 with HiveSyncTool

use of org.apache.hudi.hive.HiveSyncTool in project hudi by apache.

the class DeltaSync method syncMeta.

private void syncMeta(HoodieDeltaStreamerMetrics metrics) {
    Set<String> syncClientToolClasses = new HashSet<>(Arrays.asList(cfg.syncClientToolClass.split(",")));
    // for backward compatibility
    if (cfg.enableHiveSync) {
        cfg.enableMetaSync = true;
        syncClientToolClasses.add(HiveSyncTool.class.getName());
        LOG.info("When set --enable-hive-sync will use HiveSyncTool for backward compatibility");
    }
    if (cfg.enableMetaSync) {
        for (String impl : syncClientToolClasses) {
            Timer.Context syncContext = metrics.getMetaSyncTimerContext();
            impl = impl.trim();
            switch(impl) {
                case "org.apache.hudi.hive.HiveSyncTool":
                    syncHive();
                    break;
                default:
                    FileSystem fs = FSUtils.getFs(cfg.targetBasePath, jssc.hadoopConfiguration());
                    Properties properties = new Properties();
                    properties.putAll(props);
                    properties.put("basePath", cfg.targetBasePath);
                    properties.put("baseFileFormat", cfg.baseFileFormat);
                    AbstractSyncTool syncTool = (AbstractSyncTool) ReflectionUtils.loadClass(impl, new Class[] { Properties.class, FileSystem.class }, properties, fs);
                    syncTool.syncHoodieTable();
            }
            long metaSyncTimeMs = syncContext != null ? syncContext.stop() : 0;
            metrics.updateDeltaStreamerMetaSyncMetrics(getSyncClassShortName(impl), metaSyncTimeMs);
        }
    }
}

Example 3 with HiveSyncTool

use of org.apache.hudi.hive.HiveSyncTool in project hudi by apache.

the class DeltaSync method syncHive.

public void syncHive() {
    HiveSyncConfig hiveSyncConfig = DataSourceUtils.buildHiveSyncConfig(props, cfg.targetBasePath, cfg.baseFileFormat);
    HiveConf hiveConf = new HiveConf(conf, HiveConf.class);
    if (StringUtils.isNullOrEmpty(hiveConf.get(HiveConf.ConfVars.METASTOREURIS.varname))) {
        hiveConf.set(HiveConf.ConfVars.METASTOREURIS.varname, hiveSyncConfig.metastoreUris);
    }
    LOG.info("Hive Conf => " + hiveConf.getAllProperties().toString());
    LOG.info("Hive Sync Conf => " + hiveSyncConfig.toString());
    new HiveSyncTool(hiveSyncConfig, hiveConf, fs).syncHoodieTable();
}

Example 4 with HiveSyncTool

use of org.apache.hudi.hive.HiveSyncTool in project hudi by apache.

the class TestHiveIncrementalPuller method createSourceTable.

private void createSourceTable() throws IOException, URISyntaxException {
    String instantTime = "101";
    HiveTestUtil.createCOWTable(instantTime, 5, true);
    hiveSyncConfig.syncMode = "jdbc";
    HiveTestUtil.hiveSyncConfig.batchSyncNum = 3;
    HiveSyncTool tool = new HiveSyncTool(hiveSyncConfig, HiveTestUtil.getHiveConf(), fileSystem);
    tool.syncHoodieTable();
}

Example 5 with HiveSyncTool

use of org.apache.hudi.hive.HiveSyncTool in project hudi by apache.

the class TestHiveIncrementalPuller method createTargetTable.

private void createTargetTable() throws IOException, URISyntaxException {
    String instantTime = "100";
    targetBasePath = Files.createTempDirectory("hivesynctest1" + Instant.now().toEpochMilli()).toUri().toString();
    HiveTestUtil.createCOWTable(instantTime, 5, true, targetBasePath, "tgtdb", "test2");
    HiveSyncTool tool = new HiveSyncTool(getTargetHiveSyncConfig(targetBasePath), HiveTestUtil.getHiveConf(), fileSystem);
    tool.syncHoodieTable();
}