Examples with AbstractSyncTool org.apache.hudi.sync.common.AbstractSyncTool used on opensource projects

Example 1 with AbstractSyncTool

use of org.apache.hudi.sync.common.AbstractSyncTool in project hudi by apache.

the class DeltaSync method syncMeta.

private void syncMeta(HoodieDeltaStreamerMetrics metrics) {
    Set<String> syncClientToolClasses = new HashSet<>(Arrays.asList(cfg.syncClientToolClass.split(",")));
    // for backward compatibility
    if (cfg.enableHiveSync) {
        cfg.enableMetaSync = true;
        syncClientToolClasses.add(HiveSyncTool.class.getName());
        LOG.info("When set --enable-hive-sync will use HiveSyncTool for backward compatibility");
    }
    if (cfg.enableMetaSync) {
        for (String impl : syncClientToolClasses) {
            Timer.Context syncContext = metrics.getMetaSyncTimerContext();
            impl = impl.trim();
            switch(impl) {
                case "org.apache.hudi.hive.HiveSyncTool":
                    syncHive();
                    break;
                default:
                    FileSystem fs = FSUtils.getFs(cfg.targetBasePath, jssc.hadoopConfiguration());
                    Properties properties = new Properties();
                    properties.putAll(props);
                    properties.put("basePath", cfg.targetBasePath);
                    properties.put("baseFileFormat", cfg.baseFileFormat);
                    AbstractSyncTool syncTool = (AbstractSyncTool) ReflectionUtils.loadClass(impl, new Class[] { Properties.class, FileSystem.class }, properties, fs);
                    syncTool.syncHoodieTable();
            }
            long metaSyncTimeMs = syncContext != null ? syncContext.stop() : 0;
            metrics.updateDeltaStreamerMetaSyncMetrics(getSyncClassShortName(impl), metaSyncTimeMs);
        }
    }
}

Example 2 with AbstractSyncTool

use of org.apache.hudi.sync.common.AbstractSyncTool in project hudi by apache.

the class KafkaConnectTransactionServices method syncMeta.

private void syncMeta() {
    if (connectConfigs.isMetaSyncEnabled()) {
        Set<String> syncClientToolClasses = new HashSet<>(Arrays.asList(connectConfigs.getMetaSyncClasses().split(",")));
        for (String impl : syncClientToolClasses) {
            impl = impl.trim();
            switch(impl) {
                case "org.apache.hudi.hive.HiveSyncTool":
                    syncHive();
                    break;
                default:
                    FileSystem fs = FSUtils.getFs(tableBasePath, new Configuration());
                    Properties properties = new Properties();
                    properties.putAll(connectConfigs.getProps());
                    properties.put("basePath", tableBasePath);
                    AbstractSyncTool syncTool = (AbstractSyncTool) ReflectionUtils.loadClass(impl, new Class[] { Properties.class, FileSystem.class }, properties, fs);
                    syncTool.syncHoodieTable();
            }
        }
    }
}