Examples with HiveSourceWatermarker org.apache.gobblin.data.management.conversion.hive.watermarker.HiveSourceWatermarker used on opensource projects

Example 1 with HiveSourceWatermarker

use of org.apache.gobblin.data.management.conversion.hive.watermarker.HiveSourceWatermarker in project incubator-gobblin by apache.

the class HiveTask method executePublishQueries.

protected void executePublishQueries(QueryBasedHivePublishEntity publishEntity) {
    Set<String> cleanUpQueries = Sets.newLinkedHashSet();
    Set<String> publishQueries = Sets.newLinkedHashSet();
    List<String> directoriesToDelete = Lists.newArrayList();
    FileSystem fs = null;
    try {
        fs = HiveSource.getSourceFs(workUnitState);
        if (publishEntity.getCleanupQueries() != null) {
            cleanUpQueries.addAll(publishEntity.getCleanupQueries());
        }
        if (publishEntity.getCleanupDirectories() != null) {
            directoriesToDelete.addAll(publishEntity.getCleanupDirectories());
        }
        if (publishEntity.getPublishDirectories() != null) {
            // Publish snapshot / partition directories
            Map<String, String> publishDirectories = publishEntity.getPublishDirectories();
            try {
                for (Map.Entry<String, String> publishDir : publishDirectories.entrySet()) {
                    HadoopUtils.renamePath(fs, new Path(publishDir.getKey()), new Path(publishDir.getValue()), true);
                }
            } catch (Throwable t) {
                throw Throwables.propagate(t);
            }
        }
        if (publishEntity.getPublishQueries() != null) {
            publishQueries.addAll(publishEntity.getPublishQueries());
        }
        WorkUnitState wus = this.workUnitState;
        this.hiveJdbcConnector.executeStatements(publishQueries.toArray(new String[publishQueries.size()]));
        wus.setWorkingState(WorkUnitState.WorkingState.COMMITTED);
        if (wus.getPropAsBoolean(USE_WATERMARKER_KEY, true)) {
            HiveSourceWatermarker watermarker = GobblinConstructorUtils.invokeConstructor(HiveSourceWatermarkerFactory.class, wus.getProp(HiveSource.HIVE_SOURCE_WATERMARKER_FACTORY_CLASS_KEY, HiveSource.DEFAULT_HIVE_SOURCE_WATERMARKER_FACTORY_CLASS)).createFromState(wus);
            watermarker.setActualHighWatermark(wus);
        }
    } catch (RuntimeException re) {
        throw re;
    } catch (Exception e) {
        log.error("Error in HiveMaterializer generate publish queries", e);
    } finally {
        try {
            this.hiveJdbcConnector.executeStatements(cleanUpQueries.toArray(new String[cleanUpQueries.size()]));
            HadoopUtils.deleteDirectories(fs, directoriesToDelete, true, true);
        } catch (RuntimeException re) {
            throw re;
        } catch (Exception e) {
            log.error("Failed to cleanup staging entities.", e);
        }
    }
}