Examples with StageableTableMetadata org.apache.gobblin.data.management.conversion.hive.entities.StageableTableMetadata used on opensource projects

Example 1 with StageableTableMetadata

use of org.apache.gobblin.data.management.conversion.hive.entities.StageableTableMetadata in project incubator-gobblin by apache.

the class HiveMaterializerSource method getWorkunits.

@Override
public List<WorkUnit> getWorkunits(SourceState state) {
    try {
        FileSystem fs = HadoopUtils.getSourceFileSystem(state);
        Config config = ConfigUtils.propertiesToConfig(state.getProperties());
        if (state.contains(COPY_TABLE_KEY)) {
            HiveDataset dataset = getHiveDataset(state.getProp(COPY_TABLE_KEY), fs, state);
            WorkUnit workUnit = HiveMaterializer.tableCopyWorkUnit(dataset, new StageableTableMetadata(config.getConfig(HIVE_MATERIALIZER_SOURCE_PREFIX), dataset.getTable()), null);
            HiveTask.disableHiveWatermarker(workUnit);
            return Lists.newArrayList(workUnit);
        } else if (state.contains(MATERIALIZE_VIEW)) {
            HiveDataset dataset = getHiveDataset(state.getProp(MATERIALIZE_VIEW), fs, state);
            WorkUnit workUnit = HiveMaterializer.viewMaterializationWorkUnit(dataset, getOutputStorageFormat(state), new StageableTableMetadata(config.getConfig(HIVE_MATERIALIZER_SOURCE_PREFIX), dataset.getTable()), null);
            HiveTask.disableHiveWatermarker(workUnit);
            return Lists.newArrayList(workUnit);
        } else if (state.contains(MATERIALIZE_QUERY)) {
            String query = state.getProp(MATERIALIZE_QUERY);
            WorkUnit workUnit = HiveMaterializer.queryResultMaterializationWorkUnit(query, getOutputStorageFormat(state), new StageableTableMetadata(config.getConfig(HIVE_MATERIALIZER_SOURCE_PREFIX), null));
            HiveTask.disableHiveWatermarker(workUnit);
            return Lists.newArrayList(workUnit);
        }
    } catch (IOException ioe) {
        throw new RuntimeException(ioe);
    }
    throw new RuntimeException(String.format("Must specify either %s, %s, or %s.", COPY_TABLE_KEY, MATERIALIZE_QUERY, MATERIALIZE_VIEW));
}