Examples with SchemaAwareHivePartition org.apache.gobblin.data.management.conversion.hive.entities.SchemaAwareHivePartition used on opensource projects

Example 1 with SchemaAwareHivePartition

use of org.apache.gobblin.data.management.conversion.hive.entities.SchemaAwareHivePartition in project incubator-gobblin by apache.

the class HiveAvroToOrcConverterTest method dropReplacedPartitionsTest.

@Test
public void dropReplacedPartitionsTest() throws Exception {
    Table table = ConvertibleHiveDatasetTest.getTestTable("dbName", "tableName");
    table.setTableType("VIRTUAL_VIEW");
    table.setPartitionKeys(ImmutableList.of(new FieldSchema("year", "string", ""), new FieldSchema("month", "string", "")));
    Partition part = new Partition();
    part.setParameters(ImmutableMap.of("gobblin.replaced.partitions", "2015,12|2016,01"));
    SchemaAwareHiveTable hiveTable = new SchemaAwareHiveTable(table, null);
    SchemaAwareHivePartition partition = new SchemaAwareHivePartition(table, part, null);
    QueryBasedHiveConversionEntity conversionEntity = new QueryBasedHiveConversionEntity(null, hiveTable, Optional.of(partition));
    List<ImmutableMap<String, String>> expected = ImmutableList.of(ImmutableMap.of("year", "2015", "month", "12"), ImmutableMap.of("year", "2016", "month", "01"));
    Assert.assertEquals(AbstractAvroToOrcConverter.getDropPartitionsDDLInfo(conversionEntity), expected);
    // Make sure that a partition itself is not dropped
    Partition replacedSelf = new Partition();
    replacedSelf.setParameters(ImmutableMap.of("gobblin.replaced.partitions", "2015,12|2016,01|2016,02"));
    replacedSelf.setValues(ImmutableList.of("2016", "02"));
    conversionEntity = new QueryBasedHiveConversionEntity(null, hiveTable, Optional.of(new SchemaAwareHivePartition(table, replacedSelf, null)));
    Assert.assertEquals(AbstractAvroToOrcConverter.getDropPartitionsDDLInfo(conversionEntity), expected);
}

Example 2 with SchemaAwareHivePartition

use of org.apache.gobblin.data.management.conversion.hive.entities.SchemaAwareHivePartition in project incubator-gobblin by apache.

the class HiveQueryExecutionWriter method addPropsForPublisher.

/**
 * Method to add properties needed by publisher to preserve partition params
 */
private void addPropsForPublisher(QueryBasedHiveConversionEntity hiveConversionEntity) {
    if (!hiveConversionEntity.getPartition().isPresent()) {
        return;
    }
    ConvertibleHiveDataset convertibleHiveDataset = hiveConversionEntity.getConvertibleHiveDataset();
    for (String format : convertibleHiveDataset.getDestFormats()) {
        Optional<ConvertibleHiveDataset.ConversionConfig> conversionConfigForFormat = convertibleHiveDataset.getConversionConfigForFormat(format);
        if (!conversionConfigForFormat.isPresent()) {
            continue;
        }
        SchemaAwareHivePartition sourcePartition = hiveConversionEntity.getHivePartition().get();
        // Get complete source partition name dbName@tableName@partitionName
        String completeSourcePartitionName = StringUtils.join(Arrays.asList(sourcePartition.getTable().getDbName(), sourcePartition.getTable().getTableName(), sourcePartition.getName()), AT_CHAR);
        ConvertibleHiveDataset.ConversionConfig config = conversionConfigForFormat.get();
        // Get complete destination partition name dbName@tableName@partitionName
        String completeDestPartitionName = StringUtils.join(Arrays.asList(config.getDestinationDbName(), config.getDestinationTableName(), sourcePartition.getName()), AT_CHAR);
        workUnit.setProp(HiveConvertPublisher.COMPLETE_SOURCE_PARTITION_NAME, completeSourcePartitionName);
        workUnit.setProp(HiveConvertPublisher.COMPLETE_DEST_PARTITION_NAME, completeDestPartitionName);
    }
}