Examples with InputDataTransformer org.apache.drill.metastore.iceberg.transform.InputDataTransformer used on opensource projects

Example 1 with InputDataTransformer

use of org.apache.drill.metastore.iceberg.transform.InputDataTransformer in project drill by apache.

the class TestTablesInputDataTransformer method testValidDataSeveralRecords.

@Test
public void testValidDataSeveralRecords() {
    List<TableMetadataUnit> units = Arrays.asList(TableMetadataUnit.builder().storagePlugin("dfs").workspace("tmp").tableName("nation").metadataKey(MetadataInfo.GENERAL_INFO_KEY).column("a").build(), TableMetadataUnit.builder().storagePlugin("dfs").workspace("tmp").tableName("nation").metadataKey(MetadataInfo.GENERAL_INFO_KEY).column("b").build(), TableMetadataUnit.builder().storagePlugin("dfs").workspace("tmp").tableName("nation").metadataKey(MetadataInfo.GENERAL_INFO_KEY).column("c").build());
    WriteData writeData = new InputDataTransformer<TableMetadataUnit>(metastoreSchema, partitionSchema, unitGetters).units(units).execute();
    Record tableRecord1 = GenericRecord.create(metastoreSchema);
    tableRecord1.setField("storagePlugin", "dfs");
    tableRecord1.setField("workspace", "tmp");
    tableRecord1.setField("tableName", "nation");
    tableRecord1.setField("metadataKey", MetadataInfo.GENERAL_INFO_KEY);
    tableRecord1.setField("column", "a");
    Record tableRecord2 = GenericRecord.create(metastoreSchema);
    tableRecord2.setField("storagePlugin", "dfs");
    tableRecord2.setField("workspace", "tmp");
    tableRecord2.setField("tableName", "nation");
    tableRecord2.setField("metadataKey", MetadataInfo.GENERAL_INFO_KEY);
    tableRecord2.setField("column", "b");
    Record tableRecord3 = GenericRecord.create(metastoreSchema);
    tableRecord3.setField("storagePlugin", "dfs");
    tableRecord3.setField("workspace", "tmp");
    tableRecord3.setField("tableName", "nation");
    tableRecord3.setField("metadataKey", MetadataInfo.GENERAL_INFO_KEY);
    tableRecord3.setField("column", "c");
    Record partitionRecord = GenericRecord.create(partitionSchema);
    partitionRecord.setField("storagePlugin", "dfs");
    partitionRecord.setField("workspace", "tmp");
    partitionRecord.setField("tableName", "nation");
    partitionRecord.setField("metadataKey", MetadataInfo.GENERAL_INFO_KEY);
    assertEquals(Arrays.asList(tableRecord1, tableRecord2, tableRecord3), writeData.records());
    assertEquals(partitionRecord, writeData.partition());
}

Example 2 with InputDataTransformer

use of org.apache.drill.metastore.iceberg.transform.InputDataTransformer in project drill by apache.

the class TestTablesInputDataTransformer method testNoData.

@Test
public void testNoData() {
    WriteData writeData = new InputDataTransformer<TableMetadataUnit>(metastoreSchema, partitionSchema, unitGetters).units(Collections.emptyList()).execute();
    assertEquals(Collections.emptyList(), writeData.records());
    assertNull(writeData.partition());
}

Example 3 with InputDataTransformer

use of org.apache.drill.metastore.iceberg.transform.InputDataTransformer in project drill by apache.

the class TestTablesInputDataTransformer method testValidDataOneRecord.

@Test
public void testValidDataOneRecord() {
    Map<String, String> partitionKeys = new HashMap<>();
    partitionKeys.put("dir0", "2018");
    partitionKeys.put("dir1", "2019");
    List<String> partitionValues = Arrays.asList("a", "b", "c");
    Long lastModifiedTime = System.currentTimeMillis();
    TableMetadataUnit unit = TableMetadataUnit.builder().storagePlugin("dfs").workspace("tmp").tableName("nation").metadataKey(MetadataInfo.GENERAL_INFO_KEY).partitionKeys(partitionKeys).partitionValues(partitionValues).lastModifiedTime(lastModifiedTime).build();
    WriteData writeData = new InputDataTransformer<TableMetadataUnit>(metastoreSchema, partitionSchema, unitGetters).units(Collections.singletonList(unit)).execute();
    Record tableRecord = GenericRecord.create(metastoreSchema);
    tableRecord.setField("storagePlugin", "dfs");
    tableRecord.setField("workspace", "tmp");
    tableRecord.setField("tableName", "nation");
    tableRecord.setField("metadataKey", MetadataInfo.GENERAL_INFO_KEY);
    tableRecord.setField("partitionKeys", partitionKeys);
    tableRecord.setField("partitionValues", partitionValues);
    tableRecord.setField("lastModifiedTime", lastModifiedTime);
    Record partitionRecord = GenericRecord.create(partitionSchema);
    partitionRecord.setField("storagePlugin", "dfs");
    partitionRecord.setField("workspace", "tmp");
    partitionRecord.setField("tableName", "nation");
    partitionRecord.setField("metadataKey", MetadataInfo.GENERAL_INFO_KEY);
    assertEquals(Collections.singletonList(tableRecord), writeData.records());
    assertEquals(partitionRecord, writeData.partition());
}

Example 4 with InputDataTransformer

use of org.apache.drill.metastore.iceberg.transform.InputDataTransformer in project drill by apache.

the class TestTablesInputDataTransformer method testInvalidPartition.

@Test
public void testInvalidPartition() {
    TableMetadataUnit unit = TableMetadataUnit.builder().storagePlugin("dfs").workspace("tmp").tableName("nation").build();
    thrown.expect(IcebergMetastoreException.class);
    new InputDataTransformer<TableMetadataUnit>(metastoreSchema, partitionSchema, unitGetters).units(Collections.singletonList(unit)).execute();
}