Examples with TableMetadataUnit org.apache.drill.metastore.components.tables.TableMetadataUnit used on opensource projects

Example 1 with TableMetadataUnit

use of org.apache.drill.metastore.components.tables.TableMetadataUnit in project drill by apache.

the class TablesOperationTransformer method toOverwrite.

/**
 * Groups given list of {@link TableMetadataUnit} based on table key
 * (storage plugin, workspace and table name), each table key is grouped by metadata key.
 * Each group is converted into overwrite operation.
 *
 * @param units Metastore component units
 * @return list of overwrite operations
 */
public List<Overwrite> toOverwrite(List<TableMetadataUnit> units) {
    Map<TableKey, Map<String, List<TableMetadataUnit>>> data = units.stream().collect(Collectors.groupingBy(TableKey::of, Collectors.groupingBy(TableMetadataUnit::metadataKey)));
    return data.entrySet().parallelStream().map(dataEntry -> dataEntry.getValue().entrySet().parallelStream().map(operationEntry -> {
        TableKey tableKey = dataEntry.getKey();
        String location = tableKey.toLocation(context.table().location());
        Map<MetastoreColumn, Object> filterConditions = new HashMap<>(tableKey.toFilterConditions());
        filterConditions.put(MetastoreColumn.METADATA_KEY, operationEntry.getKey());
        Expression expression = context.transformer().filter().transform(filterConditions);
        return toOverwrite(location, expression, operationEntry.getValue());
    }).collect(Collectors.toList())).flatMap(Collection::stream).collect(Collectors.toList());
}

Example 2 with TableMetadataUnit

use of org.apache.drill.metastore.components.tables.TableMetadataUnit in project drill by apache.

the class TestTablesInputDataTransformer method testValidDataSeveralRecords.

@Test
public void testValidDataSeveralRecords() {
    List<TableMetadataUnit> units = Arrays.asList(TableMetadataUnit.builder().storagePlugin("dfs").workspace("tmp").tableName("nation").metadataKey(MetadataInfo.GENERAL_INFO_KEY).column("a").build(), TableMetadataUnit.builder().storagePlugin("dfs").workspace("tmp").tableName("nation").metadataKey(MetadataInfo.GENERAL_INFO_KEY).column("b").build(), TableMetadataUnit.builder().storagePlugin("dfs").workspace("tmp").tableName("nation").metadataKey(MetadataInfo.GENERAL_INFO_KEY).column("c").build());
    WriteData writeData = new InputDataTransformer<TableMetadataUnit>(metastoreSchema, partitionSchema, unitGetters).units(units).execute();
    Record tableRecord1 = GenericRecord.create(metastoreSchema);
    tableRecord1.setField("storagePlugin", "dfs");
    tableRecord1.setField("workspace", "tmp");
    tableRecord1.setField("tableName", "nation");
    tableRecord1.setField("metadataKey", MetadataInfo.GENERAL_INFO_KEY);
    tableRecord1.setField("column", "a");
    Record tableRecord2 = GenericRecord.create(metastoreSchema);
    tableRecord2.setField("storagePlugin", "dfs");
    tableRecord2.setField("workspace", "tmp");
    tableRecord2.setField("tableName", "nation");
    tableRecord2.setField("metadataKey", MetadataInfo.GENERAL_INFO_KEY);
    tableRecord2.setField("column", "b");
    Record tableRecord3 = GenericRecord.create(metastoreSchema);
    tableRecord3.setField("storagePlugin", "dfs");
    tableRecord3.setField("workspace", "tmp");
    tableRecord3.setField("tableName", "nation");
    tableRecord3.setField("metadataKey", MetadataInfo.GENERAL_INFO_KEY);
    tableRecord3.setField("column", "c");
    Record partitionRecord = GenericRecord.create(partitionSchema);
    partitionRecord.setField("storagePlugin", "dfs");
    partitionRecord.setField("workspace", "tmp");
    partitionRecord.setField("tableName", "nation");
    partitionRecord.setField("metadataKey", MetadataInfo.GENERAL_INFO_KEY);
    assertEquals(Arrays.asList(tableRecord1, tableRecord2, tableRecord3), writeData.records());
    assertEquals(partitionRecord, writeData.partition());
}

Example 3 with TableMetadataUnit

use of org.apache.drill.metastore.components.tables.TableMetadataUnit in project drill by apache.

the class TestTablesOutputDataTransformer method testInvalidColumns.

@Test
public void testInvalidColumns() {
    Record record = GenericRecord.create(schema);
    record.setField("tableName", "a");
    List<TableMetadataUnit> actualResult = new TablesOutputDataTransformer(unitSetters).records(Collections.singletonList(record)).columns(Arrays.asList("a", "b")).execute();
    List<TableMetadataUnit> expectedResult = Collections.singletonList(TableMetadataUnit.builder().build());
    assertEquals(expectedResult, actualResult);
}

Example 4 with TableMetadataUnit

use of org.apache.drill.metastore.components.tables.TableMetadataUnit in project drill by apache.

the class TestTableKey method testCreation.

@Test
public void testCreation() {
    TableMetadataUnit unit = TableMetadataUnit.builder().storagePlugin("dfs").workspace("tmp").tableName("nation").build();
    TableKey expected = new TableKey("dfs", "tmp", "nation");
    assertEquals(expected, TableKey.of(unit));
}

Example 5 with TableMetadataUnit

use of org.apache.drill.metastore.components.tables.TableMetadataUnit in project drill by apache.

the class TestTablesOperationTransformer method testToOverwriteOperation.

@Test
public void testToOverwriteOperation() {
    TableMetadataUnit unit = TableMetadataUnit.builder().storagePlugin("dfs").workspace("tmp").tableName("nation").metadataKey("dir0").build();
    TableKey tableKey = new TableKey(unit.storagePlugin(), unit.workspace(), unit.tableName());
    Map<MetastoreColumn, Object> filterConditions = new HashMap<>(tableKey.toFilterConditions());
    filterConditions.put(MetastoreColumn.METADATA_KEY, unit.metadataKey());
    String location = tableKey.toLocation(TestTablesOperationTransformer.location);
    Expression expression = new FilterTransformer().transform(filterConditions);
    Overwrite operation = transformer.toOverwrite(location, expression, Collections.singletonList(unit));
    assertEquals(expression.toString(), operation.filter().toString());
    Path path = new Path(String.valueOf(operation.dataFile().path()));
    File file = new File(path.toUri().getPath());
    assertTrue(file.exists());
    assertEquals(location, path.getParent().toUri().getPath());
}