Examples with TableMetadataUnit org.apache.drill.metastore.components.tables.TableMetadataUnit used on opensource projects

Example 6 with TableMetadataUnit

use of org.apache.drill.metastore.components.tables.TableMetadataUnit in project drill by apache.

the class MetadataControllerBatch method getDefaultSegment.

private TableMetadataUnit getDefaultSegment(List<TableMetadataUnit> metadataUnits) {
    TableMetadataUnit tableMetadataUnit = metadataUnits.stream().filter(metadataUnit -> metadataUnit.metadataType().equals(MetadataType.TABLE.name())).findAny().orElseThrow(() -> new IllegalStateException("Table metadata wasn't found among collected metadata."));
    List<String> paths = metadataUnits.stream().filter(metadataUnit -> metadataUnit.metadataType().equals(MetadataType.FILE.name())).map(TableMetadataUnit::path).collect(Collectors.toList());
    return tableMetadataUnit.toBuilder().metadataType(MetadataType.SEGMENT.name()).metadataKey(MetadataInfo.DEFAULT_SEGMENT_KEY).metadataIdentifier(MetadataInfo.DEFAULT_SEGMENT_KEY).owner(null).tableType(null).metadataStatistics(Collections.emptyList()).columnsStatistics(Collections.emptyMap()).path(tableMetadataUnit.location()).schema(null).locations(paths).build();
}

Example 7 with TableMetadataUnit

use of org.apache.drill.metastore.components.tables.TableMetadataUnit in project drill by apache.

the class MetadataControllerBatch method writeToMetastore.

private void writeToMetastore() {
    MetadataControllerContext mdContext = popConfig.getContext();
    FilterExpression deleteFilter = mdContext.tableInfo().toFilter();
    for (MetadataInfo metadataInfo : mdContext.metadataToRemove()) {
        deleteFilter = FilterExpression.and(deleteFilter, FilterExpression.equal(MetastoreColumn.METADATA_KEY, metadataInfo.key()));
    }
    Modify<TableMetadataUnit> modify = tables.modify();
    if (!popConfig.getContext().metadataToRemove().isEmpty()) {
        modify.delete(Delete.builder().metadataType(MetadataType.SEGMENT, MetadataType.FILE, MetadataType.ROW_GROUP, MetadataType.PARTITION).filter(deleteFilter).build());
    }
    MetastoreTableInfo metastoreTableInfo = mdContext.metastoreTableInfo();
    if (tables.basicRequests().hasMetastoreTableInfoChanged(metastoreTableInfo)) {
        throw UserException.executionError(null).message("Metadata for table [%s] was changed before analyze is finished", tableInfo.name()).build(logger);
    }
    modify.overwrite(metadataUnits).execute();
}

Example 8 with TableMetadataUnit

use of org.apache.drill.metastore.components.tables.TableMetadataUnit in project drill by apache.

the class TestTablesOutputDataTransformer method testValidDataOneRecord.

@Test
public void testValidDataOneRecord() {
    Map<String, String> partitionKeys = new HashMap<>();
    partitionKeys.put("dir0", "2018");
    partitionKeys.put("dir1", "2019");
    List<String> partitionValues = Arrays.asList("a", "b", "c");
    Long lastModifiedTime = System.currentTimeMillis();
    Document record = new Document();
    record.append("storagePlugin", "dfs");
    record.append("workspace", "tmp");
    record.append("tableName", "nation");
    record.append("partitionKeys", partitionKeys);
    record.append("partitionValues", partitionValues);
    record.append("lastModifiedTime", lastModifiedTime);
    List<TableMetadataUnit> actualResult = new TablesOutputDataTransformer(unitSetters).columns(Arrays.asList("storagePlugin", "workspace", "tableName", "partitionKeys", "partitionValues", "lastModifiedTime")).documents(Collections.singletonList(record)).execute();
    List<TableMetadataUnit> expectedResult = Collections.singletonList(TableMetadataUnit.builder().storagePlugin("dfs").workspace("tmp").tableName("nation").partitionKeys(partitionKeys).partitionValues(partitionValues).lastModifiedTime(lastModifiedTime).build());
    assertEquals(expectedResult, actualResult);
}

Example 9 with TableMetadataUnit

use of org.apache.drill.metastore.components.tables.TableMetadataUnit in project drill by apache.

the class TestTablesMetadataMapper method testToDeleteConditionsFiles.

@Test
public void testToDeleteConditionsFiles() {
    TableMetadataUnit basicUnit = TestData.basicTableMetadataUnit();
    List<TableMetadataUnit> units = Arrays.asList(basicUnit.toBuilder().storagePlugin("dfs").workspace("tmp").tableName("nation").metadataKey("2008").metadataIdentifier("2008/0_0_0.parquet").build(), basicUnit.toBuilder().storagePlugin("dfs").workspace("tmp").tableName("nation").metadataKey("2008").metadataIdentifier("2008/0_0_1.parquet").build(), basicUnit.toBuilder().storagePlugin("dfs").workspace("tmp").tableName("nation").metadataKey("2009").metadataIdentifier("2009/0_0_0.parquet").build());
    Condition[] expectedConditions = new Condition[] { DSL.and(Tables.FILES.STORAGE_PLUGIN.eq("dfs"), Tables.FILES.WORKSPACE.eq("tmp"), Tables.FILES.TABLE_NAME.eq("nation"), Tables.FILES.METADATA_KEY.eq("2008")), DSL.and(Tables.FILES.STORAGE_PLUGIN.eq("dfs"), Tables.FILES.WORKSPACE.eq("tmp"), Tables.FILES.TABLE_NAME.eq("nation"), Tables.FILES.METADATA_KEY.eq("2009")) };
    List<Condition> actualConditions = TablesMetadataMapper.FileMapper.get().toDeleteConditions(units);
    assertEquals(expectedConditions.length, actualConditions.size());
    assertThat(actualConditions, hasItems(expectedConditions));
}

Example 10 with TableMetadataUnit

use of org.apache.drill.metastore.components.tables.TableMetadataUnit in project drill by apache.

the class TestTablesMetadataMapper method testToDeleteConditionsRowGroups.

@Test
public void testToDeleteConditionsRowGroups() {
    TableMetadataUnit basicUnit = TestData.basicTableMetadataUnit();
    List<TableMetadataUnit> units = Arrays.asList(basicUnit.toBuilder().storagePlugin("dfs").workspace("tmp").tableName("nation").metadataKey("2008").metadataIdentifier("2008/0_0_0.parquet/1").build(), basicUnit.toBuilder().storagePlugin("dfs").workspace("tmp").tableName("nation").metadataKey("2008").metadataIdentifier("2008/0_0_0.parquet/2").build(), basicUnit.toBuilder().storagePlugin("dfs").workspace("tmp").tableName("nation").metadataKey("2009").metadataIdentifier("2009/0_0_0.parquet/1").build());
    Condition[] expectedConditions = new Condition[] { DSL.and(Tables.ROW_GROUPS.STORAGE_PLUGIN.eq("dfs"), Tables.ROW_GROUPS.WORKSPACE.eq("tmp"), Tables.ROW_GROUPS.TABLE_NAME.eq("nation"), Tables.ROW_GROUPS.METADATA_KEY.eq("2008")), DSL.and(Tables.ROW_GROUPS.STORAGE_PLUGIN.eq("dfs"), Tables.ROW_GROUPS.WORKSPACE.eq("tmp"), Tables.ROW_GROUPS.TABLE_NAME.eq("nation"), Tables.ROW_GROUPS.METADATA_KEY.eq("2009")) };
    List<Condition> actualConditions = TablesMetadataMapper.RowGroupMapper.get().toDeleteConditions(units);
    assertEquals(expectedConditions.length, actualConditions.size());
    assertThat(actualConditions, hasItems(expectedConditions));
}