Examples with MetadataInfo org.apache.drill.metastore.metadata.MetadataInfo used on opensource projects

Example 6 with MetadataInfo

use of org.apache.drill.metastore.metadata.MetadataInfo in project drill by apache.

the class MetadataControllerBatch method getPartitionMetadata.

private PartitionMetadata getPartitionMetadata(TupleReader reader, List<StatisticsHolder<?>> metadataStatistics, Map<SchemaPath, ColumnStatistics<?>> columnStatistics, int nestingLevel) {
    List<String> segmentColumns = popConfig.getContext().segmentColumns();
    String segmentKey = segmentColumns.size() > 0 ? reader.column(segmentColumns.iterator().next()).scalar().getString() : MetadataInfo.DEFAULT_SEGMENT_KEY;
    List<String> partitionValues = segmentColumns.stream().limit(nestingLevel).map(columnName -> reader.column(columnName).scalar().getString()).collect(Collectors.toList());
    String metadataIdentifier = MetadataIdentifierUtils.getMetadataIdentifierKey(partitionValues);
    MetadataInfo metadataInfo = MetadataInfo.builder().type(MetadataType.PARTITION).key(segmentKey).identifier(StringUtils.defaultIfEmpty(metadataIdentifier, null)).build();
    return PartitionMetadata.builder().tableInfo(tableInfo).metadataInfo(metadataInfo).columnsStatistics(columnStatistics).metadataStatistics(metadataStatistics).locations(getIncomingLocations(reader)).lastModifiedTime(Long.parseLong(reader.column(columnNamesOptions.lastModifiedTime()).scalar().getString())).schema(TupleMetadata.of(reader.column(MetastoreAnalyzeConstants.SCHEMA_FIELD).scalar().getString())).build();
}

Example 7 with MetadataInfo

use of org.apache.drill.metastore.metadata.MetadataInfo in project drill by apache.

the class MetadataControllerBatch method writeToMetastore.

private void writeToMetastore() {
    MetadataControllerContext mdContext = popConfig.getContext();
    FilterExpression deleteFilter = mdContext.tableInfo().toFilter();
    for (MetadataInfo metadataInfo : mdContext.metadataToRemove()) {
        deleteFilter = FilterExpression.and(deleteFilter, FilterExpression.equal(MetastoreColumn.METADATA_KEY, metadataInfo.key()));
    }
    Modify<TableMetadataUnit> modify = tables.modify();
    if (!popConfig.getContext().metadataToRemove().isEmpty()) {
        modify.delete(Delete.builder().metadataType(MetadataType.SEGMENT, MetadataType.FILE, MetadataType.ROW_GROUP, MetadataType.PARTITION).filter(deleteFilter).build());
    }
    MetastoreTableInfo metastoreTableInfo = mdContext.metastoreTableInfo();
    if (tables.basicRequests().hasMetastoreTableInfoChanged(metastoreTableInfo)) {
        throw UserException.executionError(null).message("Metadata for table [%s] was changed before analyze is finished", tableInfo.name()).build(logger);
    }
    modify.overwrite(metadataUnits).execute();
}

Example 8 with MetadataInfo

use of org.apache.drill.metastore.metadata.MetadataInfo in project drill by apache.

the class MetadataControllerBatch method getFileMetadata.

private FileMetadata getFileMetadata(TupleReader reader, List<StatisticsHolder<?>> metadataStatistics, Map<SchemaPath, ColumnStatistics<?>> columnStatistics, int nestingLevel) {
    List<String> segmentColumns = popConfig.getContext().segmentColumns();
    String segmentKey = segmentColumns.size() > 0 ? reader.column(segmentColumns.iterator().next()).scalar().getString() : MetadataInfo.DEFAULT_SEGMENT_KEY;
    List<String> partitionValues = segmentColumns.stream().limit(nestingLevel - 1).map(columnName -> reader.column(columnName).scalar().getString()).collect(Collectors.toList());
    Path path = new Path(reader.column(MetastoreAnalyzeConstants.LOCATION_FIELD).scalar().getString());
    String metadataIdentifier = MetadataIdentifierUtils.getFileMetadataIdentifier(partitionValues, path);
    MetadataInfo metadataInfo = MetadataInfo.builder().type(MetadataType.FILE).key(segmentKey).identifier(StringUtils.defaultIfEmpty(metadataIdentifier, null)).build();
    return FileMetadata.builder().tableInfo(tableInfo).metadataInfo(metadataInfo).columnsStatistics(columnStatistics).metadataStatistics(metadataStatistics).path(path).lastModifiedTime(Long.parseLong(reader.column(columnNamesOptions.lastModifiedTime()).scalar().getString())).schema(TupleMetadata.of(reader.column(MetastoreAnalyzeConstants.SCHEMA_FIELD).scalar().getString())).build();
}

Example 9 with MetadataInfo

use of org.apache.drill.metastore.metadata.MetadataInfo in project drill by apache.

the class TestTableMetadataUnitConversion method testBaseTableMetadata.

@Test
public void testBaseTableMetadata() {
    TableInfo tableInfo = data.fullTableInfo;
    MetadataInfo metadataInfo = MetadataInfo.builder().type(MetadataType.TABLE).key(MetadataInfo.GENERAL_INFO_KEY).build();
    Map<String, String> partitionKeys = new HashMap<>();
    partitionKeys.put("dir0", "2018");
    partitionKeys.put("dir1", "2019");
    // check required fields
    BaseTableMetadata requiredFieldsMetadata = BaseTableMetadata.builder().tableInfo(tableInfo).metadataInfo(metadataInfo).columnsStatistics(data.columnsStatistics).metadataStatistics(data.metadataStatistics).partitionKeys(partitionKeys).build();
    TableMetadataUnit requiredFieldsExpectedUnit = TableMetadataUnit.builder().storagePlugin(tableInfo.storagePlugin()).workspace(tableInfo.workspace()).tableName(tableInfo.name()).owner(tableInfo.owner()).tableType(tableInfo.type()).metadataType(metadataInfo.type().name()).metadataKey(metadataInfo.key()).metadataIdentifier(metadataInfo.identifier()).columnsStatistics(data.unitColumnsStatistics).metadataStatistics(data.unitMetadataStatistics).lastModifiedTime(BaseMetadata.UNDEFINED_TIME).partitionKeys(partitionKeys).build();
    TableMetadataUnit requiredFieldsUnit = requiredFieldsMetadata.toMetadataUnit();
    assertEquals(requiredFieldsExpectedUnit, requiredFieldsUnit);
    assertNotNull(BaseTableMetadata.builder().metadataUnit(requiredFieldsUnit).build());
    Path location = new Path("/tmp/nation");
    List<SchemaPath> interestingColumns = Arrays.asList(SchemaPath.getSimplePath("a"), SchemaPath.getSimplePath("b"));
    // check all fields
    BaseTableMetadata allFieldsMetadata = BaseTableMetadata.builder().tableInfo(tableInfo).metadataInfo(metadataInfo).schema(data.schema).columnsStatistics(data.columnsStatistics).metadataStatistics(data.metadataStatistics).lastModifiedTime(data.lastModifiedTime).location(location).partitionKeys(partitionKeys).interestingColumns(interestingColumns).build();
    TableMetadataUnit allFieldsExpectedUnit = TableMetadataUnit.builder().storagePlugin(tableInfo.storagePlugin()).workspace(tableInfo.workspace()).tableName(tableInfo.name()).owner(tableInfo.owner()).tableType(tableInfo.type()).metadataType(metadataInfo.type().name()).metadataKey(metadataInfo.key()).metadataIdentifier(metadataInfo.identifier()).schema(data.unitSchema).columnsStatistics(data.unitColumnsStatistics).metadataStatistics(data.unitMetadataStatistics).lastModifiedTime(data.lastModifiedTime).location(location.toUri().getPath()).partitionKeys(partitionKeys).interestingColumns(interestingColumns.stream().map(SchemaPath::toString).collect(Collectors.toList())).build();
    TableMetadataUnit allFieldsUnit = allFieldsMetadata.toMetadataUnit();
    assertEquals(allFieldsExpectedUnit, allFieldsUnit);
    assertNotNull(BaseTableMetadata.builder().metadataUnit(allFieldsUnit).build());
}

Example 10 with MetadataInfo

use of org.apache.drill.metastore.metadata.MetadataInfo in project drill by apache.

the class TestTableMetadataUnitConversion method testRowGroupMetadata.

@Test
public void testRowGroupMetadata() {
    TableInfo tableInfo = data.basicTableInfo;
    MetadataInfo metadataInfo = MetadataInfo.builder().type(MetadataType.ROW_GROUP).key("part_int=3").identifier("part_int=3/part_varchar=g/0_0_0.parquet").build();
    Path path = new Path("/tmp/nation/part_int=3/part_varchar=g/0_0_0.parquet");
    int rowGroupIndex = 1;
    Map<String, Float> hostAffinity = new HashMap<>();
    hostAffinity.put("host1", 1F);
    hostAffinity.put("host2", 2F);
    RowGroupMetadata metadata = RowGroupMetadata.builder().tableInfo(tableInfo).metadataInfo(metadataInfo).schema(data.schema).columnsStatistics(data.columnsStatistics).metadataStatistics(data.metadataStatistics).lastModifiedTime(data.lastModifiedTime).path(path).rowGroupIndex(rowGroupIndex).hostAffinity(hostAffinity).build();
    TableMetadataUnit expectedUnit = TableMetadataUnit.builder().storagePlugin(tableInfo.storagePlugin()).workspace(tableInfo.workspace()).tableName(tableInfo.name()).metadataType(metadataInfo.type().name()).metadataKey(metadataInfo.key()).metadataIdentifier(metadataInfo.identifier()).schema(data.unitSchema).columnsStatistics(data.unitColumnsStatistics).metadataStatistics(data.unitMetadataStatistics).lastModifiedTime(data.lastModifiedTime).path(path.toUri().getPath()).location(path.getParent().toUri().getPath()).rowGroupIndex(rowGroupIndex).hostAffinity(hostAffinity).build();
    TableMetadataUnit actualUnit = metadata.toMetadataUnit();
    assertEquals(expectedUnit, actualUnit);
    assertNotNull(RowGroupMetadata.builder().metadataUnit(actualUnit).build());
}