Examples with BaseTableMetadata org.apache.drill.metastore.metadata.BaseTableMetadata used on opensource projects

Example 21 with BaseTableMetadata

use of org.apache.drill.metastore.metadata.BaseTableMetadata in project drill by apache.

the class TestInfoSchemaWithMetastore method testTableWithStats.

@Test
public void testTableWithStats() throws Exception {
    ZonedDateTime currentTime = currentUtcTime();
    String tableName = "table_with_stats";
    BaseTableMetadata table = BaseTableMetadata.builder().tableInfo(TableInfo.builder().storagePlugin("dfs").workspace("tmp").name(tableName).type("PARQUET").build()).metadataInfo(MetadataInfo.builder().type(MetadataType.TABLE).key(MetadataInfo.GENERAL_INFO_KEY).build()).location(new Path("/tmp", tableName)).metadataStatistics(Collections.singletonList(new StatisticsHolder<>(100L, TableStatisticsKind.ROW_COUNT))).columnsStatistics(Collections.emptyMap()).partitionKeys(Collections.emptyMap()).lastModifiedTime(currentTime.toInstant().toEpochMilli()).build();
    metastore.tables().modify().overwrite(table.toMetadataUnit()).execute();
    client.testBuilder().sqlQuery("select %s from information_schema.`tables` where table_name = '%s'", String.join(", ", TABLES_COLUMNS), tableName).unOrdered().baselineColumns(TABLES_COLUMNS.toArray(new String[0])).baselineValues("DRILL", "dfs.tmp", tableName, "TABLE", table.getTableInfo().type(), table.getLocation().toUri().toString(), 100L, currentTime.toLocalDateTime()).go();
}

Example 22 with BaseTableMetadata

use of org.apache.drill.metastore.metadata.BaseTableMetadata in project drill by apache.

the class TestInfoSchemaWithMetastore method testColumns.

@Test
public void testColumns() throws Exception {
    BaseTableMetadata tableNoSchema = BaseTableMetadata.builder().tableInfo(TableInfo.builder().storagePlugin("dfs").workspace("tmp").name("table_no_schema").type("PARQUET").build()).metadataInfo(MetadataInfo.builder().type(MetadataType.TABLE).key(MetadataInfo.GENERAL_INFO_KEY).build()).location(new Path("/tmp", "table_no_schema")).metadataStatistics(Collections.emptyList()).columnsStatistics(Collections.emptyMap()).partitionKeys(Collections.emptyMap()).build();
    TupleMetadata schema = new SchemaBuilder().addNullable("bigint_col", TypeProtos.MinorType.BIGINT).addDecimal("decimal_col", TypeProtos.MinorType.VARDECIMAL, TypeProtos.DataMode.OPTIONAL, 10, 2).add("interval_col", TypeProtos.MinorType.INTERVALYEAR).addArray("array_col", TypeProtos.MinorType.BIT).addMap("struct_col").addNullable("struct_bigint", TypeProtos.MinorType.BIGINT).add("struct_varchar", TypeProtos.MinorType.VARCHAR).addMap("nested_struct").addNullable("nested_struct_boolean", TypeProtos.MinorType.BIT).add("nested_struct_varchar", TypeProtos.MinorType.VARCHAR).resumeMap().resumeSchema().buildSchema();
    PrimitiveColumnMetadata varcharCol = new PrimitiveColumnMetadata("varchar_col", TypeProtos.MajorType.newBuilder().setMinorType(TypeProtos.MinorType.VARCHAR).setMode(TypeProtos.DataMode.REQUIRED).build());
    varcharCol.setDefaultValue("ABC");
    PrimitiveColumnMetadata timestampColumn = new PrimitiveColumnMetadata("timestamp_col", TypeProtos.MajorType.newBuilder().setMinorType(TypeProtos.MinorType.TIMESTAMP).setMode(TypeProtos.DataMode.REQUIRED).build());
    timestampColumn.setFormat("yyyy-MM-dd HH:mm:ss");
    schema.addColumn(varcharCol);
    schema.addColumn(timestampColumn);
    Map<SchemaPath, ColumnStatistics<?>> columnsStatistics = new HashMap<>();
    columnsStatistics.put(SchemaPath.parseFromString("varchar_col"), new ColumnStatistics<>(Arrays.asList(new StatisticsHolder<>("aaa", ColumnStatisticsKind.MIN_VALUE), new StatisticsHolder<>("zzz", ColumnStatisticsKind.MAX_VALUE))));
    columnsStatistics.put(SchemaPath.parseFromString("struct_col.nested_struct.nested_struct_varchar"), new ColumnStatistics<>(Arrays.asList(new StatisticsHolder<>("bbb", ColumnStatisticsKind.MIN_VALUE), new StatisticsHolder<>("ccc", ColumnStatisticsKind.MAX_VALUE))));
    columnsStatistics.put(SchemaPath.parseFromString("bigint_col"), new ColumnStatistics<>(Arrays.asList(new StatisticsHolder<>(100L, ColumnStatisticsKind.NULLS_COUNT), new StatisticsHolder<>(10.5D, ColumnStatisticsKind.NDV))));
    columnsStatistics.put(SchemaPath.parseFromString("struct_col.struct_bigint"), new ColumnStatistics<>(Collections.singletonList(new StatisticsHolder<>(10.5D, ColumnStatisticsKind.NON_NULL_COUNT))));
    ZonedDateTime currentTime = currentUtcTime();
    String tableName = "table_with_schema";
    BaseTableMetadata tableWithSchema = BaseTableMetadata.builder().tableInfo(TableInfo.builder().storagePlugin("dfs").workspace("tmp").name(tableName).type("PARQUET").build()).metadataInfo(MetadataInfo.builder().type(MetadataType.TABLE).key(MetadataInfo.GENERAL_INFO_KEY).build()).location(new Path("/tmp", tableName)).schema(schema).metadataStatistics(Collections.emptyList()).columnsStatistics(columnsStatistics).partitionKeys(Collections.emptyMap()).lastModifiedTime(currentTime.toInstant().toEpochMilli()).build();
    metastore.tables().modify().overwrite(tableNoSchema.toMetadataUnit(), tableWithSchema.toMetadataUnit()).execute();
    List<String> columns = Arrays.asList(InfoSchemaConstants.SHRD_COL_TABLE_CATALOG, InfoSchemaConstants.SHRD_COL_TABLE_SCHEMA, InfoSchemaConstants.SHRD_COL_TABLE_NAME, InfoSchemaConstants.COLS_COL_COLUMN_NAME, InfoSchemaConstants.COLS_COL_ORDINAL_POSITION, InfoSchemaConstants.COLS_COL_COLUMN_DEFAULT, InfoSchemaConstants.COLS_COL_IS_NULLABLE, InfoSchemaConstants.COLS_COL_DATA_TYPE, InfoSchemaConstants.COLS_COL_CHARACTER_MAXIMUM_LENGTH, InfoSchemaConstants.COLS_COL_CHARACTER_OCTET_LENGTH, InfoSchemaConstants.COLS_COL_NUMERIC_PRECISION, InfoSchemaConstants.COLS_COL_NUMERIC_PRECISION_RADIX, InfoSchemaConstants.COLS_COL_NUMERIC_SCALE, InfoSchemaConstants.COLS_COL_DATETIME_PRECISION, InfoSchemaConstants.COLS_COL_INTERVAL_TYPE, InfoSchemaConstants.COLS_COL_INTERVAL_PRECISION, InfoSchemaConstants.COLS_COL_COLUMN_SIZE, InfoSchemaConstants.COLS_COL_COLUMN_FORMAT, InfoSchemaConstants.COLS_COL_NUM_NULLS, InfoSchemaConstants.COLS_COL_MIN_VAL, InfoSchemaConstants.COLS_COL_MAX_VAL, InfoSchemaConstants.COLS_COL_NDV, InfoSchemaConstants.COLS_COL_EST_NUM_NON_NULLS, InfoSchemaConstants.COLS_COL_IS_NESTED);
    client.testBuilder().sqlQuery("select %s from information_schema.`columns` where table_name " + "in ('%s', '%s')", String.join(", ", columns), tableNoSchema.getTableInfo().name(), tableName).unOrdered().baselineColumns(columns.toArray(new String[0])).baselineValues("DRILL", "dfs.tmp", tableName, "bigint_col", 1, null, "YES", "BIGINT", null, null, 0, 2, 0, null, null, null, 20, null, 100L, null, null, 10.5D, null, false).baselineValues("DRILL", "dfs.tmp", tableName, "decimal_col", 2, null, "YES", "DECIMAL", null, null, 10, 10, 2, null, null, null, 12, null, null, null, null, null, null, false).baselineValues("DRILL", "dfs.tmp", tableName, "interval_col", 3, null, "NO", "INTERVAL", null, null, null, null, null, null, "INTERVAL YEAR TO MONTH", 0, 9, null, null, null, null, null, null, false).baselineValues("DRILL", "dfs.tmp", tableName, "array_col", 4, null, "NO", "ARRAY", null, null, null, null, null, null, null, null, 0, null, null, null, null, null, null, false).baselineValues("DRILL", "dfs.tmp", tableName, "struct_col", 5, null, "NO", "STRUCT", null, null, null, null, null, null, null, null, 0, null, null, null, null, null, null, false).baselineValues("DRILL", "dfs.tmp", tableName, "struct_col.struct_bigint", 5, null, "YES", "BIGINT", null, null, 0, 2, 0, null, null, null, 20, null, null, null, null, null, 10.5D, true).baselineValues("DRILL", "dfs.tmp", tableName, "struct_col.struct_varchar", 5, null, "NO", "CHARACTER VARYING", 65535, 65535, null, null, null, null, null, null, 65535, null, null, null, null, null, null, true).baselineValues("DRILL", "dfs.tmp", tableName, "struct_col.nested_struct", 5, null, "NO", "STRUCT", null, null, null, null, null, null, null, null, 0, null, null, null, null, null, null, true).baselineValues("DRILL", "dfs.tmp", tableName, "struct_col.nested_struct.nested_struct_boolean", 5, null, "YES", "BOOLEAN", null, null, null, null, null, null, null, null, 1, null, null, null, null, null, null, true).baselineValues("DRILL", "dfs.tmp", tableName, "struct_col.nested_struct.nested_struct_varchar", 5, null, "NO", "CHARACTER VARYING", 65535, 65535, null, null, null, null, null, null, 65535, null, null, "bbb", "ccc", null, null, true).baselineValues("DRILL", "dfs.tmp", tableName, "varchar_col", 6, "ABC", "NO", "CHARACTER VARYING", 65535, 65535, null, null, null, null, null, null, 65535, null, null, "aaa", "zzz", null, null, false).baselineValues("DRILL", "dfs.tmp", tableName, "timestamp_col", 7, null, "NO", "TIMESTAMP", null, null, null, null, null, 19, null, null, 19, "yyyy-MM-dd HH:mm:ss", null, null, null, null, null, false).go();
}

Example 23 with BaseTableMetadata

use of org.apache.drill.metastore.metadata.BaseTableMetadata in project drill by apache.

the class TestMetastoreCommands method testAnalyzeEmptyRequiredParquetTable.

@Test
public void testAnalyzeEmptyRequiredParquetTable() throws Exception {
    String tableName = "analyze_empty_simple_required";
    run("create table dfs.tmp.%s as select 1 as `date`, 'a' as name from (values(1)) where 1 = 2", tableName);
    File table = new File(dirTestWatcher.getDfsTestTmpDir(), tableName);
    TableInfo tableInfo = getTableInfo(tableName, "tmp");
    TupleMetadata schema = new SchemaBuilder().add("date", TypeProtos.MinorType.INT).add("name", TypeProtos.MinorType.VARCHAR).build();
    Map<SchemaPath, ColumnStatistics<?>> columnStatistics = ImmutableMap.<SchemaPath, ColumnStatistics<?>>builder().put(SchemaPath.getSimplePath("name"), getColumnStatistics(null, null, 0L, TypeProtos.MinorType.VARCHAR)).put(SchemaPath.getSimplePath("date"), getColumnStatistics(null, null, 0L, TypeProtos.MinorType.INT)).build();
    BaseTableMetadata expectedTableMetadata = BaseTableMetadata.builder().tableInfo(tableInfo).metadataInfo(TABLE_META_INFO).schema(schema).location(new Path(table.toURI().getPath())).columnsStatistics(columnStatistics).metadataStatistics(Arrays.asList(new StatisticsHolder<>(0L, TableStatisticsKind.ROW_COUNT), new StatisticsHolder<>(MetadataType.ALL, TableStatisticsKind.ANALYZE_METADATA_LEVEL))).partitionKeys(Collections.emptyMap()).lastModifiedTime(getMaxLastModified(table)).build();
    try {
        testBuilder().sqlQuery("ANALYZE TABLE dfs.tmp.`%s` REFRESH METADATA", tableName).unOrdered().baselineColumns("ok", "summary").baselineValues(true, String.format("Collected / refreshed metadata for table [dfs.tmp.%s]", tableName)).go();
        MetastoreTableInfo metastoreTableInfo = cluster.drillbit().getContext().getMetastoreRegistry().get().tables().basicRequests().metastoreTableInfo(tableInfo);
        assertTrue("table metadata wasn't found", metastoreTableInfo.isExists());
        BaseTableMetadata tableMetadata = cluster.drillbit().getContext().getMetastoreRegistry().get().tables().basicRequests().tableMetadata(tableInfo);
        assertEquals(expectedTableMetadata, tableMetadata);
        List<FileMetadata> filesMetadata = cluster.drillbit().getContext().getMetastoreRegistry().get().tables().basicRequests().filesMetadata(tableInfo, null, null);
        assertEquals(1, filesMetadata.size());
        List<RowGroupMetadata> rowGroupsMetadata = cluster.drillbit().getContext().getMetastoreRegistry().get().tables().basicRequests().rowGroupsMetadata(tableInfo, (String) null, null);
        assertEquals(1, rowGroupsMetadata.size());
        testBuilder().sqlQuery("select COLUMN_NAME from INFORMATION_SCHEMA.`COLUMNS` where table_name='%s'", tableName).unOrdered().baselineColumns("COLUMN_NAME").baselineValues("date").baselineValues("name").go();
    } finally {
        run("analyze table dfs.tmp.`%s` drop metadata if exists", tableName);
        run("drop table if exists dfs.tmp.`%s`", tableName);
    }
}

Example 24 with BaseTableMetadata

use of org.apache.drill.metastore.metadata.BaseTableMetadata in project drill by apache.

the class TestMetastoreCommands method testIncrementalAnalyzeUpdatedFile.

@Test
public void testIncrementalAnalyzeUpdatedFile() throws Exception {
    String tableName = "multilevel/parquetUpdatedFile";
    File table = dirTestWatcher.copyResourceToTestTmp(Paths.get("multilevel/parquet"), Paths.get(tableName));
    TableInfo tableInfo = getTableInfo(tableName, "tmp");
    try {
        testBuilder().sqlQuery("ANALYZE TABLE dfs.tmp.`%s` REFRESH METADATA", tableName).unOrdered().baselineColumns("ok", "summary").baselineValues(true, String.format("Collected / refreshed metadata for table [dfs.tmp.%s]", tableName)).go();
        List<SegmentMetadata> segmentMetadata = cluster.drillbit().getContext().getMetastoreRegistry().get().tables().basicRequests().segmentsMetadataByMetadataKey(tableInfo, null, null);
        assertEquals(15, segmentMetadata.size());
        List<FileMetadata> filesMetadata = cluster.drillbit().getContext().getMetastoreRegistry().get().tables().basicRequests().filesMetadata(tableInfo, null, null);
        assertEquals(12, filesMetadata.size());
        List<RowGroupMetadata> rowGroupsMetadata = cluster.drillbit().getContext().getMetastoreRegistry().get().tables().basicRequests().rowGroupsMetadata(tableInfo, null, (String) null);
        assertEquals(12, rowGroupsMetadata.size());
        File fileToUpdate = new File(new File(new File(table, "1994"), "Q4"), "orders_94_q4.parquet");
        long lastModified = fileToUpdate.lastModified();
        FileUtils.deleteQuietly(fileToUpdate);
        // replaces original file
        dirTestWatcher.copyResourceToTestTmp(Paths.get("multilevel", "parquet", "1994", "Q1", "orders_94_q1.parquet"), Paths.get(tableName, "1994", "Q4", "orders_94_q4.parquet"));
        long newLastModified = lastModified + 1000;
        assertTrue(fileToUpdate.setLastModified(newLastModified));
        testBuilder().sqlQuery("ANALYZE TABLE dfs.tmp.`%s` REFRESH METADATA", tableName).unOrdered().baselineColumns("ok", "summary").baselineValues(true, String.format("Collected / refreshed metadata for table [dfs.tmp.%s]", tableName)).go();
        BaseTableMetadata actualTableMetadata = cluster.drillbit().getContext().getMetastoreRegistry().get().tables().basicRequests().tableMetadata(tableInfo);
        Map<SchemaPath, ColumnStatistics<?>> tableColumnStatistics = new HashMap<>(TABLE_COLUMN_STATISTICS);
        tableColumnStatistics.computeIfPresent(SchemaPath.getSimplePath("o_clerk"), (logicalExpressions, columnStatistics) -> columnStatistics.cloneWith(new ColumnStatistics<>(Collections.singletonList(new StatisticsHolder<>("Clerk#000000006", ColumnStatisticsKind.MIN_VALUE)))));
        tableColumnStatistics.computeIfPresent(SchemaPath.getSimplePath("o_totalprice"), (logicalExpressions, columnStatistics) -> columnStatistics.cloneWith(new ColumnStatistics<>(Collections.singletonList(new StatisticsHolder<>(328207.15, ColumnStatisticsKind.MAX_VALUE)))));
        BaseTableMetadata expectedTableMetadata = BaseTableMetadata.builder().tableInfo(tableInfo).metadataInfo(TABLE_META_INFO).schema(SCHEMA).location(new Path(table.toURI().getPath())).columnsStatistics(tableColumnStatistics).metadataStatistics(Arrays.asList(new StatisticsHolder<>(120L, TableStatisticsKind.ROW_COUNT), new StatisticsHolder<>(MetadataType.ALL, TableStatisticsKind.ANALYZE_METADATA_LEVEL))).partitionKeys(Collections.emptyMap()).lastModifiedTime(newLastModified).build();
        assertEquals(expectedTableMetadata, actualTableMetadata);
        segmentMetadata = cluster.drillbit().getContext().getMetastoreRegistry().get().tables().basicRequests().segmentsMetadataByMetadataKey(tableInfo, null, null);
        assertEquals(15, segmentMetadata.size());
        filesMetadata = cluster.drillbit().getContext().getMetastoreRegistry().get().tables().basicRequests().filesMetadata(tableInfo, null, null);
        assertEquals(12, filesMetadata.size());
        rowGroupsMetadata = cluster.drillbit().getContext().getMetastoreRegistry().get().tables().basicRequests().rowGroupsMetadata(tableInfo, null, (String) null);
        assertEquals(12, rowGroupsMetadata.size());
    } finally {
        run("analyze table dfs.tmp.`%s` drop metadata if exists", tableName);
        FileUtils.deleteQuietly(table);
    }
}

Example 25 with BaseTableMetadata

use of org.apache.drill.metastore.metadata.BaseTableMetadata in project drill by apache.

the class TestMetastoreCommands method testAnalyzeEmptyNullableParquetTable.

@Test
public void testAnalyzeEmptyNullableParquetTable() throws Exception {
    File table = dirTestWatcher.copyResourceToRoot(Paths.get("parquet", "empty", "simple", "empty_simple.parquet"));
    String tableName = "parquet/empty/simple/empty_simple.parquet";
    TableInfo tableInfo = getTableInfo(tableName, "default");
    TupleMetadata schema = new SchemaBuilder().addNullable("id", TypeProtos.MinorType.BIGINT).addNullable("name", TypeProtos.MinorType.VARCHAR).build();
    Map<SchemaPath, ColumnStatistics<?>> columnStatistics = ImmutableMap.<SchemaPath, ColumnStatistics<?>>builder().put(SchemaPath.getSimplePath("name"), getColumnStatistics(null, null, 0L, TypeProtos.MinorType.VARCHAR)).put(SchemaPath.getSimplePath("id"), getColumnStatistics(null, null, 0L, TypeProtos.MinorType.BIGINT)).build();
    BaseTableMetadata expectedTableMetadata = BaseTableMetadata.builder().tableInfo(tableInfo).metadataInfo(TABLE_META_INFO).schema(schema).location(new Path(table.toURI().getPath())).columnsStatistics(columnStatistics).metadataStatistics(Arrays.asList(new StatisticsHolder<>(0L, TableStatisticsKind.ROW_COUNT), new StatisticsHolder<>(MetadataType.ALL, TableStatisticsKind.ANALYZE_METADATA_LEVEL))).partitionKeys(Collections.emptyMap()).lastModifiedTime(getMaxLastModified(table)).build();
    try {
        testBuilder().sqlQuery("ANALYZE TABLE dfs.`%s` REFRESH METADATA", tableName).unOrdered().baselineColumns("ok", "summary").baselineValues(true, String.format("Collected / refreshed metadata for table [dfs.default.%s]", tableName)).go();
        MetastoreTableInfo metastoreTableInfo = cluster.drillbit().getContext().getMetastoreRegistry().get().tables().basicRequests().metastoreTableInfo(tableInfo);
        assertTrue("table metadata wasn't found", metastoreTableInfo.isExists());
        BaseTableMetadata tableMetadata = cluster.drillbit().getContext().getMetastoreRegistry().get().tables().basicRequests().tableMetadata(tableInfo);
        assertEquals(expectedTableMetadata, tableMetadata);
        List<FileMetadata> filesMetadata = cluster.drillbit().getContext().getMetastoreRegistry().get().tables().basicRequests().filesMetadata(tableInfo, null, null);
        assertEquals(1, filesMetadata.size());
        List<RowGroupMetadata> rowGroupsMetadata = cluster.drillbit().getContext().getMetastoreRegistry().get().tables().basicRequests().rowGroupsMetadata(tableInfo, (String) null, null);
        assertEquals(1, rowGroupsMetadata.size());
    } finally {
        run("analyze table dfs.`%s` drop metadata if exists", tableName);
    }
}