Search in sources :

Example 1 with DrillTable

use of org.apache.drill.exec.planner.logical.DrillTable in project drill by apache.

the class PruneScanRule method getDrillTable.

private static DrillTable getDrillTable(final TableScan scan) {
    DrillTable drillTable;
    drillTable = scan.getTable().unwrap(DrillTable.class);
    if (drillTable == null) {
        drillTable = scan.getTable().unwrap(DrillTranslatableTable.class).getDrillTable();
    }
    return drillTable;
}
Also used : DrillTable(org.apache.drill.exec.planner.logical.DrillTable)

Example 2 with DrillTable

use of org.apache.drill.exec.planner.logical.DrillTable in project drill by axbaretto.

the class KafkaMessageSchema method getTable.

@Override
public Table getTable(String tableName) {
    if (!drillTables.containsKey(tableName)) {
        KafkaScanSpec scanSpec = new KafkaScanSpec(tableName);
        DrillTable table = new DynamicDrillTable(plugin, getName(), scanSpec);
        drillTables.put(tableName, table);
    }
    return drillTables.get(tableName);
}
Also used : DynamicDrillTable(org.apache.drill.exec.planner.logical.DynamicDrillTable) DrillTable(org.apache.drill.exec.planner.logical.DrillTable) KafkaScanSpec(org.apache.drill.exec.store.kafka.KafkaScanSpec) DynamicDrillTable(org.apache.drill.exec.planner.logical.DynamicDrillTable)

Example 3 with DrillTable

use of org.apache.drill.exec.planner.logical.DrillTable in project drill by axbaretto.

the class RefreshMetadataHandler method getPlan.

@Override
public PhysicalPlan getPlan(SqlNode sqlNode) throws ValidationException, RelConversionException, IOException, ForemanSetupException {
    final SqlRefreshMetadata refreshTable = unwrap(sqlNode, SqlRefreshMetadata.class);
    try {
        final SchemaPlus schema = findSchema(config.getConverter().getDefaultSchema(), refreshTable.getSchemaPath());
        if (schema == null) {
            return direct(false, "Storage plugin or workspace does not exist [%s]", SchemaUtilites.SCHEMA_PATH_JOINER.join(refreshTable.getSchemaPath()));
        }
        final String tableName = refreshTable.getName();
        if (tableName.contains("*") || tableName.contains("?")) {
            return direct(false, "Glob path %s not supported for metadata refresh", tableName);
        }
        final Table table = schema.getTable(tableName);
        if (table == null) {
            return direct(false, "Table %s does not exist.", tableName);
        }
        if (!(table instanceof DrillTable)) {
            return notSupported(tableName);
        }
        final DrillTable drillTable = (DrillTable) table;
        final Object selection = drillTable.getSelection();
        if (selection instanceof FileSelection && ((FileSelection) selection).isEmptyDirectory()) {
            return direct(false, "Table %s is empty and doesn't contain any parquet files.", tableName);
        }
        if (!(selection instanceof FormatSelection)) {
            return notSupported(tableName);
        }
        FormatSelection formatSelection = (FormatSelection) selection;
        FormatPluginConfig formatConfig = formatSelection.getFormat();
        if (!((formatConfig instanceof ParquetFormatConfig) || ((formatConfig instanceof NamedFormatPluginConfig) && ((NamedFormatPluginConfig) formatConfig).name.equals("parquet")))) {
            return notSupported(tableName);
        }
        FileSystemPlugin plugin = (FileSystemPlugin) drillTable.getPlugin();
        DrillFileSystem fs = new DrillFileSystem(plugin.getFormatPlugin(formatSelection.getFormat()).getFsConf());
        String selectionRoot = formatSelection.getSelection().selectionRoot;
        if (!fs.getFileStatus(new Path(selectionRoot)).isDirectory()) {
            return notSupported(tableName);
        }
        if (!(formatConfig instanceof ParquetFormatConfig)) {
            formatConfig = new ParquetFormatConfig();
        }
        Metadata.createMeta(fs, selectionRoot, (ParquetFormatConfig) formatConfig);
        return direct(true, "Successfully updated metadata for table %s.", tableName);
    } catch (Exception e) {
        logger.error("Failed to update metadata for table '{}'", refreshTable.getName(), e);
        return DirectPlan.createDirectPlan(context, false, String.format("Error: %s", e.getMessage()));
    }
}
Also used : FileSelection(org.apache.drill.exec.store.dfs.FileSelection) Path(org.apache.hadoop.fs.Path) FileSystemPlugin(org.apache.drill.exec.store.dfs.FileSystemPlugin) Table(org.apache.calcite.schema.Table) DrillTable(org.apache.drill.exec.planner.logical.DrillTable) DrillTable(org.apache.drill.exec.planner.logical.DrillTable) SchemaPlus(org.apache.calcite.schema.SchemaPlus) FormatSelection(org.apache.drill.exec.store.dfs.FormatSelection) SqlRefreshMetadata(org.apache.drill.exec.planner.sql.parser.SqlRefreshMetadata) IOException(java.io.IOException) ValidationException(org.apache.calcite.tools.ValidationException) ForemanSetupException(org.apache.drill.exec.work.foreman.ForemanSetupException) RelConversionException(org.apache.calcite.tools.RelConversionException) NamedFormatPluginConfig(org.apache.drill.exec.store.dfs.NamedFormatPluginConfig) DrillFileSystem(org.apache.drill.exec.store.dfs.DrillFileSystem) NamedFormatPluginConfig(org.apache.drill.exec.store.dfs.NamedFormatPluginConfig) FormatPluginConfig(org.apache.drill.common.logical.FormatPluginConfig) ParquetFormatConfig(org.apache.drill.exec.store.parquet.ParquetFormatConfig)

Example 4 with DrillTable

use of org.apache.drill.exec.planner.logical.DrillTable in project drill by apache.

the class KafkaMessageSchema method getTable.

@Override
public Table getTable(String tableName) {
    if (!drillTables.containsKey(tableName)) {
        KafkaScanSpec scanSpec = new KafkaScanSpec(tableName);
        DrillTable table = new DynamicDrillTable(plugin, getName(), scanSpec);
        drillTables.put(tableName, table);
    }
    return drillTables.get(tableName);
}
Also used : DynamicDrillTable(org.apache.drill.exec.planner.logical.DynamicDrillTable) DrillTable(org.apache.drill.exec.planner.logical.DrillTable) KafkaScanSpec(org.apache.drill.exec.store.kafka.KafkaScanSpec) DynamicDrillTable(org.apache.drill.exec.planner.logical.DynamicDrillTable)

Example 5 with DrillTable

use of org.apache.drill.exec.planner.logical.DrillTable in project drill by apache.

the class DrillRelMdSelectivity method getScanSelectivity.

private Double getScanSelectivity(RelNode rel, RelMetadataQuery mq, RexNode predicate) {
    double ROWCOUNT_UNKNOWN = -1.0;
    GroupScan scan = null;
    PlannerSettings settings = PrelUtil.getPlannerSettings(rel.getCluster().getPlanner());
    final RexBuilder rexBuilder = rel.getCluster().getRexBuilder();
    if (rel instanceof DrillScanRel) {
        scan = ((DrillScanRel) rel).getGroupScan();
    } else if (rel instanceof ScanPrel) {
        scan = ((ScanPrel) rel).getGroupScan();
    }
    if (scan != null) {
        if (settings.isStatisticsEnabled() && scan instanceof DbGroupScan) {
            double filterRows = ((DbGroupScan) scan).getRowCount(predicate, rel);
            double totalRows = ((DbGroupScan) scan).getRowCount(null, rel);
            if (filterRows != ROWCOUNT_UNKNOWN && totalRows != ROWCOUNT_UNKNOWN && totalRows > 0) {
                return Math.min(1.0, filterRows / totalRows);
            }
        }
    }
    // Do not mess with statistics used for DBGroupScans.
    if (rel instanceof TableScan) {
        if (DrillRelOptUtil.guessRows(rel)) {
            return super.getSelectivity(rel, mq, predicate);
        }
        DrillTable table = Utilities.getDrillTable(rel.getTable());
        try {
            TableMetadata tableMetadata;
            if (table != null && (tableMetadata = table.getGroupScan().getTableMetadata()) != null && TableStatisticsKind.HAS_DESCRIPTIVE_STATISTICS.getValue(tableMetadata)) {
                List<SchemaPath> fieldNames;
                if (rel instanceof DrillScanRelBase) {
                    fieldNames = ((DrillScanRelBase) rel).getGroupScan().getColumns();
                } else {
                    fieldNames = rel.getRowType().getFieldNames().stream().map(SchemaPath::getSimplePath).collect(Collectors.toList());
                }
                return getScanSelectivityInternal(tableMetadata, predicate, fieldNames, rexBuilder);
            }
        } catch (IOException e) {
            super.getSelectivity(rel, mq, predicate);
        }
    }
    return super.getSelectivity(rel, mq, predicate);
}
Also used : TableMetadata(org.apache.drill.metastore.metadata.TableMetadata) TableScan(org.apache.calcite.rel.core.TableScan) DrillScanRel(org.apache.drill.exec.planner.logical.DrillScanRel) ScanPrel(org.apache.drill.exec.planner.physical.ScanPrel) DrillTable(org.apache.drill.exec.planner.logical.DrillTable) PlannerSettings(org.apache.drill.exec.planner.physical.PlannerSettings) IOException(java.io.IOException) DbGroupScan(org.apache.drill.exec.physical.base.DbGroupScan) GroupScan(org.apache.drill.exec.physical.base.GroupScan) SchemaPath(org.apache.drill.common.expression.SchemaPath) DbGroupScan(org.apache.drill.exec.physical.base.DbGroupScan) DrillScanRelBase(org.apache.drill.exec.planner.common.DrillScanRelBase) RexBuilder(org.apache.calcite.rex.RexBuilder)

Aggregations

DrillTable (org.apache.drill.exec.planner.logical.DrillTable)17 IOException (java.io.IOException)8 FormatSelection (org.apache.drill.exec.store.dfs.FormatSelection)6 SchemaPath (org.apache.drill.common.expression.SchemaPath)5 Table (org.apache.calcite.schema.Table)4 PlannerSettings (org.apache.drill.exec.planner.physical.PlannerSettings)4 TableScan (org.apache.calcite.rel.core.TableScan)3 DrillRuntimeException (org.apache.drill.common.exceptions.DrillRuntimeException)3 FormatPluginConfig (org.apache.drill.common.logical.FormatPluginConfig)3 DynamicDrillTable (org.apache.drill.exec.planner.logical.DynamicDrillTable)3 DrillFileSystem (org.apache.drill.exec.store.dfs.DrillFileSystem)3 Path (org.apache.hadoop.fs.Path)3 ArrayList (java.util.ArrayList)2 Arrays (java.util.Arrays)2 Collections (java.util.Collections)2 List (java.util.List)2 Collectors (java.util.stream.Collectors)2 RelNode (org.apache.calcite.rel.RelNode)2 RelDataType (org.apache.calcite.rel.type.RelDataType)2 SchemaPlus (org.apache.calcite.schema.SchemaPlus)2