Search in sources :

Example 1 with SqlRefreshMetadata

use of org.apache.drill.exec.planner.sql.parser.SqlRefreshMetadata in project drill by axbaretto.

the class RefreshMetadataHandler method getPlan.

@Override
public PhysicalPlan getPlan(SqlNode sqlNode) throws ValidationException, RelConversionException, IOException, ForemanSetupException {
    final SqlRefreshMetadata refreshTable = unwrap(sqlNode, SqlRefreshMetadata.class);
    try {
        final SchemaPlus schema = findSchema(config.getConverter().getDefaultSchema(), refreshTable.getSchemaPath());
        if (schema == null) {
            return direct(false, "Storage plugin or workspace does not exist [%s]", SchemaUtilites.SCHEMA_PATH_JOINER.join(refreshTable.getSchemaPath()));
        }
        final String tableName = refreshTable.getName();
        if (tableName.contains("*") || tableName.contains("?")) {
            return direct(false, "Glob path %s not supported for metadata refresh", tableName);
        }
        final Table table = schema.getTable(tableName);
        if (table == null) {
            return direct(false, "Table %s does not exist.", tableName);
        }
        if (!(table instanceof DrillTable)) {
            return notSupported(tableName);
        }
        final DrillTable drillTable = (DrillTable) table;
        final Object selection = drillTable.getSelection();
        if (selection instanceof FileSelection && ((FileSelection) selection).isEmptyDirectory()) {
            return direct(false, "Table %s is empty and doesn't contain any parquet files.", tableName);
        }
        if (!(selection instanceof FormatSelection)) {
            return notSupported(tableName);
        }
        FormatSelection formatSelection = (FormatSelection) selection;
        FormatPluginConfig formatConfig = formatSelection.getFormat();
        if (!((formatConfig instanceof ParquetFormatConfig) || ((formatConfig instanceof NamedFormatPluginConfig) && ((NamedFormatPluginConfig) formatConfig).name.equals("parquet")))) {
            return notSupported(tableName);
        }
        FileSystemPlugin plugin = (FileSystemPlugin) drillTable.getPlugin();
        DrillFileSystem fs = new DrillFileSystem(plugin.getFormatPlugin(formatSelection.getFormat()).getFsConf());
        String selectionRoot = formatSelection.getSelection().selectionRoot;
        if (!fs.getFileStatus(new Path(selectionRoot)).isDirectory()) {
            return notSupported(tableName);
        }
        if (!(formatConfig instanceof ParquetFormatConfig)) {
            formatConfig = new ParquetFormatConfig();
        }
        Metadata.createMeta(fs, selectionRoot, (ParquetFormatConfig) formatConfig);
        return direct(true, "Successfully updated metadata for table %s.", tableName);
    } catch (Exception e) {
        logger.error("Failed to update metadata for table '{}'", refreshTable.getName(), e);
        return DirectPlan.createDirectPlan(context, false, String.format("Error: %s", e.getMessage()));
    }
}
Also used : FileSelection(org.apache.drill.exec.store.dfs.FileSelection) Path(org.apache.hadoop.fs.Path) FileSystemPlugin(org.apache.drill.exec.store.dfs.FileSystemPlugin) Table(org.apache.calcite.schema.Table) DrillTable(org.apache.drill.exec.planner.logical.DrillTable) DrillTable(org.apache.drill.exec.planner.logical.DrillTable) SchemaPlus(org.apache.calcite.schema.SchemaPlus) FormatSelection(org.apache.drill.exec.store.dfs.FormatSelection) SqlRefreshMetadata(org.apache.drill.exec.planner.sql.parser.SqlRefreshMetadata) IOException(java.io.IOException) ValidationException(org.apache.calcite.tools.ValidationException) ForemanSetupException(org.apache.drill.exec.work.foreman.ForemanSetupException) RelConversionException(org.apache.calcite.tools.RelConversionException) NamedFormatPluginConfig(org.apache.drill.exec.store.dfs.NamedFormatPluginConfig) DrillFileSystem(org.apache.drill.exec.store.dfs.DrillFileSystem) NamedFormatPluginConfig(org.apache.drill.exec.store.dfs.NamedFormatPluginConfig) FormatPluginConfig(org.apache.drill.common.logical.FormatPluginConfig) ParquetFormatConfig(org.apache.drill.exec.store.parquet.ParquetFormatConfig)

Example 2 with SqlRefreshMetadata

use of org.apache.drill.exec.planner.sql.parser.SqlRefreshMetadata in project drill by apache.

the class RefreshMetadataHandler method getPlan.

@Override
public PhysicalPlan getPlan(SqlNode sqlNode) throws ForemanSetupException {
    final SqlRefreshMetadata refreshTable = unwrap(sqlNode, SqlRefreshMetadata.class);
    try {
        final SchemaPlus schema = findSchema(config.getConverter().getDefaultSchema(), refreshTable.getSchemaPath());
        if (schema == null) {
            return direct(false, "Storage plugin or workspace does not exist [%s]", SchemaUtilites.SCHEMA_PATH_JOINER.join(refreshTable.getSchemaPath()));
        }
        final String tableName = refreshTable.getName();
        final SqlNodeList columnList = getColumnList(refreshTable);
        final Set<SchemaPath> columnSet = getColumnRootSegments(columnList);
        final SqlLiteral allColumns = refreshTable.getAllColumns();
        if (tableName.contains("*") || tableName.contains("?")) {
            return direct(false, "Glob path %s not supported for metadata refresh", tableName);
        }
        final Table table = schema.getTable(tableName);
        if (table == null) {
            return direct(false, "Table %s does not exist.", tableName);
        }
        if (!(table instanceof DrillTable)) {
            return notSupported(tableName);
        }
        final DrillTable drillTable = (DrillTable) table;
        final Object selection = drillTable.getSelection();
        if (selection instanceof FileSelection && ((FileSelection) selection).isEmptyDirectory()) {
            return direct(false, "Table %s is empty and doesn't contain any parquet files.", tableName);
        }
        if (!(selection instanceof FormatSelection)) {
            return notSupported(tableName);
        }
        final FormatSelection formatSelection = (FormatSelection) selection;
        FormatPluginConfig formatConfig = formatSelection.getFormat();
        if (!((formatConfig instanceof ParquetFormatConfig) || ((formatConfig instanceof NamedFormatPluginConfig) && ((NamedFormatPluginConfig) formatConfig).getName().equals("parquet")))) {
            return notSupported(tableName);
        }
        // Always create filesystem object using process user, since it owns the metadata file
        final DrillFileSystem fs = ImpersonationUtil.createFileSystem(ImpersonationUtil.getProcessUserName(), drillTable.getPlugin().getFormatPlugin(formatConfig).getFsConf());
        final Path selectionRoot = formatSelection.getSelection().getSelectionRoot();
        if (!fs.getFileStatus(selectionRoot).isDirectory()) {
            return notSupported(tableName);
        }
        if (!(formatConfig instanceof ParquetFormatConfig)) {
            formatConfig = new ParquetFormatConfig();
        }
        final ParquetReaderConfig readerConfig = ParquetReaderConfig.builder().withFormatConfig((ParquetFormatConfig) formatConfig).withOptions(context.getOptions()).build();
        Metadata.createMeta(fs, selectionRoot, readerConfig, allColumns.booleanValue(), columnSet);
        return direct(true, "Successfully updated metadata for table %s.", tableName);
    } catch (Exception e) {
        logger.error("Failed to update metadata for table '{}'", refreshTable.getName(), e);
        return DirectPlan.createDirectPlan(context, false, String.format("Error: %s", e.getMessage()));
    }
}
Also used : FileSelection(org.apache.drill.exec.store.dfs.FileSelection) Path(org.apache.hadoop.fs.Path) SchemaPath(org.apache.drill.common.expression.SchemaPath) DrillTable(org.apache.drill.exec.planner.logical.DrillTable) Table(org.apache.calcite.schema.Table) DrillTable(org.apache.drill.exec.planner.logical.DrillTable) SchemaPlus(org.apache.calcite.schema.SchemaPlus) FormatSelection(org.apache.drill.exec.store.dfs.FormatSelection) SqlRefreshMetadata(org.apache.drill.exec.planner.sql.parser.SqlRefreshMetadata) ForemanSetupException(org.apache.drill.exec.work.foreman.ForemanSetupException) NamedFormatPluginConfig(org.apache.drill.exec.store.dfs.NamedFormatPluginConfig) DrillFileSystem(org.apache.drill.exec.store.dfs.DrillFileSystem) SchemaPath(org.apache.drill.common.expression.SchemaPath) FormatPluginConfig(org.apache.drill.common.logical.FormatPluginConfig) NamedFormatPluginConfig(org.apache.drill.exec.store.dfs.NamedFormatPluginConfig) SqlNodeList(org.apache.calcite.sql.SqlNodeList) ParquetFormatConfig(org.apache.drill.exec.store.parquet.ParquetFormatConfig) SqlLiteral(org.apache.calcite.sql.SqlLiteral) ParquetReaderConfig(org.apache.drill.exec.store.parquet.ParquetReaderConfig)

Aggregations

SchemaPlus (org.apache.calcite.schema.SchemaPlus)2 Table (org.apache.calcite.schema.Table)2 FormatPluginConfig (org.apache.drill.common.logical.FormatPluginConfig)2 DrillTable (org.apache.drill.exec.planner.logical.DrillTable)2 SqlRefreshMetadata (org.apache.drill.exec.planner.sql.parser.SqlRefreshMetadata)2 DrillFileSystem (org.apache.drill.exec.store.dfs.DrillFileSystem)2 FileSelection (org.apache.drill.exec.store.dfs.FileSelection)2 FormatSelection (org.apache.drill.exec.store.dfs.FormatSelection)2 NamedFormatPluginConfig (org.apache.drill.exec.store.dfs.NamedFormatPluginConfig)2 ParquetFormatConfig (org.apache.drill.exec.store.parquet.ParquetFormatConfig)2 ForemanSetupException (org.apache.drill.exec.work.foreman.ForemanSetupException)2 Path (org.apache.hadoop.fs.Path)2 IOException (java.io.IOException)1 SqlLiteral (org.apache.calcite.sql.SqlLiteral)1 SqlNodeList (org.apache.calcite.sql.SqlNodeList)1 RelConversionException (org.apache.calcite.tools.RelConversionException)1 ValidationException (org.apache.calcite.tools.ValidationException)1 SchemaPath (org.apache.drill.common.expression.SchemaPath)1 FileSystemPlugin (org.apache.drill.exec.store.dfs.FileSystemPlugin)1 ParquetReaderConfig (org.apache.drill.exec.store.parquet.ParquetReaderConfig)1