Search in sources :

Example 1 with SqlShowFiles

use of org.apache.drill.exec.planner.sql.parser.SqlShowFiles in project drill by axbaretto.

the class ShowFileHandler method getPlan.

@Override
public PhysicalPlan getPlan(SqlNode sqlNode) throws ValidationException, RelConversionException, IOException {
    SqlIdentifier from = ((SqlShowFiles) sqlNode).getDb();
    DrillFileSystem fs;
    String defaultLocation;
    String fromDir = "./";
    SchemaPlus defaultSchema = config.getConverter().getDefaultSchema();
    SchemaPlus drillSchema = defaultSchema;
    // Show files can be used without from clause, in which case we display the files in the default schema
    if (from != null) {
        // We are not sure if the full from clause is just the schema or includes table name,
        // first try to see if the full path specified is a schema
        drillSchema = SchemaUtilites.findSchema(defaultSchema, from.names);
        if (drillSchema == null) {
            // Entire from clause is not a schema, try to obtain the schema without the last part of the specified clause.
            drillSchema = SchemaUtilites.findSchema(defaultSchema, from.names.subList(0, from.names.size() - 1));
            fromDir = fromDir + from.names.get((from.names.size() - 1));
        }
        if (drillSchema == null) {
            throw UserException.validationError().message("Invalid FROM/IN clause [%s]", from.toString()).build(logger);
        }
    }
    WorkspaceSchema wsSchema;
    try {
        wsSchema = (WorkspaceSchema) drillSchema.unwrap(AbstractSchema.class).getDefaultSchema();
    } catch (ClassCastException e) {
        throw UserException.validationError().message("SHOW FILES is supported in workspace type schema only. Schema [%s] is not a workspace schema.", SchemaUtilites.getSchemaPath(drillSchema)).build(logger);
    }
    // Get the file system object
    fs = wsSchema.getFS();
    // Get the default path
    defaultLocation = wsSchema.getDefaultLocation();
    List<ShowFilesCommandResult> rows = new ArrayList<>();
    for (FileStatus fileStatus : FileSystemUtil.listAll(fs, new Path(defaultLocation, fromDir), false)) {
        ShowFilesCommandResult result = new ShowFilesCommandResult(fileStatus.getPath().getName(), fileStatus.isDirectory(), fileStatus.isFile(), fileStatus.getLen(), fileStatus.getOwner(), fileStatus.getGroup(), fileStatus.getPermission().toString(), fileStatus.getAccessTime(), fileStatus.getModificationTime());
        rows.add(result);
    }
    return DirectPlan.createDirectPlan(context.getCurrentEndpoint(), rows, ShowFilesCommandResult.class);
}
Also used : Path(org.apache.hadoop.fs.Path) FileStatus(org.apache.hadoop.fs.FileStatus) WorkspaceSchema(org.apache.drill.exec.store.dfs.WorkspaceSchemaFactory.WorkspaceSchema) SchemaPlus(org.apache.calcite.schema.SchemaPlus) ArrayList(java.util.ArrayList) SqlIdentifier(org.apache.calcite.sql.SqlIdentifier) SqlShowFiles(org.apache.drill.exec.planner.sql.parser.SqlShowFiles) DrillFileSystem(org.apache.drill.exec.store.dfs.DrillFileSystem) AbstractSchema(org.apache.drill.exec.store.AbstractSchema)

Example 2 with SqlShowFiles

use of org.apache.drill.exec.planner.sql.parser.SqlShowFiles in project drill by apache.

the class ShowFilesHandler method getPlan.

@Override
public PhysicalPlan getPlan(SqlNode sqlNode) throws ForemanSetupException {
    SchemaPlus defaultSchema = config.getConverter().getDefaultSchema();
    SchemaPlus drillSchema = defaultSchema;
    SqlShowFiles showFiles = unwrap(sqlNode, SqlShowFiles.class);
    SqlIdentifier from = showFiles.getDb();
    String fromDir = null;
    // Show files can be used without from clause, in which case we display the files in the default schema
    if (from != null) {
        // We are not sure if the full from clause is just the schema or includes table name,
        // first try to see if the full path specified is a schema
        drillSchema = SchemaUtilites.findSchema(defaultSchema, from.names);
        if (drillSchema == null) {
            // Entire from clause is not a schema, try to obtain the schema without the last part of the specified clause.
            drillSchema = SchemaUtilites.findSchema(defaultSchema, from.names.subList(0, from.names.size() - 1));
            // Listing for specific directory: show files in dfs.tmp.specific_directory
            fromDir = from.names.get((from.names.size() - 1));
        }
        if (drillSchema == null) {
            throw UserException.validationError().message("Invalid FROM/IN clause [%s]", from.toString()).build(logger);
        }
    }
    WorkspaceSchema wsSchema;
    try {
        wsSchema = (WorkspaceSchema) drillSchema.unwrap(AbstractSchema.class).getDefaultSchema();
    } catch (ClassCastException e) {
        throw UserException.validationError().message("SHOW FILES is supported in workspace type schema only. Schema [%s] is not a workspace schema.", SchemaUtilites.getSchemaPath(drillSchema)).build(logger);
    }
    Path endPath = fromDir == null ? new Path(wsSchema.getDefaultLocation()) : new Path(wsSchema.getDefaultLocation(), fromDir);
    // add URI to the path to ensure that directory objects are skipped (see S3AFileSystem.listStatus method)
    Path path = new Path(wsSchema.getFS().getUri().toString(), endPath);
    List<ShowFilesCommandResult> records = FileSystemUtil.listAllSafe(wsSchema.getFS(), path, false).stream().map(fileStatus -> new ShowFilesCommandResult(new Records.File(wsSchema.getFullSchemaName(), wsSchema, fileStatus))).collect(Collectors.toList());
    return DirectPlan.createDirectPlan(context.getCurrentEndpoint(), records, ShowFilesCommandResult.class);
}
Also used : Path(org.apache.hadoop.fs.Path) SchemaUtilites(org.apache.drill.exec.planner.sql.SchemaUtilites) SqlShowFiles(org.apache.drill.exec.planner.sql.parser.SqlShowFiles) SchemaPlus(org.apache.calcite.schema.SchemaPlus) Records(org.apache.drill.exec.store.ischema.Records) Logger(org.slf4j.Logger) UserException(org.apache.drill.common.exceptions.UserException) Timestamp(java.sql.Timestamp) WorkspaceSchema(org.apache.drill.exec.store.dfs.WorkspaceSchemaFactory.WorkspaceSchema) Collectors(java.util.stream.Collectors) FileSystemUtil(org.apache.drill.exec.util.FileSystemUtil) List(java.util.List) SqlNode(org.apache.calcite.sql.SqlNode) ForemanSetupException(org.apache.drill.exec.work.foreman.ForemanSetupException) AbstractSchema(org.apache.drill.exec.store.AbstractSchema) DirectPlan(org.apache.drill.exec.planner.sql.DirectPlan) LoggerFactory.getLogger(org.slf4j.LoggerFactory.getLogger) SqlIdentifier(org.apache.calcite.sql.SqlIdentifier) PhysicalPlan(org.apache.drill.exec.physical.PhysicalPlan) Path(org.apache.hadoop.fs.Path) WorkspaceSchema(org.apache.drill.exec.store.dfs.WorkspaceSchemaFactory.WorkspaceSchema) SchemaPlus(org.apache.calcite.schema.SchemaPlus) SqlIdentifier(org.apache.calcite.sql.SqlIdentifier) SqlShowFiles(org.apache.drill.exec.planner.sql.parser.SqlShowFiles) AbstractSchema(org.apache.drill.exec.store.AbstractSchema) Records(org.apache.drill.exec.store.ischema.Records)

Example 3 with SqlShowFiles

use of org.apache.drill.exec.planner.sql.parser.SqlShowFiles in project drill by apache.

the class ShowFileHandler method getPlan.

@Override
public PhysicalPlan getPlan(SqlNode sqlNode) throws ValidationException, RelConversionException, IOException {
    SqlIdentifier from = ((SqlShowFiles) sqlNode).getDb();
    DrillFileSystem fs = null;
    String defaultLocation = null;
    String fromDir = "./";
    SchemaPlus defaultSchema = config.getConverter().getDefaultSchema();
    SchemaPlus drillSchema = defaultSchema;
    // Show files can be used without from clause, in which case we display the files in the default schema
    if (from != null) {
        // We are not sure if the full from clause is just the schema or includes table name,
        // first try to see if the full path specified is a schema
        drillSchema = SchemaUtilites.findSchema(defaultSchema, from.names);
        if (drillSchema == null) {
            // Entire from clause is not a schema, try to obtain the schema without the last part of the specified clause.
            drillSchema = SchemaUtilites.findSchema(defaultSchema, from.names.subList(0, from.names.size() - 1));
            fromDir = fromDir + from.names.get((from.names.size() - 1));
        }
        if (drillSchema == null) {
            throw UserException.validationError().message("Invalid FROM/IN clause [%s]", from.toString()).build(logger);
        }
    }
    WorkspaceSchema wsSchema;
    try {
        wsSchema = (WorkspaceSchema) drillSchema.unwrap(AbstractSchema.class).getDefaultSchema();
    } catch (ClassCastException e) {
        throw UserException.validationError().message("SHOW FILES is supported in workspace type schema only. Schema [%s] is not a workspace schema.", SchemaUtilites.getSchemaPath(drillSchema)).build(logger);
    }
    // Get the file system object
    fs = wsSchema.getFS();
    // Get the default path
    defaultLocation = wsSchema.getDefaultLocation();
    List<ShowFilesCommandResult> rows = new ArrayList<>();
    for (FileStatus fileStatus : fs.list(false, new Path(defaultLocation, fromDir))) {
        ShowFilesCommandResult result = new ShowFilesCommandResult(fileStatus.getPath().getName(), fileStatus.isDir(), !fileStatus.isDir(), fileStatus.getLen(), fileStatus.getOwner(), fileStatus.getGroup(), fileStatus.getPermission().toString(), fileStatus.getAccessTime(), fileStatus.getModificationTime());
        rows.add(result);
    }
    return DirectPlan.createDirectPlan(context.getCurrentEndpoint(), rows.iterator(), ShowFilesCommandResult.class);
}
Also used : Path(org.apache.hadoop.fs.Path) FileStatus(org.apache.hadoop.fs.FileStatus) WorkspaceSchema(org.apache.drill.exec.store.dfs.WorkspaceSchemaFactory.WorkspaceSchema) SchemaPlus(org.apache.calcite.schema.SchemaPlus) ArrayList(java.util.ArrayList) SqlIdentifier(org.apache.calcite.sql.SqlIdentifier) SqlShowFiles(org.apache.drill.exec.planner.sql.parser.SqlShowFiles) DrillFileSystem(org.apache.drill.exec.store.dfs.DrillFileSystem) AbstractSchema(org.apache.drill.exec.store.AbstractSchema)

Aggregations

SchemaPlus (org.apache.calcite.schema.SchemaPlus)3 SqlIdentifier (org.apache.calcite.sql.SqlIdentifier)3 SqlShowFiles (org.apache.drill.exec.planner.sql.parser.SqlShowFiles)3 AbstractSchema (org.apache.drill.exec.store.AbstractSchema)3 WorkspaceSchema (org.apache.drill.exec.store.dfs.WorkspaceSchemaFactory.WorkspaceSchema)3 Path (org.apache.hadoop.fs.Path)3 ArrayList (java.util.ArrayList)2 DrillFileSystem (org.apache.drill.exec.store.dfs.DrillFileSystem)2 FileStatus (org.apache.hadoop.fs.FileStatus)2 Timestamp (java.sql.Timestamp)1 List (java.util.List)1 Collectors (java.util.stream.Collectors)1 SqlNode (org.apache.calcite.sql.SqlNode)1 UserException (org.apache.drill.common.exceptions.UserException)1 PhysicalPlan (org.apache.drill.exec.physical.PhysicalPlan)1 DirectPlan (org.apache.drill.exec.planner.sql.DirectPlan)1 SchemaUtilites (org.apache.drill.exec.planner.sql.SchemaUtilites)1 Records (org.apache.drill.exec.store.ischema.Records)1 FileSystemUtil (org.apache.drill.exec.util.FileSystemUtil)1 ForemanSetupException (org.apache.drill.exec.work.foreman.ForemanSetupException)1