use of org.apache.drill.exec.store.dfs.DrillFileSystem in project drill by apache.
the class LocalPersistentStore method getFileSystem.
public static DrillFileSystem getFileSystem(DrillConfig config, Path root) throws IOException {
Path blobRoot = root == null ? getLogDir() : root;
Configuration fsConf = new Configuration();
if (blobRoot.toUri().getScheme() != null) {
fsConf.set(FileSystem.FS_DEFAULT_NAME_KEY, blobRoot.toUri().toString());
}
DrillFileSystem fs = new DrillFileSystem(fsConf);
fs.mkdirs(blobRoot);
return fs;
}
use of org.apache.drill.exec.store.dfs.DrillFileSystem in project drill by apache.
the class EasyFormatPlugin method getReaderBatch.
@SuppressWarnings("resource")
CloseableRecordBatch getReaderBatch(FragmentContext context, EasySubScan scan) throws ExecutionSetupException {
final ImplicitColumnExplorer columnExplorer = new ImplicitColumnExplorer(context, scan.getColumns());
if (!columnExplorer.isStarQuery()) {
scan = new EasySubScan(scan.getUserName(), scan.getWorkUnits(), scan.getFormatPlugin(), columnExplorer.getTableColumns(), scan.getSelectionRoot());
scan.setOperatorId(scan.getOperatorId());
}
OperatorContext oContext = context.newOperatorContext(scan);
final DrillFileSystem dfs;
try {
dfs = oContext.newFileSystem(fsConf);
} catch (IOException e) {
throw new ExecutionSetupException(String.format("Failed to create FileSystem: %s", e.getMessage()), e);
}
List<RecordReader> readers = Lists.newArrayList();
List<Map<String, String>> implicitColumns = Lists.newArrayList();
Map<String, String> mapWithMaxColumns = Maps.newLinkedHashMap();
for (FileWork work : scan.getWorkUnits()) {
RecordReader recordReader = getRecordReader(context, dfs, work, scan.getColumns(), scan.getUserName());
readers.add(recordReader);
Map<String, String> implicitValues = columnExplorer.populateImplicitColumns(work, scan.getSelectionRoot());
implicitColumns.add(implicitValues);
if (implicitValues.size() > mapWithMaxColumns.size()) {
mapWithMaxColumns = implicitValues;
}
}
// all readers should have the same number of implicit columns, add missing ones with value null
Map<String, String> diff = Maps.transformValues(mapWithMaxColumns, Functions.constant((String) null));
for (Map<String, String> map : implicitColumns) {
map.putAll(Maps.difference(map, diff).entriesOnlyOnRight());
}
return new ScanBatch(scan, context, oContext, readers.iterator(), implicitColumns);
}
use of org.apache.drill.exec.store.dfs.DrillFileSystem in project drill by apache.
the class RefreshMetadataHandler method getPlan.
@Override
public PhysicalPlan getPlan(SqlNode sqlNode) throws ValidationException, RelConversionException, IOException, ForemanSetupException {
final SqlRefreshMetadata refreshTable = unwrap(sqlNode, SqlRefreshMetadata.class);
try {
final SchemaPlus schema = findSchema(config.getConverter().getDefaultSchema(), refreshTable.getSchemaPath());
if (schema == null) {
return direct(false, "Storage plugin or workspace does not exist [%s]", SchemaUtilites.SCHEMA_PATH_JOINER.join(refreshTable.getSchemaPath()));
}
final String tableName = refreshTable.getName();
if (tableName.contains("*") || tableName.contains("?")) {
return direct(false, "Glob path %s not supported for metadata refresh", tableName);
}
final Table table = schema.getTable(tableName);
if (table == null) {
return direct(false, "Table %s does not exist.", tableName);
}
if (!(table instanceof DrillTable)) {
return notSupported(tableName);
}
final DrillTable drillTable = (DrillTable) table;
final Object selection = drillTable.getSelection();
if (!(selection instanceof FormatSelection)) {
return notSupported(tableName);
}
FormatSelection formatSelection = (FormatSelection) selection;
FormatPluginConfig formatConfig = formatSelection.getFormat();
if (!((formatConfig instanceof ParquetFormatConfig) || ((formatConfig instanceof NamedFormatPluginConfig) && ((NamedFormatPluginConfig) formatConfig).name.equals("parquet")))) {
return notSupported(tableName);
}
FileSystemPlugin plugin = (FileSystemPlugin) drillTable.getPlugin();
DrillFileSystem fs = new DrillFileSystem(plugin.getFormatPlugin(formatSelection.getFormat()).getFsConf());
String selectionRoot = formatSelection.getSelection().selectionRoot;
if (!fs.getFileStatus(new Path(selectionRoot)).isDirectory()) {
return notSupported(tableName);
}
if (!(formatConfig instanceof ParquetFormatConfig)) {
formatConfig = new ParquetFormatConfig();
}
Metadata.createMeta(fs, selectionRoot, (ParquetFormatConfig) formatConfig);
return direct(true, "Successfully updated metadata for table %s.", tableName);
} catch (Exception e) {
logger.error("Failed to update metadata for table '{}'", refreshTable.getName(), e);
return DirectPlan.createDirectPlan(context, false, String.format("Error: %s", e.getMessage()));
}
}
use of org.apache.drill.exec.store.dfs.DrillFileSystem in project drill by apache.
the class ShowFileHandler method getPlan.
@Override
public PhysicalPlan getPlan(SqlNode sqlNode) throws ValidationException, RelConversionException, IOException {
SqlIdentifier from = ((SqlShowFiles) sqlNode).getDb();
DrillFileSystem fs = null;
String defaultLocation = null;
String fromDir = "./";
SchemaPlus defaultSchema = config.getConverter().getDefaultSchema();
SchemaPlus drillSchema = defaultSchema;
// Show files can be used without from clause, in which case we display the files in the default schema
if (from != null) {
// We are not sure if the full from clause is just the schema or includes table name,
// first try to see if the full path specified is a schema
drillSchema = SchemaUtilites.findSchema(defaultSchema, from.names);
if (drillSchema == null) {
// Entire from clause is not a schema, try to obtain the schema without the last part of the specified clause.
drillSchema = SchemaUtilites.findSchema(defaultSchema, from.names.subList(0, from.names.size() - 1));
fromDir = fromDir + from.names.get((from.names.size() - 1));
}
if (drillSchema == null) {
throw UserException.validationError().message("Invalid FROM/IN clause [%s]", from.toString()).build(logger);
}
}
WorkspaceSchema wsSchema;
try {
wsSchema = (WorkspaceSchema) drillSchema.unwrap(AbstractSchema.class).getDefaultSchema();
} catch (ClassCastException e) {
throw UserException.validationError().message("SHOW FILES is supported in workspace type schema only. Schema [%s] is not a workspace schema.", SchemaUtilites.getSchemaPath(drillSchema)).build(logger);
}
// Get the file system object
fs = wsSchema.getFS();
// Get the default path
defaultLocation = wsSchema.getDefaultLocation();
List<ShowFilesCommandResult> rows = new ArrayList<>();
for (FileStatus fileStatus : fs.list(false, new Path(defaultLocation, fromDir))) {
ShowFilesCommandResult result = new ShowFilesCommandResult(fileStatus.getPath().getName(), fileStatus.isDir(), !fileStatus.isDir(), fileStatus.getLen(), fileStatus.getOwner(), fileStatus.getGroup(), fileStatus.getPermission().toString(), fileStatus.getAccessTime(), fileStatus.getModificationTime());
rows.add(result);
}
return DirectPlan.createDirectPlan(context.getCurrentEndpoint(), rows.iterator(), ShowFilesCommandResult.class);
}
use of org.apache.drill.exec.store.dfs.DrillFileSystem in project drill by apache.
the class TestMiniPlan method initFS.
@BeforeClass
public static void initFS() throws Exception {
Configuration conf = new Configuration();
conf.set(FileSystem.FS_DEFAULT_NAME_KEY, FileSystem.DEFAULT_FS);
fs = new DrillFileSystem(conf);
}
Aggregations