Examples with FormatPluginConfig org.apache.drill.common.logical.FormatPluginConfig used on opensource projects

Example 1 with FormatPluginConfig

use of org.apache.drill.common.logical.FormatPluginConfig in project drill by axbaretto.

the class RefreshMetadataHandler method getPlan.

@Override
public PhysicalPlan getPlan(SqlNode sqlNode) throws ValidationException, RelConversionException, IOException, ForemanSetupException {
    final SqlRefreshMetadata refreshTable = unwrap(sqlNode, SqlRefreshMetadata.class);
    try {
        final SchemaPlus schema = findSchema(config.getConverter().getDefaultSchema(), refreshTable.getSchemaPath());
        if (schema == null) {
            return direct(false, "Storage plugin or workspace does not exist [%s]", SchemaUtilites.SCHEMA_PATH_JOINER.join(refreshTable.getSchemaPath()));
        }
        final String tableName = refreshTable.getName();
        if (tableName.contains("*") || tableName.contains("?")) {
            return direct(false, "Glob path %s not supported for metadata refresh", tableName);
        }
        final Table table = schema.getTable(tableName);
        if (table == null) {
            return direct(false, "Table %s does not exist.", tableName);
        }
        if (!(table instanceof DrillTable)) {
            return notSupported(tableName);
        }
        final DrillTable drillTable = (DrillTable) table;
        final Object selection = drillTable.getSelection();
        if (selection instanceof FileSelection && ((FileSelection) selection).isEmptyDirectory()) {
            return direct(false, "Table %s is empty and doesn't contain any parquet files.", tableName);
        }
        if (!(selection instanceof FormatSelection)) {
            return notSupported(tableName);
        }
        FormatSelection formatSelection = (FormatSelection) selection;
        FormatPluginConfig formatConfig = formatSelection.getFormat();
        if (!((formatConfig instanceof ParquetFormatConfig) || ((formatConfig instanceof NamedFormatPluginConfig) && ((NamedFormatPluginConfig) formatConfig).name.equals("parquet")))) {
            return notSupported(tableName);
        }
        FileSystemPlugin plugin = (FileSystemPlugin) drillTable.getPlugin();
        DrillFileSystem fs = new DrillFileSystem(plugin.getFormatPlugin(formatSelection.getFormat()).getFsConf());
        String selectionRoot = formatSelection.getSelection().selectionRoot;
        if (!fs.getFileStatus(new Path(selectionRoot)).isDirectory()) {
            return notSupported(tableName);
        }
        if (!(formatConfig instanceof ParquetFormatConfig)) {
            formatConfig = new ParquetFormatConfig();
        }
        Metadata.createMeta(fs, selectionRoot, (ParquetFormatConfig) formatConfig);
        return direct(true, "Successfully updated metadata for table %s.", tableName);
    } catch (Exception e) {
        logger.error("Failed to update metadata for table '{}'", refreshTable.getName(), e);
        return DirectPlan.createDirectPlan(context, false, String.format("Error: %s", e.getMessage()));
    }
}

Example 2 with FormatPluginConfig

use of org.apache.drill.common.logical.FormatPluginConfig in project drill by axbaretto.

the class FormatPluginOptionsDescriptor method createConfigForTable.

/**
 * creates an instance of the FormatPluginConfig based on the passed parameters
 * @param t the signature and the parameters passed to the table function
 * @return the corresponding config
 */
FormatPluginConfig createConfigForTable(TableInstance t) {
    // Per the constructor, the first param is always "type"
    TableParamDef typeParamDef = t.sig.params.get(0);
    Object typeParam = t.params.get(0);
    if (!typeParamDef.name.equals("type") || typeParamDef.type != String.class || !(typeParam instanceof String) || !typeName.equalsIgnoreCase((String) typeParam)) {
        // if we reach here, there's a bug as all signatures generated start with a type parameter
        throw UserException.parseError().message("This function signature is not supported: %s\n" + "expecting %s", t.presentParams(), this.presentParams()).addContext("table", t.sig.name).build(logger);
    }
    FormatPluginConfig config;
    try {
        config = pluginConfigClass.newInstance();
    } catch (InstantiationException | IllegalAccessException e) {
        throw UserException.parseError(e).message("configuration for format of type %s can not be created (class: %s)", this.typeName, pluginConfigClass.getName()).addContext("table", t.sig.name).build(logger);
    }
    for (int i = 1; i < t.params.size(); i++) {
        Object param = t.params.get(i);
        if (param == null) {
            // when null is passed, we leave the default defined in the config class
            continue;
        }
        if (param instanceof String) {
            // normalize Java literals, ex: \t, \n, \r
            param = StringEscapeUtils.unescapeJava((String) param);
        }
        TableParamDef paramDef = t.sig.params.get(i);
        TableParamDef expectedParamDef = this.functionParamsByName.get(paramDef.name);
        if (expectedParamDef == null || expectedParamDef.type != paramDef.type) {
            throw UserException.parseError().message("The parameters provided are not applicable to the type specified:\n" + "provided: %s\nexpected: %s", t.presentParams(), this.presentParams()).addContext("table", t.sig.name).build(logger);
        }
        try {
            Field field = pluginConfigClass.getField(paramDef.name);
            field.setAccessible(true);
            if (field.getType() == char.class && param instanceof String) {
                String stringParam = (String) param;
                if (stringParam.length() != 1) {
                    throw UserException.parseError().message("Expected single character but was String: %s", stringParam).addContext("table", t.sig.name).addContext("parameter", paramDef.name).build(logger);
                }
                param = stringParam.charAt(0);
            }
            field.set(config, param);
        } catch (IllegalAccessException | NoSuchFieldException | SecurityException e) {
            throw UserException.parseError(e).message("can not set value %s to parameter %s: %s", param, paramDef.name, paramDef.type).addContext("table", t.sig.name).addContext("parameter", paramDef.name).build(logger);
        }
    }
    return config;
}

Example 3 with FormatPluginConfig

use of org.apache.drill.common.logical.FormatPluginConfig in project drill by apache.

the class DrillStatsTable method readStatistics.

private TableStatistics readStatistics(DrillTable drillTable, Path path) throws IOException {
    final Object selection = drillTable.getSelection();
    if (selection instanceof FormatSelection) {
        StoragePlugin storagePlugin = drillTable.getPlugin();
        FormatSelection formatSelection = (FormatSelection) selection;
        FormatPluginConfig formatConfig = formatSelection.getFormat();
        if (storagePlugin instanceof FileSystemPlugin && (formatConfig instanceof ParquetFormatConfig)) {
            FormatPlugin fmtPlugin = storagePlugin.getFormatPlugin(formatConfig);
            if (fmtPlugin.supportsStatistics()) {
                return fmtPlugin.readStatistics(fs, path);
            }
        }
    }
    return null;
}

Example 4 with FormatPluginConfig

use of org.apache.drill.common.logical.FormatPluginConfig in project drill by apache.

the class PluginBootstrapLoaderImpl method loadFormatPlugins.

/**
 * Loads format plugins from the given URL and adds the formats to the
 * specified storage plugins
 *
 * @param url
 *          URL to the format plugins bootstrap file
 * @param bootstrapPlugins
 *          a collection with loaded storage plugins. New formats will be
 *          added to them
 * @param pluginURLMap
 *          a map to store correspondence between storage plugins and
 *          bootstrap files in which they are defined. Used for logging
 * @throws IOException
 *           if failed to retrieve a plugin from a bootstrap file
 */
private void loadFormatPlugins(URL url, StoragePlugins bootstrapPlugins, Map<String, URL> pluginURLMap) throws IOException {
    StoragePlugins plugins = getPluginsFromResource(url);
    for (Entry<String, StoragePluginConfig> sourceEntry : plugins) {
        String pluginName = sourceEntry.getKey();
        StoragePluginConfig sourcePlugin = sourceEntry.getValue();
        if (!(sourcePlugin instanceof FileSystemConfig)) {
            logger.warn("Formats are only supported by File System plugins. Source name '{}' is of type '{}'.", pluginName, sourcePlugin.getClass().getName());
            continue;
        }
        StoragePluginConfig targetPlugin = bootstrapPlugins.getConfig(pluginName);
        if (targetPlugin == null) {
            logger.warn("No boostrap storage plugin matches the name '{}'", pluginName);
            continue;
        }
        if (!(targetPlugin instanceof FileSystemConfig)) {
            logger.warn("Formats are only supported by File System plugins. Source name '{}' " + "is of type '{}' but the bootstrap plugin of that name is of type '{}.", pluginName, sourcePlugin.getClass().getName(), targetPlugin.getClass().getName());
            continue;
        }
        FileSystemConfig targetFsConfig = (FileSystemConfig) targetPlugin;
        FileSystemConfig sourceFsConfig = (FileSystemConfig) sourcePlugin;
        sourceFsConfig.getFormats().forEach((formatName, formatValue) -> {
            FormatPluginConfig oldPluginConfig = targetFsConfig.getFormats().putIfAbsent(formatName, formatValue);
            if (oldPluginConfig != null) {
                logger.warn("Duplicate format instance '{}' defined in '{}' and '{}', ignoring the later one.", formatName, pluginURLMap.get(pluginName), url);
            }
        });
    }
}

Example 5 with FormatPluginConfig

use of org.apache.drill.common.logical.FormatPluginConfig in project drill by apache.

the class DropboxFileSystemTest method setup.

/*
   All test files can be found in java-exec/src/test/resources/dropboxTestFiles

  Instructions for running Dropbox Unit Tests
  1.  Get your Dropbox API key as explained in the Drill docs and paste it above into the ACCESS_TOKEN variable.
  2.  In your dropbox account, create a folder called 'csv' and upload the file hdf-test.csvh into that folder
  3.  In your dropbox account, upload the file http-pcap.json to the root directory of your dropbox account
  4.  In the testListFiles test, you will have to update the modified dates
  5.  Run tests.
   */
@BeforeClass
public static void setup() throws Exception {
    assertTrue(!ACCESS_TOKEN.equalsIgnoreCase("<Your Dropbox Access Token Here>"));
    ClusterTest.startCluster(ClusterFixture.builder(dirTestWatcher));
    Map<String, String> dropboxConfigVars = new HashMap<>();
    dropboxConfigVars.put("dropboxAccessToken", ACCESS_TOKEN);
    // Create workspaces
    WorkspaceConfig rootWorkspace = new WorkspaceConfig("/", false, null, false);
    WorkspaceConfig csvWorkspace = new WorkspaceConfig("/csv", false, null, false);
    Map<String, WorkspaceConfig> workspaces = new HashMap<>();
    workspaces.put("root", rootWorkspace);
    workspaces.put("csv", csvWorkspace);
    // Add formats
    Map<String, FormatPluginConfig> formats = new HashMap<>();
    List<String> jsonExtensions = new ArrayList<>();
    jsonExtensions.add("json");
    FormatPluginConfig jsonFormatConfig = new JSONFormatConfig(jsonExtensions);
    // CSV Format
    List<String> csvExtensions = new ArrayList<>();
    csvExtensions.add("csv");
    csvExtensions.add("csvh");
    FormatPluginConfig csvFormatConfig = new TextFormatConfig(csvExtensions, "\n", ",", "\"", null, null, false, true);
    StoragePluginConfig dropboxConfig = new FileSystemConfig("dropbox:///", dropboxConfigVars, workspaces, formats, null);
    dropboxConfig.setEnabled(true);
    cluster.defineStoragePlugin("dropbox_test", dropboxConfig);
    cluster.defineFormat("dropbox_test", "json", jsonFormatConfig);
    cluster.defineFormat("dropbox_test", "csv", csvFormatConfig);
}