Examples with StoragePlugins org.apache.drill.exec.planner.logical.StoragePlugins used on opensource projects

Example 1 with StoragePlugins

use of org.apache.drill.exec.planner.logical.StoragePlugins in project drill by apache.

the class PluginBootstrapLoaderImpl method bootstrapPlugins.

@Override
public StoragePlugins bootstrapPlugins() throws IOException {
    Map<String, URL> pluginURLMap = new HashMap<>();
    StoragePlugins bootstrapPlugins = loadBootstrapPlugins(pluginURLMap);
    // Upgrade the bootstrap plugins with any updates. Seems odd,
    // but this is how Drill 1.17 works, so keeping the code.
    // Uses the enabled status from the updates if different than
    // the bootstrap version.
    StoragePlugins updatedPlugins = updatedPlugins();
    if (updatedPlugins != null) {
        bootstrapPlugins.putAll(updatedPlugins);
    }
    applyFormatPlugins(bootstrapPlugins, pluginURLMap);
    return bootstrapPlugins;
}

Example 2 with StoragePlugins

use of org.apache.drill.exec.planner.logical.StoragePlugins in project drill by apache.

the class PluginBootstrapLoaderImpl method loadBootstrapPlugins.

@VisibleForTesting
protected StoragePlugins loadBootstrapPlugins(Map<String, URL> pluginURLMap) throws IOException {
    // bootstrap load the config since no plugins are stored.
    String storageBootstrapFileName = context.config().getString(ExecConstants.BOOTSTRAP_STORAGE_PLUGINS_FILE);
    Set<URL> storageUrls = ClassPathScanner.forResource(storageBootstrapFileName, false);
    if (storageUrls == null || storageUrls.isEmpty()) {
        throw new IOException("Cannot find storage plugin boostrap file: " + storageBootstrapFileName);
    }
    logger.info("Loading the storage plugin configs from URLs {}.", storageUrls);
    StoragePlugins bootstrapPlugins = new StoragePlugins();
    for (URL url : storageUrls) {
        try {
            loadStoragePlugins(url, bootstrapPlugins, pluginURLMap);
        } catch (IOException e) {
            throw new IOException("Failed to load bootstrap plugins from " + url.toString(), e);
        }
    }
    return bootstrapPlugins;
}

Example 3 with StoragePlugins

use of org.apache.drill.exec.planner.logical.StoragePlugins in project drill by apache.

the class PluginBootstrapLoaderImpl method loadFormatPlugins.

/**
 * Loads format plugins from the given URL and adds the formats to the
 * specified storage plugins
 *
 * @param url
 *          URL to the format plugins bootstrap file
 * @param bootstrapPlugins
 *          a collection with loaded storage plugins. New formats will be
 *          added to them
 * @param pluginURLMap
 *          a map to store correspondence between storage plugins and
 *          bootstrap files in which they are defined. Used for logging
 * @throws IOException
 *           if failed to retrieve a plugin from a bootstrap file
 */
private void loadFormatPlugins(URL url, StoragePlugins bootstrapPlugins, Map<String, URL> pluginURLMap) throws IOException {
    StoragePlugins plugins = getPluginsFromResource(url);
    for (Entry<String, StoragePluginConfig> sourceEntry : plugins) {
        String pluginName = sourceEntry.getKey();
        StoragePluginConfig sourcePlugin = sourceEntry.getValue();
        if (!(sourcePlugin instanceof FileSystemConfig)) {
            logger.warn("Formats are only supported by File System plugins. Source name '{}' is of type '{}'.", pluginName, sourcePlugin.getClass().getName());
            continue;
        }
        StoragePluginConfig targetPlugin = bootstrapPlugins.getConfig(pluginName);
        if (targetPlugin == null) {
            logger.warn("No boostrap storage plugin matches the name '{}'", pluginName);
            continue;
        }
        if (!(targetPlugin instanceof FileSystemConfig)) {
            logger.warn("Formats are only supported by File System plugins. Source name '{}' " + "is of type '{}' but the bootstrap plugin of that name is of type '{}.", pluginName, sourcePlugin.getClass().getName(), targetPlugin.getClass().getName());
            continue;
        }
        FileSystemConfig targetFsConfig = (FileSystemConfig) targetPlugin;
        FileSystemConfig sourceFsConfig = (FileSystemConfig) sourcePlugin;
        sourceFsConfig.getFormats().forEach((formatName, formatValue) -> {
            FormatPluginConfig oldPluginConfig = targetFsConfig.getFormats().putIfAbsent(formatName, formatValue);
            if (oldPluginConfig != null) {
                logger.warn("Duplicate format instance '{}' defined in '{}' and '{}', ignoring the later one.", formatName, pluginURLMap.get(pluginName), url);
            }
        });
    }
}

Example 4 with StoragePlugins

use of org.apache.drill.exec.planner.logical.StoragePlugins in project drill by apache.

the class PluginBootstrapLoaderImpl method loadStoragePlugins.

/**
 * Loads storage plugins from the given URL
 *
 * @param url
 *          URL to the storage plugins bootstrap file
 * @param bootstrapPlugins
 *          a collection where the plugins should be loaded to
 * @param pluginURLMap
 *          a map to store correspondence between storage plugins and
 *          bootstrap files in which they are defined. Used for logging
 * @throws IOException
 *           if failed to retrieve a plugin from a bootstrap file
 */
private void loadStoragePlugins(URL url, StoragePlugins bootstrapPlugins, Map<String, URL> pluginURLMap) throws IOException {
    StoragePlugins plugins = getPluginsFromResource(url);
    plugins.forEach(plugin -> {
        StoragePluginConfig oldPluginConfig = bootstrapPlugins.putIfAbsent(plugin.getKey(), plugin.getValue());
        if (oldPluginConfig != null) {
            logger.warn("Duplicate plugin instance '[{}]' defined in [{}, {}], ignoring the later one.", plugin.getKey(), pluginURLMap.get(plugin.getKey()), url);
        } else {
            pluginURLMap.put(plugin.getKey(), url);
        }
    });
}

Example 5 with StoragePlugins

use of org.apache.drill.exec.planner.logical.StoragePlugins in project drill by apache.

the class TestClassicLocator method testBootstrap.

@Test
public void testBootstrap() throws Exception {
    try (OperatorFixture fixture = OperatorFixture.standardFixture(dirTestWatcher)) {
        PluginRegistryContextFixture context = new PluginRegistryContextFixture(fixture);
        ConnectorLocator locator = new ClassicConnectorLocator(context);
        locator.init();
        StoragePlugins plugins = locator.bootstrapPlugins();
        // Sanity test. Change this if the bootstrap file changes.
        // No need to test contents; here we assume serialization works.
        // See FormatPluginSerDeTest
        assertNotNull(plugins.getConfig("dfs"));
        assertNotNull(plugins.getConfig("s3"));
        assertNotNull(plugins.getConfig(StoragePluginTestUtils.CP_PLUGIN_NAME));
        // No-op
        locator.close();
    }
}