Examples with StoragePlugins org.apache.drill.exec.planner.logical.StoragePlugins used on opensource projects

Example 6 with StoragePlugins

use of org.apache.drill.exec.planner.logical.StoragePlugins in project drill by apache.

the class TestBootstrapLoader method testMissingBootstrapUpgrades.

@Test
public void testMissingBootstrapUpgrades() throws Exception {
    OperatorFixture.Builder builder = OperatorFixture.builder(dirTestWatcher);
    // Note: file does not actually exist, which is intentional.
    String bootstrapFile = RESOURCE_BASE + "missing-plugin-upgrade.json";
    builder.configBuilder().put(ExecConstants.UPGRADE_STORAGE_PLUGINS_FILE, bootstrapFile);
    try (OperatorFixture fixture = builder.build()) {
        PluginRegistryContextFixture context = new PluginRegistryContextFixture(fixture);
        PluginBootstrapLoader loader = new PluginBootstrapLoaderImpl(context);
        StoragePlugins plugins = loader.updatedPlugins();
        assertNull(plugins);
    }
}

Example 7 with StoragePlugins

use of org.apache.drill.exec.planner.logical.StoragePlugins in project drill by apache.

the class TestBootstrapLoader method testBootstrapLoader.

@Test
public void testBootstrapLoader() throws Exception {
    try (OperatorFixture fixture = OperatorFixture.standardFixture(dirTestWatcher)) {
        PluginRegistryContextFixture context = new PluginRegistryContextFixture(fixture);
        PluginBootstrapLoaderImpl loader = new PluginBootstrapLoaderImpl(context);
        Map<String, URL> pluginURLMap = new HashMap<>();
        StoragePlugins plugins = loader.loadBootstrapPlugins(pluginURLMap);
        // Sanity test. Change this if the bootstrap file changes.
        // No need to test contents; here we assume serialization works.
        // See FormatPluginSerDeTest
        assertNotNull(plugins.getConfig("dfs"));
        assertNotNull(plugins.getConfig("s3"));
        assertNotNull(plugins.getConfig("cp"));
    // Cannot test contrib plugins here: they are not yet
    // available when this test is run. We'll trust the
    // classpath scanner.
    }
}

Example 8 with StoragePlugins

use of org.apache.drill.exec.planner.logical.StoragePlugins in project drill by apache.

the class StoragePluginRegistryImpl method createPlugins.

@SuppressWarnings("resource")
private Map<String, StoragePlugin> createPlugins() throws DrillbitStartupException {
    try {
        /*
       * Check if the storage plugins system table has any entries. If not, load the boostrap-storage-plugin file into
       * the system table.
       */
        if (!pluginSystemTable.getAll().hasNext()) {
            // bootstrap load the config since no plugins are stored.
            logger.info("No storage plugin instances configured in persistent store, loading bootstrap configuration.");
            Collection<URL> urls = ClassPathScanner.forResource(ExecConstants.BOOTSTRAP_STORAGE_PLUGINS_FILE, false);
            if (urls != null && !urls.isEmpty()) {
                logger.info("Loading the storage plugin configs from URLs {}.", urls);
                Map<String, URL> pluginURLMap = Maps.newHashMap();
                for (URL url : urls) {
                    String pluginsData = Resources.toString(url, Charsets.UTF_8);
                    StoragePlugins plugins = lpPersistence.getMapper().readValue(pluginsData, StoragePlugins.class);
                    for (Map.Entry<String, StoragePluginConfig> config : plugins) {
                        if (!definePluginConfig(config.getKey(), config.getValue())) {
                            logger.warn("Duplicate plugin instance '{}' defined in [{}, {}], ignoring the later one.", config.getKey(), pluginURLMap.get(config.getKey()), url);
                            continue;
                        }
                        pluginURLMap.put(config.getKey(), url);
                    }
                }
            } else {
                throw new IOException("Failure finding " + ExecConstants.BOOTSTRAP_STORAGE_PLUGINS_FILE);
            }
        }
        Map<String, StoragePlugin> activePlugins = new HashMap<String, StoragePlugin>();
        for (Map.Entry<String, StoragePluginConfig> entry : Lists.newArrayList(pluginSystemTable.getAll())) {
            String name = entry.getKey();
            StoragePluginConfig config = entry.getValue();
            if (config.isEnabled()) {
                try {
                    StoragePlugin plugin = create(name, config);
                    activePlugins.put(name, plugin);
                } catch (ExecutionSetupException e) {
                    logger.error("Failure while setting up StoragePlugin with name: '{}', disabling.", name, e);
                    config.setEnabled(false);
                    pluginSystemTable.put(name, config);
                }
            }
        }
        activePlugins.put(INFORMATION_SCHEMA_PLUGIN, new InfoSchemaStoragePlugin(new InfoSchemaConfig(), context, INFORMATION_SCHEMA_PLUGIN));
        activePlugins.put(SYS_PLUGIN, new SystemTablePlugin(SystemTablePluginConfig.INSTANCE, context, SYS_PLUGIN));
        return activePlugins;
    } catch (IOException e) {
        logger.error("Failure setting up storage plugins.  Drillbit exiting.", e);
        throw new IllegalStateException(e);
    }
}

Example 9 with StoragePlugins

use of org.apache.drill.exec.planner.logical.StoragePlugins in project drill by axbaretto.

the class StoragePluginRegistryImpl method createPlugins.

@SuppressWarnings("resource")
private Map<String, StoragePlugin> createPlugins() throws DrillbitStartupException {
    try {
        /*
       * Check if the storage plugins system table has any entries. If not, load the boostrap-storage-plugin file into
       * the system table.
       */
        if (!pluginSystemTable.getAll().hasNext()) {
            // bootstrap load the config since no plugins are stored.
            logger.info("No storage plugin instances configured in persistent store, loading bootstrap configuration.");
            Collection<URL> urls = ClassPathScanner.forResource(ExecConstants.BOOTSTRAP_STORAGE_PLUGINS_FILE, false);
            if (urls != null && !urls.isEmpty()) {
                logger.info("Loading the storage plugin configs from URLs {}.", urls);
                Map<String, URL> pluginURLMap = Maps.newHashMap();
                for (URL url : urls) {
                    String pluginsData = Resources.toString(url, Charsets.UTF_8);
                    StoragePlugins plugins = lpPersistence.getMapper().readValue(pluginsData, StoragePlugins.class);
                    for (Map.Entry<String, StoragePluginConfig> config : plugins) {
                        if (!definePluginConfig(config.getKey(), config.getValue())) {
                            logger.warn("Duplicate plugin instance '{}' defined in [{}, {}], ignoring the later one.", config.getKey(), pluginURLMap.get(config.getKey()), url);
                            continue;
                        }
                        pluginURLMap.put(config.getKey(), url);
                    }
                }
            } else {
                throw new IOException("Failure finding " + ExecConstants.BOOTSTRAP_STORAGE_PLUGINS_FILE);
            }
        }
        Map<String, StoragePlugin> activePlugins = new HashMap<String, StoragePlugin>();
        for (Map.Entry<String, StoragePluginConfig> entry : Lists.newArrayList(pluginSystemTable.getAll())) {
            String name = entry.getKey();
            StoragePluginConfig config = entry.getValue();
            if (config.isEnabled()) {
                try {
                    StoragePlugin plugin = create(name, config);
                    activePlugins.put(name, plugin);
                } catch (ExecutionSetupException e) {
                    logger.error("Failure while setting up StoragePlugin with name: '{}', disabling.", name, e);
                    config.setEnabled(false);
                    pluginSystemTable.put(name, config);
                }
            }
        }
        activePlugins.put(INFORMATION_SCHEMA_PLUGIN, new InfoSchemaStoragePlugin(new InfoSchemaConfig(), context, INFORMATION_SCHEMA_PLUGIN));
        activePlugins.put(SYS_PLUGIN, new SystemTablePlugin(SystemTablePluginConfig.INSTANCE, context, SYS_PLUGIN));
        return activePlugins;
    } catch (IOException e) {
        logger.error("Failure setting up storage plugins.  Drillbit exiting.", e);
        throw new IllegalStateException(e);
    }
}

Example 10 with StoragePlugins

use of org.apache.drill.exec.planner.logical.StoragePlugins in project drill by apache.

the class TestBootstrapLoader method testDuplicateBootstrapEntries.

@Test
public void testDuplicateBootstrapEntries() throws Exception {
    OperatorFixture.Builder builder = OperatorFixture.builder(dirTestWatcher);
    String bootstrapFile = RESOURCE_BASE + "dup-bootstrap.json";
    builder.configBuilder().put(ExecConstants.BOOTSTRAP_STORAGE_PLUGINS_FILE, bootstrapFile);
    try (OperatorFixture fixture = builder.build()) {
        PluginRegistryContextFixture context = new PluginRegistryContextFixture(fixture);
        PluginBootstrapLoaderImpl loader = new PluginBootstrapLoaderImpl(context);
        StoragePlugins plugins = loader.loadBootstrapPlugins(new HashMap<>());
        // Duplicates noted in log; last one wins.
        StoragePluginConfig pluginConfig = plugins.getConfig("cp");
        assertNotNull(pluginConfig);
        assertTrue(pluginConfig instanceof FileSystemConfig);
        FileSystemConfig cpConfig = (FileSystemConfig) pluginConfig;
        assertNotNull(cpConfig.getFormats().get("tsv"));
    }
}