Examples with StoragePluginConfig org.apache.drill.common.logical.StoragePluginConfig used on opensource projects

Example 11 with StoragePluginConfig

use of org.apache.drill.common.logical.StoragePluginConfig in project drill by apache.

the class TestPluginRegistry method testEnableWithPut.

/**
 * Tests the other way to enable/disabled plugins: make a **COPY** of the
 * config and set the enable/disable status. Note: race conditions happen
 * if a client modifies a stored config. Old code would do that, but the
 * results are undefined. Either use setEnabled(), or make a copy of the
 * config. This case also models where the user edits the config by hand
 * in the Web Console to disable the plugin: the deserialize/serialize
 * steps make the copy.
 */
@Test
public void testEnableWithPut() throws Exception {
    ClusterFixtureBuilder builder = ClusterFixture.builder(dirTestWatcher);
    try (ClusterFixture cluster = builder.build()) {
        StoragePluginRegistry registry = cluster.storageRegistry();
        FileSystemConfig pConfig1 = myConfig1();
        registry.put(MY_PLUGIN_NAME, pConfig1);
        assertTrue(registry.getStoredConfig(MY_PLUGIN_NAME).isEnabled());
        // Enable the same plugin using a different, but equal, config.
        // The original config remains in place.
        StoragePluginConfig pConfig2 = registry.copyConfig(MY_PLUGIN_NAME);
        pConfig2.setEnabled(true);
        assertEquals(pConfig1, pConfig2);
        registry.put(MY_PLUGIN_NAME, pConfig2);
        StoragePluginConfig savedConfig = registry.getStoredConfig(MY_PLUGIN_NAME);
        assertEquals(pConfig1, savedConfig);
        assertTrue(savedConfig.isEnabled());
        // Force resolution of the plugin so there is something to cache
        StoragePlugin plugin = registry.getPlugin(MY_PLUGIN_NAME);
        assertNotNull(plugin);
        // Disable an enabled plugin. The old plugin lives in ephemeral
        // store, but is not visible by name. If requested, the
        // registry obtains a new copy from persistent storage.
        StoragePluginConfig pConfig3 = registry.copyConfig(MY_PLUGIN_NAME);
        pConfig3.setEnabled(false);
        registry.put(MY_PLUGIN_NAME, pConfig3);
        savedConfig = registry.getStoredConfig(MY_PLUGIN_NAME);
        assertEquals(pConfig1, savedConfig);
        assertFalse(savedConfig.isEnabled());
        // OK to disable twice
        StoragePluginConfig pConfig4 = registry.copyConfig(MY_PLUGIN_NAME);
        pConfig4.setEnabled(false);
        registry.put(MY_PLUGIN_NAME, pConfig4);
        savedConfig = registry.getStoredConfig(MY_PLUGIN_NAME);
        assertEquals(pConfig1, savedConfig);
        assertFalse(savedConfig.isEnabled());
        // Disabled plugins appear in the stored config map
        Map<String, StoragePluginConfig> configMap = registry.storedConfigs();
        assertTrue(configMap.containsKey(MY_PLUGIN_KEY));
        assertEquals(pConfig3, configMap.get(MY_PLUGIN_KEY));
        // Re-enable, the original plugin instance reappears.
        StoragePluginConfig pConfig5 = registry.copyConfig(MY_PLUGIN_NAME);
        pConfig5.setEnabled(true);
        registry.put(MY_PLUGIN_NAME, pConfig5);
        assertSame(plugin, registry.getPlugin(MY_PLUGIN_NAME));
        assertTrue(plugin.getConfig().isEnabled());
    }
}

Example 12 with StoragePluginConfig

use of org.apache.drill.common.logical.StoragePluginConfig in project drill by apache.

the class TestPluginRegistry method testLifecycle.

@Test
public void testLifecycle() throws Exception {
    ClusterFixtureBuilder builder = ClusterFixture.builder(dirTestWatcher);
    try (ClusterFixture cluster = builder.build()) {
        StoragePluginRegistry registry = cluster.storageRegistry();
        // Bootstrap file loaded.
        // Normal
        assertNotNull(registry.getPlugin(CP_PLUGIN_NAME));
        // System
        assertNotNull(registry.getPlugin(SYS_PLUGIN_NAME));
        // Not editable
        assertNull(registry.getStoredConfig(SYS_PLUGIN_NAME));
        assertNull(registry.getPlugin("bogus"));
        // Enabled plugins
        Map<String, StoragePluginConfig> configMap = registry.enabledConfigs();
        assertTrue(configMap.containsKey(CP_PLUGIN_NAME));
        // Disabled, but still appears
        assertFalse(configMap.containsKey(S3_PLUGIN_NAME));
        assertFalse(configMap.containsKey(SYS_PLUGIN_NAME));
        assertNotNull(registry.getDefinedConfig(CP_PLUGIN_NAME));
        assertNull(registry.getDefinedConfig(S3_PLUGIN_NAME));
        assertNotNull(registry.getDefinedConfig(SYS_PLUGIN_NAME));
        // All stored plugins, including disabled
        configMap = registry.storedConfigs();
        assertTrue(configMap.containsKey(CP_PLUGIN_NAME));
        // Disabled, but still appears
        assertTrue(configMap.containsKey(S3_PLUGIN_NAME));
        assertNotNull(configMap.get(S3_PLUGIN_NAME));
        assertSame(registry.getStoredConfig(S3_PLUGIN_NAME), configMap.get(S3_PLUGIN_NAME));
        assertFalse(configMap.containsKey(SYS_PLUGIN_NAME));
        int bootstrapCount = configMap.size();
        // Enabled only
        configMap = registry.storedConfigs(PluginFilter.ENABLED);
        assertTrue(configMap.containsKey(CP_PLUGIN_NAME));
        assertFalse(configMap.containsKey(S3_PLUGIN_NAME));
        // Disabled only
        configMap = registry.storedConfigs(PluginFilter.DISABLED);
        assertFalse(configMap.containsKey(CP_PLUGIN_NAME));
        assertTrue(configMap.containsKey(S3_PLUGIN_NAME));
        // Create a new plugin
        FileSystemConfig pConfig1 = myConfig1();
        registry.put(MY_PLUGIN_NAME, pConfig1);
        StoragePlugin plugin1 = registry.getPlugin(MY_PLUGIN_NAME);
        assertNotNull(plugin1);
        assertSame(plugin1, registry.getPluginByConfig(pConfig1));
        configMap = registry.storedConfigs();
        // Names converted to lowercase in persistent storage
        assertTrue(configMap.containsKey(MY_PLUGIN_KEY));
        assertEquals(bootstrapCount + 1, configMap.size());
        // Names are case-insensitive
        assertSame(plugin1, registry.getPlugin(MY_PLUGIN_KEY));
        assertSame(plugin1, registry.getPlugin(MY_PLUGIN_NAME.toUpperCase()));
        // Update the plugin
        FileSystemConfig pConfig2 = myConfig2();
        registry.put(MY_PLUGIN_NAME, pConfig2);
        StoragePlugin plugin2 = registry.getPlugin(MY_PLUGIN_NAME);
        assertNotSame(plugin1, plugin2);
        assertTrue(plugin2 instanceof FileSystemPlugin);
        FileSystemPlugin fsStorage = (FileSystemPlugin) plugin2;
        assertSame(pConfig2, fsStorage.getConfig());
        assertSame(plugin2, registry.getPluginByConfig(pConfig2));
        // Cannot create/update a plugin with null or blank name
        FileSystemConfig pConfig3 = myConfig1();
        try {
            registry.put(null, pConfig3);
            fail();
        } catch (PluginException e) {
        // Expected
        }
        try {
            registry.put("  ", pConfig3);
            fail();
        } catch (PluginException e) {
        // Expected
        }
    }
}

Example 13 with StoragePluginConfig

use of org.apache.drill.common.logical.StoragePluginConfig in project drill by apache.

the class StoragePluginRegistryImpl method createPlugins.

@SuppressWarnings("resource")
private Map<String, StoragePlugin> createPlugins() throws DrillbitStartupException {
    try {
        /*
       * Check if the storage plugins system table has any entries. If not, load the boostrap-storage-plugin file into
       * the system table.
       */
        if (!pluginSystemTable.getAll().hasNext()) {
            // bootstrap load the config since no plugins are stored.
            logger.info("No storage plugin instances configured in persistent store, loading bootstrap configuration.");
            Collection<URL> urls = ClassPathScanner.forResource(ExecConstants.BOOTSTRAP_STORAGE_PLUGINS_FILE, false);
            if (urls != null && !urls.isEmpty()) {
                logger.info("Loading the storage plugin configs from URLs {}.", urls);
                Map<String, URL> pluginURLMap = Maps.newHashMap();
                for (URL url : urls) {
                    String pluginsData = Resources.toString(url, Charsets.UTF_8);
                    StoragePlugins plugins = lpPersistence.getMapper().readValue(pluginsData, StoragePlugins.class);
                    for (Map.Entry<String, StoragePluginConfig> config : plugins) {
                        if (!definePluginConfig(config.getKey(), config.getValue())) {
                            logger.warn("Duplicate plugin instance '{}' defined in [{}, {}], ignoring the later one.", config.getKey(), pluginURLMap.get(config.getKey()), url);
                            continue;
                        }
                        pluginURLMap.put(config.getKey(), url);
                    }
                }
            } else {
                throw new IOException("Failure finding " + ExecConstants.BOOTSTRAP_STORAGE_PLUGINS_FILE);
            }
        }
        Map<String, StoragePlugin> activePlugins = new HashMap<String, StoragePlugin>();
        for (Map.Entry<String, StoragePluginConfig> entry : Lists.newArrayList(pluginSystemTable.getAll())) {
            String name = entry.getKey();
            StoragePluginConfig config = entry.getValue();
            if (config.isEnabled()) {
                try {
                    StoragePlugin plugin = create(name, config);
                    activePlugins.put(name, plugin);
                } catch (ExecutionSetupException e) {
                    logger.error("Failure while setting up StoragePlugin with name: '{}', disabling.", name, e);
                    config.setEnabled(false);
                    pluginSystemTable.put(name, config);
                }
            }
        }
        activePlugins.put(INFORMATION_SCHEMA_PLUGIN, new InfoSchemaStoragePlugin(new InfoSchemaConfig(), context, INFORMATION_SCHEMA_PLUGIN));
        activePlugins.put(SYS_PLUGIN, new SystemTablePlugin(SystemTablePluginConfig.INSTANCE, context, SYS_PLUGIN));
        return activePlugins;
    } catch (IOException e) {
        logger.error("Failure setting up storage plugins.  Drillbit exiting.", e);
        throw new IllegalStateException(e);
    }
}

Example 14 with StoragePluginConfig

use of org.apache.drill.common.logical.StoragePluginConfig in project drill by apache.

the class StoragePluginRegistryImpl method getPlugin.

@Override
public StoragePlugin getPlugin(String name) throws ExecutionSetupException {
    StoragePlugin plugin = plugins.get(name);
    if (name.equals(SYS_PLUGIN) || name.equals(INFORMATION_SCHEMA_PLUGIN)) {
        return plugin;
    }
    // since we lazily manage the list of plugins per server, we need to update this once we know that it is time.
    StoragePluginConfig config = this.pluginSystemTable.get(name);
    if (config == null) {
        if (plugin != null) {
            plugins.remove(name);
        }
        return null;
    } else {
        if (plugin == null || !plugin.getConfig().equals(config) || plugin.getConfig().isEnabled() != config.isEnabled()) {
            plugin = createOrUpdate(name, config, false);
        }
        return plugin;
    }
}

Example 15 with StoragePluginConfig

use of org.apache.drill.common.logical.StoragePluginConfig in project drill by axbaretto.

the class StoragePluginRegistryImpl method createPlugins.

@SuppressWarnings("resource")
private Map<String, StoragePlugin> createPlugins() throws DrillbitStartupException {
    try {
        /*
       * Check if the storage plugins system table has any entries. If not, load the boostrap-storage-plugin file into
       * the system table.
       */
        if (!pluginSystemTable.getAll().hasNext()) {
            // bootstrap load the config since no plugins are stored.
            logger.info("No storage plugin instances configured in persistent store, loading bootstrap configuration.");
            Collection<URL> urls = ClassPathScanner.forResource(ExecConstants.BOOTSTRAP_STORAGE_PLUGINS_FILE, false);
            if (urls != null && !urls.isEmpty()) {
                logger.info("Loading the storage plugin configs from URLs {}.", urls);
                Map<String, URL> pluginURLMap = Maps.newHashMap();
                for (URL url : urls) {
                    String pluginsData = Resources.toString(url, Charsets.UTF_8);
                    StoragePlugins plugins = lpPersistence.getMapper().readValue(pluginsData, StoragePlugins.class);
                    for (Map.Entry<String, StoragePluginConfig> config : plugins) {
                        if (!definePluginConfig(config.getKey(), config.getValue())) {
                            logger.warn("Duplicate plugin instance '{}' defined in [{}, {}], ignoring the later one.", config.getKey(), pluginURLMap.get(config.getKey()), url);
                            continue;
                        }
                        pluginURLMap.put(config.getKey(), url);
                    }
                }
            } else {
                throw new IOException("Failure finding " + ExecConstants.BOOTSTRAP_STORAGE_PLUGINS_FILE);
            }
        }
        Map<String, StoragePlugin> activePlugins = new HashMap<String, StoragePlugin>();
        for (Map.Entry<String, StoragePluginConfig> entry : Lists.newArrayList(pluginSystemTable.getAll())) {
            String name = entry.getKey();
            StoragePluginConfig config = entry.getValue();
            if (config.isEnabled()) {
                try {
                    StoragePlugin plugin = create(name, config);
                    activePlugins.put(name, plugin);
                } catch (ExecutionSetupException e) {
                    logger.error("Failure while setting up StoragePlugin with name: '{}', disabling.", name, e);
                    config.setEnabled(false);
                    pluginSystemTable.put(name, config);
                }
            }
        }
        activePlugins.put(INFORMATION_SCHEMA_PLUGIN, new InfoSchemaStoragePlugin(new InfoSchemaConfig(), context, INFORMATION_SCHEMA_PLUGIN));
        activePlugins.put(SYS_PLUGIN, new SystemTablePlugin(SystemTablePluginConfig.INSTANCE, context, SYS_PLUGIN));
        return activePlugins;
    } catch (IOException e) {
        logger.error("Failure setting up storage plugins.  Drillbit exiting.", e);
        throw new IllegalStateException(e);
    }
}