Examples with StoragePlugins org.apache.drill.exec.planner.logical.StoragePlugins used on opensource projects

Example 11 with StoragePlugins

use of org.apache.drill.exec.planner.logical.StoragePlugins in project drill by apache.

the class TestBootstrapLoader method testBootstrapLoaderWithFormats.

/**
 * Test a format bootstrap with a mock file. Can't use a real
 * file because those appear in modules not yet available when
 * this test runs.
 */
@Test
public void testBootstrapLoaderWithFormats() throws Exception {
    OperatorFixture.Builder builder = OperatorFixture.builder(dirTestWatcher);
    String bootstrapFile = RESOURCE_BASE + "mock-format-bootstrap.json";
    builder.configBuilder().put(ExecConstants.BOOTSTRAP_FORMAT_PLUGINS_FILE, bootstrapFile);
    try (OperatorFixture fixture = builder.build()) {
        PluginRegistryContextFixture context = new PluginRegistryContextFixture(fixture);
        PluginBootstrapLoader loader = new PluginBootstrapLoaderImpl(context);
        StoragePlugins plugins = loader.bootstrapPlugins();
        // bsv added to dfs
        StoragePluginConfig pluginConfig = plugins.getConfig("dfs");
        assertNotNull(pluginConfig);
        FileSystemConfig dfs = (FileSystemConfig) pluginConfig;
        assertNotNull(dfs.getFormats().get("bsv"));
    }
}

Example 12 with StoragePlugins

use of org.apache.drill.exec.planner.logical.StoragePlugins in project drill by apache.

the class TestBootstrapLoader method testBootstrapLoaderWithUpgrades.

@Test
public void testBootstrapLoaderWithUpgrades() throws Exception {
    OperatorFixture.Builder builder = OperatorFixture.builder(dirTestWatcher);
    String bootstrapFile = RESOURCE_BASE + "mock-plugin-upgrade.json";
    builder.configBuilder().put(ExecConstants.UPGRADE_STORAGE_PLUGINS_FILE, bootstrapFile);
    try (OperatorFixture fixture = builder.build()) {
        PluginRegistryContextFixture context = new PluginRegistryContextFixture(fixture);
        PluginBootstrapLoader loader = new PluginBootstrapLoaderImpl(context);
        StoragePlugins plugins = loader.bootstrapPlugins();
        // dfs: removed psv
        StoragePluginConfig pluginConfig = plugins.getConfig("dfs");
        assertNotNull(pluginConfig);
        FileSystemConfig dfs = (FileSystemConfig) pluginConfig;
        assertNull(dfs.getFormats().get("psv"));
        assertNotNull(dfs.getFormats().get("csv"));
        // local added
        assertNotNull(plugins.getConfig("local"));
        // S3, bsv added
        pluginConfig = plugins.getConfig("s3");
        assertNotNull(pluginConfig);
        FileSystemConfig s3 = (FileSystemConfig) pluginConfig;
        assertNotNull(s3.getFormats().get("bsv"));
        // cp, left unchanged (not in upgrade file)
        assertNotNull(plugins.getConfig("cp"));
    }
}

Example 13 with StoragePlugins

use of org.apache.drill.exec.planner.logical.StoragePlugins in project drill by apache.

the class TestBootstrapLoader method testBootstrapUpgrades.

@Test
public void testBootstrapUpgrades() throws Exception {
    OperatorFixture.Builder builder = OperatorFixture.builder(dirTestWatcher);
    String bootstrapFile = RESOURCE_BASE + "mock-plugin-upgrade.json";
    builder.configBuilder().put(ExecConstants.UPGRADE_STORAGE_PLUGINS_FILE, bootstrapFile);
    try (OperatorFixture fixture = builder.build()) {
        PluginRegistryContextFixture context = new PluginRegistryContextFixture(fixture);
        PluginBootstrapLoader loader = new PluginBootstrapLoaderImpl(context);
        StoragePlugins plugins = loader.updatedPlugins();
        assertNotNull(plugins);
        // dfs: removed psv
        StoragePluginConfig pluginConfig = plugins.getConfig("dfs");
        assertNotNull(pluginConfig);
        FileSystemConfig dfs = (FileSystemConfig) pluginConfig;
        assertNull(dfs.getFormats().get("psv"));
        assertNotNull(dfs.getFormats().get("csv"));
        // local added
        assertNotNull(plugins.getConfig("local"));
        // S3, bsv added
        pluginConfig = plugins.getConfig("s3");
        assertNotNull(pluginConfig);
        FileSystemConfig s3 = (FileSystemConfig) pluginConfig;
        assertNotNull(s3.getFormats().get("bsv"));
        // cp, left unchanged (not in upgrade file)
        assertNull(plugins.getConfig("cp"));
    }
}

Example 14 with StoragePlugins

use of org.apache.drill.exec.planner.logical.StoragePlugins in project drill by apache.

the class StoragePluginRegistryImpl method initStore.

private void initStore() {
    logger.info("No storage plugin instances configured in persistent store, loading bootstrap configuration.");
    StoragePlugins bootstrapPlugins = new StoragePlugins();
    try {
        for (ConnectorLocator locator : locators) {
            StoragePlugins locatorPlugins = locator.bootstrapPlugins();
            if (locatorPlugins != null) {
                bootstrapPlugins.putAll(locatorPlugins);
            }
        }
    } catch (IOException e) {
        throw new IllegalStateException("Failure initializing the plugin store. Drillbit exiting.", e);
    }
    pluginStore.putAll(bootstrapPlugins);
    locators.stream().forEach(loc -> loc.onUpgrade());
}

Example 15 with StoragePlugins

use of org.apache.drill.exec.planner.logical.StoragePlugins in project drill by apache.

the class StoragePluginRegistryImpl method upgradeStore.

/**
 * Upgrade an existing persistent plugin config store with
 * updates available from each locator.
 */
private void upgradeStore() {
    StoragePlugins upgraded = new StoragePlugins();
    for (ConnectorLocator locator : locators) {
        StoragePlugins locatorPlugins = locator.updatedPlugins();
        if (upgraded != null) {
            upgraded.putAll(locatorPlugins);
        }
    }
    if (upgraded.isEmpty()) {
        return;
    }
    for (Map.Entry<String, StoragePluginConfig> newPlugin : upgraded) {
        StoragePluginConfig oldPluginConfig = getStoredConfig(newPlugin.getKey());
        if (oldPluginConfig != null) {
            copyPluginStatus(oldPluginConfig, newPlugin.getValue());
        }
        pluginStore.put(newPlugin.getKey(), newPlugin.getValue());
    }
}