Examples with ClusterFixtureBuilder org.apache.drill.test.ClusterFixtureBuilder used on opensource projects

Example 96 with ClusterFixtureBuilder

use of org.apache.drill.test.ClusterFixtureBuilder in project drill by apache.

the class TestPluginRegistry method testStoreSync.

@Test
public void testStoreSync() throws Exception {
    ClusterFixtureBuilder builder = ClusterFixture.builder(dirTestWatcher).withBits("bit1", "bit2");
    // We want a non-buffered, local file system store, in a known location
    // so that the two Drillbits will coordinate roughly he same way they
    // will when using the ZK store in distributed mode.
    builder.configBuilder().put(ExecConstants.SYS_STORE_PROVIDER_LOCAL_ENABLE_WRITE, true).put(ExecConstants.SYS_STORE_PROVIDER_LOCAL_PATH, dirTestWatcher.getStoreDir().getAbsolutePath());
    try (ClusterFixture cluster = builder.build()) {
        StoragePluginRegistry registry1 = cluster.storageRegistry("bit1");
        StoragePluginRegistry registry2 = cluster.storageRegistry("bit2");
        // Define a plugin in Drillbit 1
        FileSystemConfig pConfig1 = myConfig1();
        registry1.put(MY_PLUGIN_NAME, pConfig1);
        StoragePlugin plugin1 = registry1.getPlugin(MY_PLUGIN_NAME);
        assertNotNull(plugin1);
        // Should appear in Drillbit 2
        assertTrue(registry2.storedConfigs().containsKey(MY_PLUGIN_KEY));
        StoragePlugin plugin2 = registry2.getPlugin(MY_PLUGIN_NAME);
        assertNotNull(plugin2);
        assertEquals(pConfig1, plugin2.getConfig());
        // Change in Drillbit 1
        FileSystemConfig pConfig3 = myConfig2();
        registry1.put(MY_PLUGIN_NAME, pConfig3);
        plugin1 = registry1.getPlugin(MY_PLUGIN_NAME);
        assertEquals(pConfig3, plugin1.getConfig());
        // Change should appear in Drillbit 2
        assertTrue(registry2.storedConfigs().containsValue(pConfig3));
        plugin2 = registry2.getPlugin(MY_PLUGIN_NAME);
        assertNotNull(plugin2);
        assertEquals(pConfig3, plugin1.getConfig());
        // Delete in Drillbit 2
        registry2.remove(MY_PLUGIN_NAME);
        // Should not be available in Drillbit 1
        assertFalse(registry1.storedConfigs().containsKey(MY_PLUGIN_KEY));
        assertNull(registry1.getPlugin(MY_PLUGIN_NAME));
    }
}

Example 97 with ClusterFixtureBuilder

use of org.apache.drill.test.ClusterFixtureBuilder in project drill by apache.

the class TestPluginRegistry method testEphemeralLifecycle.

/**
 * Test the ephemeral cache where plugin instances live after their
 * configs are changed or disabled so that any running queries see
 * the prior plugin config an instance.
 * <p>
 * Note that each call to update a plugin must work with a copy of
 * a config. Results are undefined if a client changes a stored
 * config.
 */
@Test
public void testEphemeralLifecycle() throws Exception {
    ClusterFixtureBuilder builder = ClusterFixture.builder(dirTestWatcher);
    try (ClusterFixture cluster = builder.build()) {
        StoragePluginRegistry registry = cluster.storageRegistry();
        // Create a plugin
        FileSystemConfig pConfig1 = myConfig1();
        registry.put(MY_PLUGIN_NAME, pConfig1);
        StoragePlugin plugin1 = registry.getPlugin(MY_PLUGIN_NAME);
        // Update the plugin
        FileSystemConfig pConfig2 = myConfig2();
        registry.put(MY_PLUGIN_NAME, pConfig2);
        StoragePlugin plugin2 = registry.getPlugin(MY_PLUGIN_NAME);
        assertNotSame(plugin1, plugin2);
        assertTrue(plugin2 instanceof FileSystemPlugin);
        FileSystemPlugin fsStorage = (FileSystemPlugin) plugin2;
        assertSame(pConfig2, fsStorage.getConfig());
        assertSame(plugin2, registry.getPluginByConfig(pConfig2));
        // Suppose a query was planned with plugin1 and now starts
        // to execute. Plugin1 has been replaced with plugin2. However
        // the registry moved the old plugin to ephemeral storage where
        // it can still be found by configuration.
        FileSystemConfig pConfig1a = myConfig1();
        StoragePlugin ePlugin1 = registry.getPluginByConfig(pConfig1a);
        assertSame(plugin1, ePlugin1);
        assertNotSame(plugin2, ePlugin1);
        // Change the stored plugin back to the first config.
        FileSystemConfig pConfig1b = myConfig1();
        registry.put(MY_PLUGIN_NAME, pConfig1b);
        // Now, lets suppose thread 3 starts to execute. It sees the original plugin
        assertSame(plugin1, registry.getPlugin(MY_PLUGIN_NAME));
        // But, the ephemeral plugin lives on. Go back to the second
        // config.
        FileSystemConfig pConfig2b = myConfig2();
        registry.put(MY_PLUGIN_NAME, pConfig2b);
        assertSame(plugin2, registry.getPlugin(MY_PLUGIN_NAME));
        // Thread 4, using the first config from planning in thread 3,
        // still sees the first plugin.
        assertSame(plugin1, registry.getPluginByConfig(pConfig1b));
        // Disable
        registry.setEnabled(MY_PLUGIN_NAME, false);
        assertNull(registry.getPlugin(MY_PLUGIN_NAME));
        // Though disabled, a running query will create an ephemeral
        // plugin for the config.
        assertSame(plugin1, registry.getPluginByConfig(pConfig1b));
        assertSame(plugin2, registry.getPluginByConfig(pConfig2b));
        // Enable. We notice the config is in the ephemeral store and
        // so we restore it.
        registry.setEnabled(MY_PLUGIN_NAME, true);
        assertSame(plugin2, registry.getPlugin(MY_PLUGIN_NAME));
        assertSame(plugin2, registry.getPluginByConfig(pConfig2b));
        assertTrue(registry.storedConfigs().containsKey(MY_PLUGIN_KEY));
        assertTrue(registry.enabledConfigs().containsKey(MY_PLUGIN_KEY));
        // Delete the plugin
        registry.remove(MY_PLUGIN_NAME);
        assertNull(registry.getPlugin(MY_PLUGIN_NAME));
        // Again a running query will retrieve the plugin from ephemeral storage
        assertSame(plugin1, registry.getPluginByConfig(pConfig1));
        assertSame(plugin2, registry.getPluginByConfig(pConfig2));
        // Delete again, no-op
        registry.remove(MY_PLUGIN_NAME);
        // The retrieve-from-ephemeral does not kick in if we create
        // a new plugin with the same config but a different name.
        FileSystemConfig pConfig1c = myConfig1();
        pConfig1c.setEnabled(true);
        registry.put("alias", pConfig1c);
        StoragePlugin plugin4 = registry.getPlugin("alias");
        assertNotNull(plugin4);
        assertNotSame(plugin1, plugin4);
        // Delete the second name. The config is the same as one already
        // in ephemeral store, so the second is closed. The first will
        // be returned on subsequent queries.
        registry.remove("alias");
        assertNull(registry.getPlugin("alias"));
        assertSame(plugin1, registry.getPluginByConfig(pConfig1));
        // Try to change a system plugin
        StoragePlugin sysPlugin = registry.getPlugin(SYS_PLUGIN_NAME);
        assertNotNull(sysPlugin);
        FileSystemConfig pConfig3 = myConfig2();
        try {
            registry.put(SYS_PLUGIN_NAME, pConfig3);
            fail();
        } catch (PluginException e) {
        // Expected
        }
        pConfig3.setEnabled(false);
        try {
            registry.put(SYS_PLUGIN_NAME, pConfig3);
            fail();
        } catch (PluginException e) {
        // Expected
        }
        assertSame(sysPlugin, registry.getPlugin(SYS_PLUGIN_NAME));
        // Try to delete a system plugin
        try {
            registry.remove(SYS_PLUGIN_NAME);
            fail();
        } catch (PluginException e) {
        // Expected
        }
    }
}

Example 98 with ClusterFixtureBuilder

use of org.apache.drill.test.ClusterFixtureBuilder in project drill by apache.

the class TestPluginRegistry method testBadPlugin.

/**
 * Test to illustrate problems discussed in DRILL-7624
 */
@Test
public void testBadPlugin() throws Exception {
    ClusterFixtureBuilder builder = ClusterFixture.builder(dirTestWatcher);
    builder.configBuilder().put(ExecConstants.PRIVATE_CONNECTORS, Collections.singletonList(StoragePluginFixture.class.getName()));
    try (ClusterFixture cluster = builder.build()) {
        StoragePluginRegistry registry = cluster.storageRegistry();
        // Create a config that causes a crash because the plugin
        // is not created on update.
        StoragePluginFixtureConfig badConfig = new StoragePluginFixtureConfig("crash-ctor");
        badConfig.setEnabled(true);
        // instantiating the plugin.
        try {
            registry.validatedPut("bad", badConfig);
            fail();
        } catch (PluginException e) {
        // Expected
        }
        assertNull(registry.getStoredConfig("bad"));
        assertFalse(registry.availablePlugins().contains("bad"));
        // Try the same with JSON
        String json = registry.encode(badConfig);
        try {
            registry.putJson("bad", json);
            fail();
        } catch (PluginException e) {
        // Expected
        }
        assertFalse(registry.availablePlugins().contains("bad"));
        // Now, lets pretend the plugin was valid when we did the above,
        // but later the external system failed.
        registry.put("bad", badConfig);
        assertEquals(badConfig, registry.getStoredConfig("bad"));
        assertTrue(registry.availablePlugins().contains("bad"));
        // Ask for the actual plugin. Now will fail.
        try {
            registry.getPlugin("bad");
            fail();
        } catch (UserException e) {
            assertTrue(e.getMessage().contains("bad"));
        }
        assertTrue(registry.availablePlugins().contains("bad"));
        // No plugin created. Will fail the next time also.
        try {
            registry.getPlugin("bad");
            fail();
        } catch (UserException e) {
        // Expected
        }
        assertTrue(registry.availablePlugins().contains("bad"));
        // The iterator used to find planning rules will skip the failed
        // plugin. (That the planner uses all rules is, itself, a bug.)
        int n = registry.availablePlugins().size();
        int count = 0;
        for (@SuppressWarnings("unused") Entry<String, StoragePlugin> entry : registry) {
            count++;
        }
        assertEquals(n - 1, count);
        // Reset to known good state
        registry.remove("bad");
        // Get tricky. Create a good plugin, then replace with
        // a disabled bad one.
        StoragePluginFixtureConfig goodConfig = new StoragePluginFixtureConfig("ok");
        goodConfig.setEnabled(true);
        json = registry.encode(goodConfig);
        registry.putJson("test", json);
        assertTrue(registry.availablePlugins().contains("test"));
        assertEquals(goodConfig, registry.getPlugin("test").getConfig());
        // Replace with a disabled bad plugin
        badConfig = new StoragePluginFixtureConfig("crash-ctor");
        badConfig.setEnabled(false);
        json = registry.encode(badConfig);
        registry.putJson("test", json);
        assertFalse(registry.availablePlugins().contains("test"));
        assertNull(registry.getPlugin("test"));
        assertNotNull(registry.getStoredConfig("test"));
        assertEquals(badConfig, registry.getStoredConfig("test"));
        // Attempt to disable a disabled plugin. Should be OK.
        registry.setEnabled("test", false);
        // system, fix that system first.)
        try {
            registry.setEnabled("test", true);
            fail();
        } catch (PluginException e) {
        // Expected
        }
        assertFalse(registry.availablePlugins().contains("test"));
    }
}

Example 99 with ClusterFixtureBuilder

use of org.apache.drill.test.ClusterFixtureBuilder in project drill by apache.

the class TestPluginRegistry method testEphemeralWithoutInstance.

@Test
public void testEphemeralWithoutInstance() throws Exception {
    ClusterFixtureBuilder builder = ClusterFixture.builder(dirTestWatcher);
    try (ClusterFixture cluster = builder.build()) {
        StoragePluginRegistry registry = cluster.storageRegistry();
        // Create a plugin
        // Since we've created no plugin instance, the configs come from
        // the persistent store, there is no guarantee we get the same
        // instance on each retrieval.
        FileSystemConfig pConfig1 = myConfig1();
        registry.put(MY_PLUGIN_NAME, pConfig1);
        StoragePluginConfig savedConfig = registry.getStoredConfig(MY_PLUGIN_NAME);
        assertEquals(pConfig1, savedConfig);
        assertTrue(savedConfig.isEnabled());
        assertEquals(pConfig1, registry.getDefinedConfig(MY_PLUGIN_NAME));
        // Do not refer to the instance. As a result, no reason to
        // cache the plugin in ephemeral cache.
        // Change config
        FileSystemConfig pConfig1b = myConfig2();
        registry.put(MY_PLUGIN_NAME, pConfig1b);
        assertEquals(pConfig1b, registry.getDefinedConfig(MY_PLUGIN_NAME));
        // Put it back
        FileSystemConfig pConfig1c = myConfig1();
        registry.put(MY_PLUGIN_NAME, pConfig1c);
        assertEquals(pConfig1c, registry.getDefinedConfig(MY_PLUGIN_NAME));
        assertEquals(pConfig1c, registry.getPlugin(MY_PLUGIN_NAME).getConfig());
        // Odd case. Some thread refers to the config while not in
        // the store which forces an instance which later reappears.
        FileSystemConfig pConfig2a = myConfig1();
        registry.put("myplugin2", pConfig2a);
        // Didn't instantiate. Change
        FileSystemConfig pConfig2b = myConfig2();
        registry.put("myplugin2", pConfig2b);
        // Force a resync
        assertEquals(pConfig2b, registry.getDefinedConfig("myplugin2"));
        // Refer by original config. We didn't cache the original config
        // because there was no plugin instance. Must make up a new plugin
        // Which goes into the ephemeral cache.
        FileSystemConfig pConfig2c = myConfig1();
        StoragePlugin plugin2 = registry.getPluginByConfig(pConfig2c);
        assertEquals(pConfig2c, plugin2.getConfig());
        // Put the original config into the persistent store and local cache.
        // Should not dredge up the ephemeral version to reuse since that
        // version had an unknown name.
        // It is unfortunate that we have to instances with the same config,
        // but different names. But, since the name is immutable, and not
        // known above, the two-instance situation is the least bad option.
        FileSystemConfig pConfig2d = myConfig1();
        registry.put("myplugin2", pConfig2d);
        assertEquals(pConfig2d, registry.getPlugin("myplugin2").getConfig());
    }
}

Example 100 with ClusterFixtureBuilder

use of org.apache.drill.test.ClusterFixtureBuilder in project drill by apache.

the class TestConfigLinkage method testScopeSystem.

/* Test if the option is set at SYSTEM scope and the scope is actually SYSTEM */
@Test
public void testScopeSystem() throws Exception {
    ClusterFixtureBuilder builder = ClusterFixture.bareBuilder(dirTestWatcher).systemOption(ExecConstants.SLICE_TARGET, 10000);
    try (ClusterFixture cluster = builder.build();
        ClientFixture client = cluster.clientFixture()) {
        String scope = client.queryBuilder().sql("SELECT optionScope from sys.%s where name='planner.slice_target'", SystemTable.OPTIONS.getTableName()).singletonString();
        Assert.assertEquals("SYSTEM", scope);
    }
}