Examples with ExternalResourceDriver org.apache.flink.api.common.externalresource.ExternalResourceDriver used on opensource projects

Example 1 with ExternalResourceDriver

use of org.apache.flink.api.common.externalresource.ExternalResourceDriver in project flink by apache.

the class ExternalResourceUtils method externalResourceDriversFromConfig.

/**
 * Instantiate the {@link ExternalResourceDriver ExternalResourceDrivers} for all of enabled
 * external resources. {@link ExternalResourceDriver ExternalResourceDrivers} are mapped to its
 * resource name.
 */
@VisibleForTesting
static Map<String, ExternalResourceDriver> externalResourceDriversFromConfig(Configuration config, PluginManager pluginManager) {
    final Set<String> resourceSet = getExternalResourceSet(config);
    if (resourceSet.isEmpty()) {
        return Collections.emptyMap();
    }
    final Iterator<ExternalResourceDriverFactory> factoryIterator = pluginManager.load(ExternalResourceDriverFactory.class);
    final Map<String, ExternalResourceDriverFactory> externalResourceFactories = new HashMap<>();
    factoryIterator.forEachRemaining(externalResourceDriverFactory -> externalResourceFactories.put(externalResourceDriverFactory.getClass().getName(), externalResourceDriverFactory));
    final Map<String, ExternalResourceDriver> externalResourceDrivers = new HashMap<>();
    for (String resourceName : resourceSet) {
        final ConfigOption<String> driverClassOption = key(ExternalResourceOptions.getExternalResourceDriverFactoryConfigOptionForResource(resourceName)).stringType().noDefaultValue();
        final String driverFactoryClassName = config.getString(driverClassOption);
        if (StringUtils.isNullOrWhitespaceOnly(driverFactoryClassName)) {
            LOG.warn("Could not find driver class name for {}. Please make sure {} is configured.", resourceName, driverClassOption.key());
            continue;
        }
        ExternalResourceDriverFactory externalResourceDriverFactory = externalResourceFactories.get(driverFactoryClassName);
        if (externalResourceDriverFactory != null) {
            DelegatingConfiguration delegatingConfiguration = new DelegatingConfiguration(config, ExternalResourceOptions.getExternalResourceParamConfigPrefixForResource(resourceName));
            try {
                externalResourceDrivers.put(resourceName, externalResourceDriverFactory.createExternalResourceDriver(delegatingConfiguration));
                LOG.info("Add external resources driver for {}.", resourceName);
            } catch (Exception e) {
                LOG.warn("Could not instantiate driver with factory {} for {}. {}", driverFactoryClassName, resourceName, e);
            }
        } else {
            LOG.warn("Could not find factory class {} for {}.", driverFactoryClassName, resourceName);
        }
    }
    return externalResourceDrivers;
}

Example 2 with ExternalResourceDriver

use of org.apache.flink.api.common.externalresource.ExternalResourceDriver in project flink by apache.

the class ExternalResourceUtilsTest method testConstructExternalResourceDriversFromConfig.

@Test
public void testConstructExternalResourceDriversFromConfig() {
    final Configuration config = new Configuration();
    final String driverFactoryClassName = TestingExternalResourceDriverFactory.class.getName();
    final Map<Class<?>, Iterator<?>> plugins = new HashMap<>();
    plugins.put(ExternalResourceDriverFactory.class, IteratorUtils.singletonIterator(new TestingExternalResourceDriverFactory()));
    final PluginManager testingPluginManager = new TestingPluginManager(plugins);
    config.set(ExternalResourceOptions.EXTERNAL_RESOURCE_LIST, Collections.singletonList(RESOURCE_NAME_1));
    config.setString(ExternalResourceOptions.getExternalResourceDriverFactoryConfigOptionForResource(RESOURCE_NAME_1), driverFactoryClassName);
    final Map<String, ExternalResourceDriver> externalResourceDrivers = ExternalResourceUtils.externalResourceDriversFromConfig(config, testingPluginManager);
    assertThat(externalResourceDrivers.size(), is(1));
    assertThat(externalResourceDrivers.get(RESOURCE_NAME_1), instanceOf(TestingExternalResourceDriver.class));
}

Example 3 with ExternalResourceDriver

use of org.apache.flink.api.common.externalresource.ExternalResourceDriver in project flink by apache.

the class ExternalResourceUtilsTest method testFactoryFailedToCreateDriver.

@Test
public void testFactoryFailedToCreateDriver() {
    final Configuration config = new Configuration();
    final String driverFactoryClassName = TestingFailedExternalResourceDriverFactory.class.getName();
    final Map<Class<?>, Iterator<?>> plugins = new HashMap<>();
    plugins.put(ExternalResourceDriverFactory.class, IteratorUtils.singletonIterator(new TestingFailedExternalResourceDriverFactory()));
    final PluginManager testingPluginManager = new TestingPluginManager(plugins);
    config.set(ExternalResourceOptions.EXTERNAL_RESOURCE_LIST, Collections.singletonList(RESOURCE_NAME_1));
    config.setString(ExternalResourceOptions.getExternalResourceDriverFactoryConfigOptionForResource(RESOURCE_NAME_1), driverFactoryClassName);
    final Map<String, ExternalResourceDriver> externalResourceDrivers = ExternalResourceUtils.externalResourceDriversFromConfig(config, testingPluginManager);
    assertThat(externalResourceDrivers.entrySet(), is(empty()));
}

Example 4 with ExternalResourceDriver

use of org.apache.flink.api.common.externalresource.ExternalResourceDriver in project flink by apache.

the class ExternalResourceUtilsTest method testFactoryPluginDoesNotExist.

@Test
public void testFactoryPluginDoesNotExist() {
    final Configuration config = new Configuration();
    final String driverFactoryClassName = TestingExternalResourceDriverFactory.class.getName();
    final PluginManager testingPluginManager = new TestingPluginManager(Collections.emptyMap());
    config.set(ExternalResourceOptions.EXTERNAL_RESOURCE_LIST, Collections.singletonList(RESOURCE_NAME_1));
    config.setString(ExternalResourceOptions.getExternalResourceDriverFactoryConfigOptionForResource(RESOURCE_NAME_1), driverFactoryClassName);
    final Map<String, ExternalResourceDriver> externalResourceDrivers = ExternalResourceUtils.externalResourceDriversFromConfig(config, testingPluginManager);
    assertThat(externalResourceDrivers.entrySet(), is(empty()));
}

Example 5 with ExternalResourceDriver

use of org.apache.flink.api.common.externalresource.ExternalResourceDriver in project flink by apache.

the class ExternalResourceUtils method createStaticExternalResourceInfoProvider.

/**
 * Instantiate {@link StaticExternalResourceInfoProvider} for all of enabled external resources.
 */
@VisibleForTesting
static ExternalResourceInfoProvider createStaticExternalResourceInfoProvider(Map<String, Long> externalResourceAmountMap, Map<String, ExternalResourceDriver> externalResourceDrivers) {
    final Map<String, Set<? extends ExternalResourceInfo>> externalResources = new HashMap<>();
    for (Map.Entry<String, ExternalResourceDriver> externalResourceDriverEntry : externalResourceDrivers.entrySet()) {
        final String resourceName = externalResourceDriverEntry.getKey();
        final ExternalResourceDriver externalResourceDriver = externalResourceDriverEntry.getValue();
        if (externalResourceAmountMap.containsKey(resourceName)) {
            try {
                final Set<? extends ExternalResourceInfo> externalResourceInfos;
                externalResourceInfos = externalResourceDriver.retrieveResourceInfo(externalResourceAmountMap.get(resourceName));
                externalResources.put(resourceName, externalResourceInfos);
            } catch (Exception e) {
                LOG.warn("Failed to retrieve information of external resource {}.", resourceName, e);
            }
        } else {
            LOG.warn("Could not found legal amount configuration for {}.", resourceName);
        }
    }
    return new StaticExternalResourceInfoProvider(externalResources);
}