Examples with DynamicConfiguration org.talend.hadoop.distribution.dynamic.DynamicConfiguration used on opensource projects

Example 6 with DynamicConfiguration

use of org.talend.hadoop.distribution.dynamic.DynamicConfiguration in project tbd-studio-se by Talend.

the class DynamicClassLoaderExtensionAdaper method adapt.

public IDynamicExtension adapt(IDynamicMonitor monitor) throws Exception {
    DynamicDistributionUtils.checkCancelOrNot(monitor);
    resolve();
    TemplateBean templateBean = getTemplateBean();
    DynamicConfiguration configuration = getConfiguration();
    String id = configuration.getId();
    String distributionName = templateBean.getDistribution();
    String templateId = templateBean.getId();
    IDynamicExtension classLoaderExtension = DynamicFactory.getInstance().createDynamicExtension();
    classLoaderExtension.setExtensionId(DynamicDistributionUtils.getExtensionId(DynamicDistributionUtils.getPluginKey(distributionName, templateId, id, ATTR_POINT)));
    classLoaderExtension.setExtensionPoint(ATTR_POINT);
    List<ClassLoaderBean> classLoaders = templateBean.getClassLoaders();
    if (classLoaders != null) {
        for (ClassLoaderBean classLoader : classLoaders) {
            DynamicDistributionUtils.checkCancelOrNot(monitor);
            DynamicClassloaderAdapter classLoaderAdapter = new DynamicClassloaderAdapter(templateBean, configuration, classLoader, moduleGroupBeanAdapterMap);
            IDynamicConfiguration dynamicClassLoader = classLoaderAdapter.adapt(monitor);
            classLoaderExtension.addConfiguration(dynamicClassLoader);
        }
    }
    return classLoaderExtension;
}

Example 7 with DynamicConfiguration

use of org.talend.hadoop.distribution.dynamic.DynamicConfiguration in project tbd-studio-se by Talend.

the class DynamicModuleAdapter method adaptExclusions.

private List<ExclusionNode> adaptExclusions(List<ExclusionBean> exclusionBeans, IDynamicMonitor monitor) throws Exception {
    List<ExclusionNode> exclusionNodes = new ArrayList<>();
    if (exclusionBeans != null && !exclusionBeans.isEmpty()) {
        TemplateBean templBean = getTemplateBean();
        DynamicConfiguration dynamicConfiguration = getConfiguration();
        for (ExclusionBean exclusionBean : exclusionBeans) {
            DynamicExclusionAdapter adapter = new DynamicExclusionAdapter(templBean, dynamicConfiguration, exclusionBean);
            adapter.adapt(monitor);
            exclusionNodes.add(adapter.getExclusionNode());
        }
    }
    return exclusionNodes;
}

Example 8 with DynamicConfiguration

use of org.talend.hadoop.distribution.dynamic.DynamicConfiguration in project tbd-studio-se by Talend.

the class DynamicModuleAdapter method adapt.

public List<IDynamicConfiguration> adapt(IDynamicMonitor monitor, boolean multiThread) throws Exception {
    DynamicDistributionUtils.checkCancelOrNot(monitor);
    resolve();
    if (monitor != null) {
        String mvnUri = moduleBean.getMvnUri();
        if (StringUtils.isEmpty(mvnUri)) {
            mvnUri = getMvnUri();
        }
        monitor.setTaskName(// $NON-NLS-1$
        Messages.getString("DynamicModuleAdapter.monitor.buildModule", moduleBean.getId(), mvnUri));
    }
    TemplateBean templateBean = getTemplateBean();
    DynamicConfiguration configuration = getConfiguration();
    String distribution = configuration.getDistribution();
    String hadoopVersion = configuration.getVersion();
    String id = configuration.getId();
    List<ESparkVersion> selectedSparkVersions = configuration.getSelectedSparkVersions();
    List<IDynamicConfiguration> librariesNeeded = new ArrayList<>();
    List<String> sparkVersions = moduleBean.getSupportedSparkVersions();
    if (sparkVersions != null && !sparkVersions.isEmpty()) {
        boolean isSupport = false;
        for (String sparkVersion : sparkVersions) {
            try {
                ESparkVersion eSparkVersion = ESparkVersion.valueOf(sparkVersion);
                if (selectedSparkVersions.contains(eSparkVersion)) {
                    isSupport = true;
                    break;
                }
            } catch (Exception e) {
                ExceptionHandler.process(e);
            }
        }
        if (!isSupport) {
            this.isSkipped = true;
            return librariesNeeded;
        }
    }
    String type = moduleBean.getType();
    if (ModuleBean.TYPE_BASE.equalsIgnoreCase(type)) {
        String groupId = moduleBean.getGroupId();
        String artifactId = moduleBean.getArtifactId();
        String scope = moduleBean.getScope();
        String extension = moduleBean.getExtension();
        String classifier = moduleBean.getClassifier();
        String moduleVersion = moduleBean.getVersion();
        String useStudioRepository = moduleBean.getUseStudioRepository();
        if (StringUtils.isNotEmpty(useStudioRepository)) {
            ExceptionHandler.process(// $NON-NLS-1$
            new Exception("Currently useStudioRepository is only supported by STANDARD type, will be ignored"), Priority.WARN);
        }
        boolean useLatest = Boolean.valueOf(moduleBean.getUseLatest());
        if (StringUtils.isBlank(extension)) {
            extension = null;
        }
        List<ExclusionBean> exclusionBeans = moduleBean.getExclusions();
        List<ExclusionNode> exclusions = null;
        if (exclusionBeans != null && !exclusionBeans.isEmpty()) {
            exclusions = adaptExclusions(exclusionBeans, monitor);
        }
        DependencyNode dependencyNode = null;
        DependencyNode baseNode = new DependencyNode();
        baseNode.setGroupId(groupId);
        baseNode.setArtifactId(artifactId);
        baseNode.setClassifier(classifier);
        baseNode.setExtension(extension);
        baseNode.setScope(scope);
        if (useLatest) {
            moduleVersion = dependencyResolver.getLatestVersion(groupId, artifactId, null, null, monitor);
        }
        baseNode.setVersion(moduleVersion);
        if (exclusions != null && !exclusions.isEmpty()) {
            baseNode.setExclusions(exclusions);
        }
        try {
            dependencyNode = dependencyResolver.collectDependencies(baseNode, monitor, multiThread);
            librariesNeeded = createLibrariesNeeded(dependencyNode, distribution, hadoopVersion, id, moduleBean, runtimeIds, templateBean);
        } catch (VersionNotFoundException e) {
            ExceptionHandler.process(e);
        }
    } else if (ModuleBean.TYPE_REFERENCE.equalsIgnoreCase(type)) {
        List<ExclusionBean> exclusions = moduleBean.getExclusions();
        if (exclusions != null && !exclusions.isEmpty()) {
            throw new UnsupportedOperationException(// $NON-NLS-1$
            Messages.getString("DynamicModuleAdapter.exception.exclusion.unsupport", type));
        }
        String jarName = moduleBean.getJarName();
        ModuleNeeded moduleNeeded = existingModuleMap.get(jarName);
        if (moduleNeeded == null) {
            throw new UnsupportedOperationException(// $NON-NLS-1$
            Messages.getString("DynamicModuleAdapter.exception.reference.notFound", jarName));
        }
        runtimeIds.add(moduleNeeded.getId());
    } else if (ModuleBean.TYPE_STANDARD.equalsIgnoreCase(type)) {
        List<ExclusionBean> exclusions = moduleBean.getExclusions();
        if (exclusions != null && !exclusions.isEmpty()) {
            throw new UnsupportedOperationException(// $NON-NLS-1$
            Messages.getString("DynamicModuleAdapter.exception.exclusion.unsupport", type));
        }
        String beanId = moduleBean.getId();
        String runtimeId = DynamicDistributionUtils.getPluginKey(distribution, hadoopVersion, id, beanId);
        if (!registedModules.contains(runtimeId)) {
            IDynamicConfiguration libraryNeeded = createLibraryNeeded(moduleBean);
            libraryNeeded.setAttribute(ATTR_ID, runtimeId);
            librariesNeeded.add(libraryNeeded);
            List<String> registedRuntimeIds = new ArrayList<>();
            registedRuntimeIds.add(runtimeId);
            registedModules.add(runtimeId);
        }
        runtimeIds.add(runtimeId);
    }
    return librariesNeeded;
}

Example 9 with DynamicConfiguration

use of org.talend.hadoop.distribution.dynamic.DynamicConfiguration in project tbd-studio-se by Talend.

the class DynamicModuleGroupAdapter method adapt.

public IDynamicConfiguration adapt(IDynamicMonitor monitor) throws Exception {
    DynamicDistributionUtils.checkCancelOrNot(monitor);
    resolve();
    if (monitor != null) {
        monitor.setTaskName(// $NON-NLS-1$
        Messages.getString("DynamicModuleGroupAdapter.monitor.buildModuleGroup", moduleGroupBean.getId()));
    }
    DynamicConfiguration configuration = getConfiguration();
    String distribution = configuration.getDistribution();
    String version = configuration.getVersion();
    String id = configuration.getId();
    String moduleGroupId = moduleGroupBean.getId();
    String description = moduleGroupBean.getDescription();
    List<ESparkVersion> selectedSparkVersions = configuration.getSelectedSparkVersions();
    List<String> sparkVersions = moduleGroupBean.getSupportedSparkVersions();
    if (sparkVersions != null && !sparkVersions.isEmpty()) {
        boolean isSupport = false;
        for (String sparkVersion : sparkVersions) {
            try {
                ESparkVersion eSparkVersion = ESparkVersion.valueOf(sparkVersion);
                if (selectedSparkVersions.contains(eSparkVersion)) {
                    isSupport = true;
                    break;
                }
            } catch (Exception e) {
                ExceptionHandler.process(e);
            }
        }
        if (!isSupport) {
            return null;
        }
    }
    runtimeId = DynamicDistributionUtils.getPluginKey(distribution, version, id, moduleGroupId);
    IDynamicConfiguration dynamicModuleGroup = DynamicFactory.getInstance().createDynamicConfiguration();
    dynamicModuleGroup.setConfigurationName(TAG_NAME);
    dynamicModuleGroup.setAttribute(ATTR_ID, runtimeId);
    dynamicModuleGroup.setAttribute(ATTR_GROUP_TEMPLATE_ID, moduleGroupId);
    dynamicModuleGroup.setAttribute(ATTR_DESCRIPTION, description);
    List<String> modules = moduleGroupBean.getModules();
    Set<String> runtimeModulesSet = new HashSet<>();
    if (modules != null) {
        for (String module : modules) {
            DynamicModuleAdapter moduleAdapter = moduleBeanAdapterMap.get(module);
            if (moduleAdapter == null) {
                throw new Exception(// $NON-NLS-1$
                Messages.getString(// $NON-NLS-1$
                "DynamicModuleGroupAdapter.exception.noModuleAdapterFound", // $NON-NLS-1$
                module, DynamicModuleAdapter.class.getName()));
            }
            if (moduleAdapter.isSkipped()) {
                continue;
            }
            List<String> runtimeIds = moduleAdapter.getRuntimeIds();
            if (runtimeIds == null || runtimeIds.isEmpty()) {
                // means it is using existing modules of studio
                runtimeModulesSet.add(module);
            } else {
                runtimeModulesSet.addAll(runtimeIds);
            }
        }
        if (runtimeModulesSet != null) {
            for (String runtimeModule : runtimeModulesSet) {
                IDynamicConfiguration createDynamicLibrary = createDynamicLibrary(runtimeModule);
                dynamicModuleGroup.addChildConfiguration(createDynamicLibrary);
            }
        }
    }
    this.runtimeModules.addAll(runtimeModulesSet);
    return dynamicModuleGroup;
}

Example 10 with DynamicConfiguration

use of org.talend.hadoop.distribution.dynamic.DynamicConfiguration in project tbd-studio-se by Talend.

the class DynamicTemplateAdapter method adapt.

public void adapt(IDynamicMonitor monitor) throws Exception {
    DynamicDistributionUtils.checkCancelOrNot(monitor);
    resolve();
    TemplateBean templateBean = getTemplateBean();
    DynamicConfiguration configuration = getConfiguration();
    // use id instead of version
    templateBean.setDynamicVersion(configuration.getId());
    DynamicDistributionManager dynamicDistributionManager = DynamicDistributionManager.getInstance();
    IDependencyResolver dependencyResolver = dynamicDistributionManager.getDependencyResolver(configuration);
    dynamicPlugin = DynamicFactory.getInstance().createDynamicPlugin();
    distriConfigAdapter = new DynamicDistribConfigAdapter(templateBean, configuration);
    IDynamicPluginConfiguration pluginConfiguration = distriConfigAdapter.adapt(monitor);
    dynamicPlugin.setPluginConfiguration(pluginConfiguration);
    moduleBeanAdapterMap = new HashMap<>();
    moduleGroupBeanAdapterMap = new HashMap<>();
    DynamicLibraryNeededExtensionAdaper libNeededExtAdapter = new DynamicLibraryNeededExtensionAdaper(templateBean, configuration, dependencyResolver, moduleBeanAdapterMap, moduleGroupBeanAdapterMap);
    libNeededExtAdapter.enableMultiThread(!Boolean.getBoolean(PROPERTY_DISABLE_MULTI_THTREAD));
    IDynamicExtension dynamicLibNeededExtension = libNeededExtAdapter.adapt(monitor);
    dynamicPlugin.addExtension(dynamicLibNeededExtension);
    DynamicClassLoaderExtensionAdaper clsLoaderAdapter = new DynamicClassLoaderExtensionAdaper(templateBean, configuration, moduleGroupBeanAdapterMap);
    IDynamicExtension dynamicClsLoaderExtension = clsLoaderAdapter.adapt(monitor);
    dynamicPlugin.addExtension(dynamicClsLoaderExtension);
}