Search in sources :

Example 1 with DataTransferTaskConfig

use of org.apache.airavata.gfac.core.config.DataTransferTaskConfig in project airavata by apache.

the class Factory method loadConfiguration.

public static void loadConfiguration() throws GFacException {
    GFacYamlConfigruation config = new GFacYamlConfigruation();
    try {
        for (JobSubmitterTaskConfig jobSubmitterTaskConfig : config.getJobSbumitters()) {
            String taskClass = jobSubmitterTaskConfig.getTaskClass();
            Class<?> aClass = Class.forName(taskClass);
            Constructor<?> constructor = aClass.getConstructor();
            JobSubmissionTask task = (JobSubmissionTask) constructor.newInstance();
            task.init(jobSubmitterTaskConfig.getProperties());
            jobSubmissionTask.put(jobSubmitterTaskConfig.getSubmissionProtocol(), task);
        }
        for (DataTransferTaskConfig dataTransferTaskConfig : config.getFileTransferTasks()) {
            String taskClass = dataTransferTaskConfig.getTaskClass();
            Class<?> aClass = Class.forName(taskClass);
            Constructor<?> constructor = aClass.getConstructor();
            Task task = (Task) constructor.newInstance();
            task.init(dataTransferTaskConfig.getProperties());
            dataMovementTask.put(dataTransferTaskConfig.getTransferProtocol(), task);
        }
        for (ResourceConfig resourceConfig : config.getResourceConfiguration()) {
            resources.put(resourceConfig.getJobManagerType(), resourceConfig);
        }
    } catch (Exception e) {
        throw new GFacException("Gfac config issue", e);
    }
    sessionCache = CacheBuilder.newBuilder().expireAfterAccess(ServerSettings.getSessionCacheAccessTimeout(), TimeUnit.MINUTES).removalListener((RemovalListener<String, Session>) removalNotification -> {
        if (removalNotification.getValue().isConnected()) {
            log.info("Disconnecting ssh session with key: " + removalNotification.getKey());
            removalNotification.getValue().disconnect();
        }
        log.info("Removed ssh session with key: " + removalNotification.getKey());
    }).build();
}
Also used : JobSubmitterTaskConfig(org.apache.airavata.gfac.core.config.JobSubmitterTaskConfig) JobManagerConfiguration(org.apache.airavata.gfac.core.JobManagerConfiguration) ArchiveTask(org.apache.airavata.gfac.impl.task.ArchiveTask) ExperimentCatalog(org.apache.airavata.registry.cpi.ExperimentCatalog) JobMonitor(org.apache.airavata.gfac.core.monitor.JobMonitor) LoggerFactory(org.slf4j.LoggerFactory) CancelRequestWatcherImpl(org.apache.airavata.gfac.impl.watcher.CancelRequestWatcherImpl) ResourceJobManagerType(org.apache.airavata.model.appcatalog.computeresource.ResourceJobManagerType) Credential(org.apache.airavata.credential.store.credential.Credential) ServerSettings(org.apache.airavata.common.utils.ServerSettings) Publisher(org.apache.airavata.messaging.core.Publisher) Map(java.util.Map) CredentialStoreException(org.apache.airavata.credential.store.store.CredentialStoreException) SlurmJobConfiguration(org.apache.airavata.gfac.impl.job.SlurmJobConfiguration) MessagingFactory(org.apache.airavata.messaging.core.MessagingFactory) AppCatalogException(org.apache.airavata.registry.cpi.AppCatalogException) ProcessContext(org.apache.airavata.gfac.core.context.ProcessContext) SSHKeyAuthentication(org.apache.airavata.gfac.core.authentication.SSHKeyAuthentication) RabbitMQPublisher(org.apache.airavata.messaging.core.impl.RabbitMQPublisher) UUID(java.util.UUID) HostScheduler(org.apache.airavata.gfac.core.scheduler.HostScheduler) RedeliveryRequestWatcherImpl(org.apache.airavata.gfac.impl.watcher.RedeliveryRequestWatcherImpl) MessageHandler(org.apache.airavata.messaging.core.MessageHandler) AppCatalog(org.apache.airavata.registry.cpi.AppCatalog) List(java.util.List) AuroraJobMonitor(org.apache.airavata.gfac.monitor.cloud.AuroraJobMonitor) CuratorFramework(org.apache.curator.framework.CuratorFramework) RetryPolicy(org.apache.curator.RetryPolicy) RedeliveryRequestWatcher(org.apache.airavata.gfac.core.watcher.RedeliveryRequestWatcher) CacheBuilder(com.google.common.cache.CacheBuilder) SSHCredential(org.apache.airavata.credential.store.credential.impl.ssh.SSHCredential) DataTransferTaskConfig(org.apache.airavata.gfac.core.config.DataTransferTaskConfig) JobSubmitterTaskConfig(org.apache.airavata.gfac.core.config.JobSubmitterTaskConfig) Task(org.apache.airavata.gfac.core.task.Task) CuratorFrameworkFactory(org.apache.curator.framework.CuratorFrameworkFactory) ForkJobConfiguration(org.apache.airavata.gfac.impl.job.ForkJobConfiguration) PBSJobConfiguration(org.apache.airavata.gfac.impl.job.PBSJobConfiguration) Type(org.apache.airavata.messaging.core.Type) DataMovementProtocol(org.apache.airavata.model.data.movement.DataMovementProtocol) HashMap(java.util.HashMap) UGEJobConfiguration(org.apache.airavata.gfac.impl.job.UGEJobConfiguration) Constructor(java.lang.reflect.Constructor) CancelRequestWatcher(org.apache.airavata.gfac.core.watcher.CancelRequestWatcher) ArrayList(java.util.ArrayList) ExponentialBackoffRetry(org.apache.curator.retry.ExponentialBackoffRetry) CredentialReader(org.apache.airavata.credential.store.store.CredentialReader) ResourceJobManager(org.apache.airavata.model.appcatalog.computeresource.ResourceJobManager) JobSubmissionTask(org.apache.airavata.gfac.core.task.JobSubmissionTask) RegistryException(org.apache.airavata.registry.cpi.RegistryException) MonitorMode(org.apache.airavata.model.appcatalog.computeresource.MonitorMode) AiravataException(org.apache.airavata.common.exception.AiravataException) GFacUtils(org.apache.airavata.gfac.core.GFacUtils) LSFJobConfiguration(org.apache.airavata.gfac.impl.job.LSFJobConfiguration) RegistryFactory(org.apache.airavata.registry.core.experiment.catalog.impl.RegistryFactory) ResourceConfig(org.apache.airavata.gfac.core.config.ResourceConfig) Logger(org.slf4j.Logger) GFacException(org.apache.airavata.gfac.core.GFacException) RemoteCluster(org.apache.airavata.gfac.core.cluster.RemoteCluster) GFacYamlConfigruation(org.apache.airavata.gfac.core.config.GFacYamlConfigruation) JobSubmissionProtocol(org.apache.airavata.model.appcatalog.computeresource.JobSubmissionProtocol) EmailBasedMonitor(org.apache.airavata.gfac.monitor.email.EmailBasedMonitor) TimeUnit(java.util.concurrent.TimeUnit) AuthenticationInfo(org.apache.airavata.gfac.core.authentication.AuthenticationInfo) GFacContext(org.apache.airavata.gfac.core.context.GFacContext) GFacEngine(org.apache.airavata.gfac.core.GFacEngine) RemovalListener(com.google.common.cache.RemovalListener) Cache(com.google.common.cache.Cache) ApplicationSettingsException(org.apache.airavata.common.exception.ApplicationSettingsException) OutputParser(org.apache.airavata.gfac.core.cluster.OutputParser) ServerInfo(org.apache.airavata.gfac.core.cluster.ServerInfo) com.jcraft.jsch(com.jcraft.jsch) Subscriber(org.apache.airavata.messaging.core.Subscriber) ArchiveTask(org.apache.airavata.gfac.impl.task.ArchiveTask) Task(org.apache.airavata.gfac.core.task.Task) JobSubmissionTask(org.apache.airavata.gfac.core.task.JobSubmissionTask) CredentialStoreException(org.apache.airavata.credential.store.store.CredentialStoreException) AppCatalogException(org.apache.airavata.registry.cpi.AppCatalogException) RegistryException(org.apache.airavata.registry.cpi.RegistryException) AiravataException(org.apache.airavata.common.exception.AiravataException) GFacException(org.apache.airavata.gfac.core.GFacException) ApplicationSettingsException(org.apache.airavata.common.exception.ApplicationSettingsException) GFacException(org.apache.airavata.gfac.core.GFacException) JobSubmissionTask(org.apache.airavata.gfac.core.task.JobSubmissionTask) ResourceConfig(org.apache.airavata.gfac.core.config.ResourceConfig) DataTransferTaskConfig(org.apache.airavata.gfac.core.config.DataTransferTaskConfig) GFacYamlConfigruation(org.apache.airavata.gfac.core.config.GFacYamlConfigruation)

Aggregations

Cache (com.google.common.cache.Cache)1 CacheBuilder (com.google.common.cache.CacheBuilder)1 RemovalListener (com.google.common.cache.RemovalListener)1 com.jcraft.jsch (com.jcraft.jsch)1 Constructor (java.lang.reflect.Constructor)1 ArrayList (java.util.ArrayList)1 HashMap (java.util.HashMap)1 List (java.util.List)1 Map (java.util.Map)1 UUID (java.util.UUID)1 TimeUnit (java.util.concurrent.TimeUnit)1 AiravataException (org.apache.airavata.common.exception.AiravataException)1 ApplicationSettingsException (org.apache.airavata.common.exception.ApplicationSettingsException)1 ServerSettings (org.apache.airavata.common.utils.ServerSettings)1 Credential (org.apache.airavata.credential.store.credential.Credential)1 SSHCredential (org.apache.airavata.credential.store.credential.impl.ssh.SSHCredential)1 CredentialReader (org.apache.airavata.credential.store.store.CredentialReader)1 CredentialStoreException (org.apache.airavata.credential.store.store.CredentialStoreException)1 GFacEngine (org.apache.airavata.gfac.core.GFacEngine)1 GFacException (org.apache.airavata.gfac.core.GFacException)1