Examples with TwillRunnerService org.apache.twill.api.TwillRunnerService used on opensource projects

Example 1 with TwillRunnerService

use of org.apache.twill.api.TwillRunnerService in project cdap by caskdata.

the class RemoteExecutionJobMain method initialize.

@VisibleForTesting
RemoteExecutionRuntimeJobEnvironment initialize(CConfiguration cConf) throws Exception {
    zkServer = InMemoryZKServer.builder().build();
    zkServer.startAndWait();
    InetSocketAddress zkAddr = ResolvingDiscoverable.resolve(zkServer.getLocalAddress());
    String zkConnectStr = String.format("%s:%d", zkAddr.getHostString(), zkAddr.getPort());
    LOG.debug("In memory ZK started at {}", zkConnectStr);
    cConf.set(Constants.Zookeeper.QUORUM, zkConnectStr);
    Injector injector = Guice.createInjector(new ConfigModule(cConf), RemoteAuthenticatorModules.getDefaultModule(), new DFSLocationModule(), new InMemoryDiscoveryModule(), new TwillModule(), new AuthenticationContextModules().getProgramContainerModule(cConf), new AbstractModule() {

        @Override
        protected void configure() {
            // don't need to perform any impersonation from within user programs
            bind(UGIProvider.class).to(CurrentUGIProvider.class).in(Scopes.SINGLETON);
            // Binds a no-op SecureStore for the TwillModule to setup TokenSecureStoreRenewer.
            bind(SecureStore.class).toInstance(new SecureStore() {

                @Override
                public List<SecureStoreMetadata> list(String namespace) {
                    return Collections.emptyList();
                }

                @Override
                public SecureStoreData get(String namespace, String name) throws Exception {
                    throw new NotFoundException("Secure key " + name + " not found in namespace " + namespace);
                }
            });
        }
    });
    Map<String, String> properties = new HashMap<>();
    properties.put(Constants.Zookeeper.QUORUM, zkConnectStr);
    locationFactory = injector.getInstance(LocationFactory.class);
    locationFactory.create("/").mkdirs();
    twillRunnerService = injector.getInstance(TwillRunnerService.class);
    twillRunnerService.start();
    if (UserGroupInformation.isSecurityEnabled()) {
        TokenSecureStoreRenewer secureStoreRenewer = injector.getInstance(TokenSecureStoreRenewer.class);
        secureStoreUpdateCancellable = twillRunnerService.setSecureStoreRenewer(secureStoreRenewer, 30000L, secureStoreRenewer.getUpdateInterval(), 30000L, TimeUnit.MILLISECONDS);
    }
    return new RemoteExecutionRuntimeJobEnvironment(locationFactory, twillRunnerService, properties);
}

Example 2 with TwillRunnerService

use of org.apache.twill.api.TwillRunnerService in project cdap by caskdata.

the class AppFabricServiceMain method addServices.

@Override
protected void addServices(Injector injector, List<? super Service> services, List<? super AutoCloseable> closeableResources, MasterEnvironment masterEnv, MasterEnvironmentContext masterEnvContext, EnvironmentOptions options) {
    CConfiguration cConf = injector.getInstance(CConfiguration.class);
    if (SecurityUtil.isInternalAuthEnabled(cConf)) {
        services.add(injector.getInstance(TokenManager.class));
    }
    closeableResources.add(injector.getInstance(AccessControllerInstantiator.class));
    services.add(injector.getInstance(OperationalStatsService.class));
    services.add(injector.getInstance(SecureStoreService.class));
    services.add(injector.getInstance(DatasetOpExecutorService.class));
    services.add(injector.getInstance(ServiceStore.class));
    HealthCheckService healthCheckService = injector.getInstance(HealthCheckService.class);
    healthCheckService.helper(Constants.AppFabricHealthCheck.APP_FABRIC_HEALTH_CHECK_SERVICE, cConf, Constants.Service.MASTER_SERVICES_BIND_ADDRESS);
    services.add(healthCheckService);
    Binding<ZKClientService> zkBinding = injector.getExistingBinding(Key.get(ZKClientService.class));
    if (zkBinding != null) {
        services.add(zkBinding.getProvider().get());
    }
    // Start both the remote TwillRunnerService and regular TwillRunnerService
    TwillRunnerService remoteTwillRunner = injector.getInstance(Key.get(TwillRunnerService.class, Constants.AppFabric.RemoteExecution.class));
    services.add(new TwillRunnerServiceWrapper(remoteTwillRunner));
    services.add(new TwillRunnerServiceWrapper(injector.getInstance(TwillRunnerService.class)));
    services.add(new RetryOnStartFailureService(() -> injector.getInstance(DatasetService.class), RetryStrategies.exponentialDelay(200, 5000, TimeUnit.MILLISECONDS)));
    services.add(injector.getInstance(AppFabricServer.class));
    services.add(injector.getInstance(TetheringClientSubscriberService.class));
    if (cConf.getBoolean(Constants.TaskWorker.POOL_ENABLE)) {
        services.add(injector.getInstance(TaskWorkerServiceLauncher.class));
    }
    if (cConf.getBoolean(SystemWorker.POOL_ENABLE)) {
        services.add(injector.getInstance(SystemWorkerServiceLauncher.class));
    }
    // Optionally adds the master environment task
    masterEnv.getTask().ifPresent(task -> services.add(new MasterTaskExecutorService(task, masterEnvContext)));
}