Examples with MasterEnvironmentContext io.cdap.cdap.master.spi.environment.MasterEnvironmentContext used on opensource projects

Example 1 with MasterEnvironmentContext

use of io.cdap.cdap.master.spi.environment.MasterEnvironmentContext in project cdap by caskdata.

the class AbstractServiceMain method init.

@Override
public final void init(String[] args) throws Exception {
    LOG.info("Initializing master service class {}", getClass().getName());
    // System wide setup
    Thread.setDefaultUncaughtExceptionHandler(new UncaughtExceptionHandler());
    // Intercept JUL loggers
    SLF4JBridgeHandler.removeHandlersForRootLogger();
    SLF4JBridgeHandler.install();
    TypeToken<?> type = TypeToken.of(getClass()).resolveType(AbstractServiceMain.class.getTypeParameters()[0]);
    T options = (T) type.getRawType().newInstance();
    OptionsParser.init(options, args, getClass().getSimpleName(), ProjectInfo.getVersion().toString(), System.out);
    CConfiguration cConf = CConfiguration.create();
    SecurityUtil.loginForMasterService(cConf);
    SConfiguration sConf = SConfiguration.create();
    if (options.getExtraConfPath() != null) {
        cConf.addResource(new File(options.getExtraConfPath(), "cdap-site.xml").toURI().toURL());
        sConf.addResource(new File(options.getExtraConfPath(), "cdap-security.xml").toURI().toURL());
    }
    cConf = updateCConf(cConf);
    Configuration hConf = new Configuration();
    masterEnv = MasterEnvironments.setMasterEnvironment(MasterEnvironments.create(cConf, options.getEnvProvider()));
    MasterEnvironmentContext masterEnvContext = MasterEnvironments.createContext(cConf, hConf, masterEnv.getName());
    masterEnv.initialize(masterEnvContext);
    List<Module> modules = new ArrayList<>();
    modules.add(new ConfigModule(cConf, hConf, sConf));
    modules.add(RemoteAuthenticatorModules.getDefaultModule());
    modules.add(new PreviewConfigModule(cConf, hConf, sConf));
    modules.add(new IOModule());
    modules.add(new MetricsClientRuntimeModule().getDistributedModules());
    modules.add(new AbstractModule() {

        @Override
        protected void configure() {
            bind(DiscoveryService.class).toProvider(new SupplierProviderBridge<>(masterEnv.getDiscoveryServiceSupplier()));
            bind(DiscoveryServiceClient.class).toProvider(new SupplierProviderBridge<>(masterEnv.getDiscoveryServiceClientSupplier()));
        }
    });
    modules.add(getLogAppenderModule());
    CoreSecurityModule coreSecurityModule = CoreSecurityRuntimeModule.getDistributedModule(cConf);
    modules.add(coreSecurityModule);
    if (coreSecurityModule.requiresZKClient()) {
        modules.add(new ZKClientModule());
    }
    modules.add(new AuthenticationContextModules().getMasterModule());
    modules.addAll(getServiceModules(masterEnv, options, cConf));
    injector = Guice.createInjector(modules);
    // Initialize logging context
    LogAppenderInitializer logAppenderInitializer = injector.getInstance(LogAppenderInitializer.class);
    closeableResources.add(logAppenderInitializer);
    logAppenderInitializer.initialize();
    Optional.ofNullable(getLoggingContext(options)).ifPresent(LoggingContextAccessor::setLoggingContext);
    // Add Services
    services.add(injector.getInstance(MetricsCollectionService.class));
    addServices(injector, services, closeableResources, masterEnv, masterEnvContext, options);
    // Optionally get the storage provider. It is for destroy() method to close it on shutdown.
    Binding<StorageProvider> storageBinding = injector.getExistingBinding(Key.get(StorageProvider.class));
    if (storageBinding != null) {
        storageProvider = storageBinding.getProvider().get();
    }
    LOG.info("Service {} initialized", getClass().getName());
}

Example 2 with MasterEnvironmentContext

use of io.cdap.cdap.master.spi.environment.MasterEnvironmentContext in project cdap by caskdata.

the class MasterEnvironmentMain method doMain.

/**
 * The actual main method that get invoke through reflection from the {@link #main(String[])} method.
 */
@SuppressWarnings("unused")
public static void doMain(String[] args) throws Exception {
    CountDownLatch shutdownLatch = new CountDownLatch(1);
    try {
        // System wide setup
        Thread.setDefaultUncaughtExceptionHandler(new UncaughtExceptionHandler());
        // Intercept JUL loggers
        SLF4JBridgeHandler.removeHandlersForRootLogger();
        SLF4JBridgeHandler.install();
        EnvironmentOptions options = new EnvironmentOptions();
        String[] runnableArgs = OptionsParser.init(options, args, MasterEnvironmentMain.class.getSimpleName(), ProjectInfo.getVersion().toString(), System.out).toArray(new String[0]);
        String runnableClass = options.getRunnableClass();
        if (runnableClass == null) {
            throw new IllegalArgumentException("Missing runnable class name");
        }
        CConfiguration cConf = CConfiguration.create();
        SConfiguration sConf = SConfiguration.create();
        if (options.getExtraConfPath() != null) {
            cConf.addResource(new File(options.getExtraConfPath(), "cdap-site.xml").toURI().toURL());
            sConf.addResource(new File(options.getExtraConfPath(), "cdap-security.xml").toURI().toURL());
        }
        SecurityUtil.loginForMasterService(cConf);
        Configuration hConf = new Configuration();
        // Creates the master environment and load the MasterEnvironmentRunnable class from it.
        MasterEnvironment masterEnv = MasterEnvironments.setMasterEnvironment(MasterEnvironments.create(cConf, options.getEnvProvider()));
        MasterEnvironmentContext context = MasterEnvironments.createContext(cConf, hConf, masterEnv.getName());
        masterEnv.initialize(context);
        try {
            Class<?> cls = masterEnv.getClass().getClassLoader().loadClass(runnableClass);
            if (!MasterEnvironmentRunnable.class.isAssignableFrom(cls)) {
                throw new IllegalArgumentException("Runnable class " + runnableClass + " is not an instance of " + MasterEnvironmentRunnable.class);
            }
            RemoteClientFactory remoteClientFactory = new RemoteClientFactory(masterEnv.getDiscoveryServiceClientSupplier().get(), getInternalAuthenticator(cConf), getRemoteAuthenticator(cConf));
            MasterEnvironmentRunnableContext runnableContext = new DefaultMasterEnvironmentRunnableContext(context.getLocationFactory(), remoteClientFactory);
            @SuppressWarnings("unchecked") MasterEnvironmentRunnable runnable = masterEnv.createRunnable(runnableContext, (Class<? extends MasterEnvironmentRunnable>) cls);
            AtomicBoolean completed = new AtomicBoolean();
            Runtime.getRuntime().addShutdownHook(new Thread(() -> {
                if (!completed.get()) {
                    runnable.stop();
                    Uninterruptibles.awaitUninterruptibly(shutdownLatch, 30, TimeUnit.SECONDS);
                }
                Optional.ofNullable(tokenManager).ifPresent(TokenManager::stopAndWait);
            }));
            runnable.run(runnableArgs);
            completed.set(true);
        } finally {
            masterEnv.destroy();
        }
    } catch (Exception e) {
        LOG.error("Failed to execute with arguments {}", Arrays.toString(args), e);
        throw e;
    } finally {
        shutdownLatch.countDown();
    }
}

Example 3 with MasterEnvironmentContext

use of io.cdap.cdap.master.spi.environment.MasterEnvironmentContext in project cdap by caskdata.

the class PodKillerTaskTest method test.

@Test
public void test() {
    PodKillerTask task = new PodKillerTask("default", "cdap.container=preview", 1000);
    task.run(new MasterEnvironmentContext() {

        @Override
        public LocationFactory getLocationFactory() {
            throw new UnsupportedOperationException("LocationFactory is not supported");
        }

        @Override
        public Map<String, String> getConfigurations() {
            return Collections.emptyMap();
        }

        @Override
        public String[] getRunnableArguments(Class<? extends MasterEnvironmentRunnable> runnableClass, String... runnableArgs) {
            throw new UnsupportedOperationException("Master environment runnable execution is not supported");
        }
    });
}

Example 4 with MasterEnvironmentContext

use of io.cdap.cdap.master.spi.environment.MasterEnvironmentContext in project cdap by caskdata.

the class SparkContainerDriverLauncher method createArtifactLocalizerClient.

private static ArtifactLocalizerClient createArtifactLocalizerClient(CConfiguration cConf, Configuration hConf) throws Exception {
    MasterEnvironment masterEnv = MasterEnvironments.create(cConf, "k8s");
    if (masterEnv == null) {
        throw new RuntimeException("Unable to initialize k8s masterEnv from cConf.");
    }
    MasterEnvironmentContext context = MasterEnvironments.createContext(cConf, hConf, masterEnv.getName());
    masterEnv.initialize(context);
    MasterEnvironments.setMasterEnvironment(masterEnv);
    Injector injector = createInjector(cConf, hConf, masterEnv);
    return injector.getInstance(ArtifactLocalizerClient.class);
}

Example 5 with MasterEnvironmentContext

use of io.cdap.cdap.master.spi.environment.MasterEnvironmentContext in project cdap by caskdata.

the class SparkRuntimeContextProvider method createIfNotExists.

/**
 * Creates a singleton {@link SparkRuntimeContext}.
 * It has assumption on file location that are localized by the SparkRuntimeService.
 */
private static synchronized SparkRuntimeContext createIfNotExists() {
    if (sparkRuntimeContext != null) {
        return sparkRuntimeContext;
    }
    try {
        CConfiguration cConf = createCConf();
        Configuration hConf = createHConf();
        SparkRuntimeContextConfig contextConfig = new SparkRuntimeContextConfig(hConf);
        ProgramOptions programOptions = contextConfig.getProgramOptions();
        ClusterMode clusterMode = ProgramRunners.getClusterMode(programOptions);
        if (masterEnvName != null) {
            MasterEnvironment masterEnv = MasterEnvironments.create(cConf, masterEnvName);
            MasterEnvironmentContext context = MasterEnvironments.createContext(cConf, hConf, masterEnv.getName());
            masterEnv.initialize(context);
            MasterEnvironments.setMasterEnvironment(masterEnv);
        }
        // Create the program
        Program program = createProgram(cConf, contextConfig);
        ProgramRunId programRunId = program.getId().run(ProgramRunners.getRunId(programOptions));
        Injector injector = createInjector(cConf, hConf, contextConfig.getProgramId(), programOptions);
        LogAppenderInitializer logAppenderInitializer = injector.getInstance(LogAppenderInitializer.class);
        logAppenderInitializer.initialize();
        SystemArguments.setLogLevel(programOptions.getUserArguments(), logAppenderInitializer);
        ProxySelector oldProxySelector = ProxySelector.getDefault();
        if (clusterMode == ClusterMode.ISOLATED) {
            RuntimeMonitors.setupMonitoring(injector, programOptions);
        }
        MetricsCollectionService metricsCollectionService = injector.getInstance(MetricsCollectionService.class);
        SparkServiceAnnouncer serviceAnnouncer = injector.getInstance(SparkServiceAnnouncer.class);
        Deque<Service> coreServices = new LinkedList<>();
        if (clusterMode == ClusterMode.ON_PREMISE && masterEnvName == null) {
            // Add ZK for discovery and Kafka
            coreServices.add(injector.getInstance(ZKClientService.class));
            // Add the Kafka client for logs collection
            coreServices.add(injector.getInstance(KafkaClientService.class));
        }
        coreServices.add(metricsCollectionService);
        coreServices.add(serviceAnnouncer);
        for (Service coreService : coreServices) {
            coreService.startAndWait();
        }
        AtomicBoolean closed = new AtomicBoolean();
        Closeable closeable = () -> {
            if (!closed.compareAndSet(false, true)) {
                return;
            }
            // Close to flush out all important logs
            logAppenderInitializer.close();
            // Stop all services. Reverse the order.
            for (Service service : (Iterable<Service>) coreServices::descendingIterator) {
                try {
                    service.stopAndWait();
                } catch (Exception e) {
                    LOG.warn("Exception raised when stopping service {} during program termination.", service, e);
                }
            }
            Authenticator.setDefault(null);
            ProxySelector.setDefault(oldProxySelector);
            LOG.debug("Spark runtime services shutdown completed");
        };
        // Constructor the DatasetFramework
        DatasetFramework datasetFramework = injector.getInstance(DatasetFramework.class);
        WorkflowProgramInfo workflowInfo = contextConfig.getWorkflowProgramInfo();
        DatasetFramework programDatasetFramework = workflowInfo == null ? datasetFramework : NameMappedDatasetFramework.createFromWorkflowProgramInfo(datasetFramework, workflowInfo, contextConfig.getApplicationSpecification());
        // Setup dataset framework context, if required
        if (programDatasetFramework instanceof ProgramContextAware) {
            ((ProgramContextAware) programDatasetFramework).setContext(new BasicProgramContext(programRunId));
        }
        PluginInstantiator pluginInstantiator = createPluginInstantiator(cConf, contextConfig, program.getClassLoader());
        // Create the context object
        sparkRuntimeContext = new SparkRuntimeContext(contextConfig.getConfiguration(), program, programOptions, cConf, getHostname(), injector.getInstance(TransactionSystemClient.class), programDatasetFramework, metricsCollectionService, contextConfig.getWorkflowProgramInfo(), pluginInstantiator, injector.getInstance(SecureStore.class), injector.getInstance(SecureStoreManager.class), injector.getInstance(AccessEnforcer.class), injector.getInstance(AuthenticationContext.class), injector.getInstance(MessagingService.class), serviceAnnouncer, injector.getInstance(PluginFinder.class), injector.getInstance(LocationFactory.class), injector.getInstance(MetadataReader.class), injector.getInstance(MetadataPublisher.class), injector.getInstance(NamespaceQueryAdmin.class), injector.getInstance(FieldLineageWriter.class), injector.getInstance(RemoteClientFactory.class), closeable);
        LoggingContextAccessor.setLoggingContext(sparkRuntimeContext.getLoggingContext());
        return sparkRuntimeContext;
    } catch (Exception e) {
        throw Throwables.propagate(e);
    }
}