Examples with MasterEnvironment io.cdap.cdap.master.spi.environment.MasterEnvironment used on opensource projects

Example 1 with MasterEnvironment

use of io.cdap.cdap.master.spi.environment.MasterEnvironment in project cdap by caskdata.

the class PreviewRunnerTwillRunnable method createInjector.

@VisibleForTesting
static Injector createInjector(CConfiguration cConf, Configuration hConf, PreviewRequestPollerInfo pollerInfo) {
    List<Module> modules = new ArrayList<>();
    byte[] pollerInfoBytes = Bytes.toBytes(new Gson().toJson(pollerInfo));
    SConfiguration sConf = SConfiguration.create();
    modules.add(new ConfigModule(cConf, hConf, sConf));
    modules.add(RemoteAuthenticatorModules.getDefaultModule());
    modules.add(new PreviewConfigModule(cConf, hConf, sConf));
    modules.add(new IOModule());
    modules.add(new MetricsClientRuntimeModule().getDistributedModules());
    // If MasterEnvironment is not available, assuming it is the old hadoop stack with ZK, Kafka
    MasterEnvironment masterEnv = MasterEnvironments.getMasterEnvironment();
    if (masterEnv == null) {
        modules.add(new ZKClientModule());
        modules.add(new ZKDiscoveryModule());
        modules.add(new KafkaClientModule());
        modules.add(new KafkaLogAppenderModule());
    } else {
        modules.add(new AbstractModule() {

            @Override
            protected void configure() {
                bind(DiscoveryService.class).toProvider(new SupplierProviderBridge<>(masterEnv.getDiscoveryServiceSupplier()));
                bind(DiscoveryServiceClient.class).toProvider(new SupplierProviderBridge<>(masterEnv.getDiscoveryServiceClientSupplier()));
            }
        });
        modules.add(new RemoteLogAppenderModule());
    }
    modules.add(new PreviewRunnerManagerModule().getDistributedModules());
    modules.add(new DataSetServiceModules().getStandaloneModules());
    modules.add(new DataSetsModules().getStandaloneModules());
    modules.add(new AppFabricServiceRuntimeModule(cConf).getStandaloneModules());
    modules.add(new ProgramRunnerRuntimeModule().getStandaloneModules());
    modules.add(new MetricsStoreModule());
    modules.add(new MessagingClientModule());
    modules.add(new AuditModule());
    modules.add(new SecureStoreClientModule());
    modules.add(new MetadataReaderWriterModules().getStandaloneModules());
    modules.add(new DFSLocationModule());
    modules.add(new MetadataServiceModule());
    modules.add(new CoreSecurityRuntimeModule().getInMemoryModules());
    modules.add(new AuthenticationContextModules().getMasterWorkerModule());
    modules.add(new AuthorizationModule());
    modules.add(new AuthorizationEnforcementModule().getNoOpModules());
    modules.add(Modules.override(new DataFabricModules("master").getDistributedModules()).with(new AbstractModule() {

        @Override
        protected void configure() {
            // Bind transaction system to a constant one, basically no transaction, with every write become
            // visible immediately.
            // TODO: Ideally we shouldn't need this at all. However, it is needed now to satisfy dependencies
            bind(TransactionSystemClientService.class).to(DelegatingTransactionSystemClientService.class);
            bind(TransactionSystemClient.class).to(ConstantTransactionSystemClient.class);
            bind(ExploreClient.class).to(UnsupportedExploreClient.class);
            bind(PreviewRequestPollerInfoProvider.class).toInstance(() -> pollerInfoBytes);
        }
    }));
    return Guice.createInjector(modules);
}

Example 2 with MasterEnvironment

use of io.cdap.cdap.master.spi.environment.MasterEnvironment in project cdap by caskdata.

the class MasterEnvironmentMain method doMain.

/**
 * The actual main method that get invoke through reflection from the {@link #main(String[])} method.
 */
@SuppressWarnings("unused")
public static void doMain(String[] args) throws Exception {
    CountDownLatch shutdownLatch = new CountDownLatch(1);
    try {
        // System wide setup
        Thread.setDefaultUncaughtExceptionHandler(new UncaughtExceptionHandler());
        // Intercept JUL loggers
        SLF4JBridgeHandler.removeHandlersForRootLogger();
        SLF4JBridgeHandler.install();
        EnvironmentOptions options = new EnvironmentOptions();
        String[] runnableArgs = OptionsParser.init(options, args, MasterEnvironmentMain.class.getSimpleName(), ProjectInfo.getVersion().toString(), System.out).toArray(new String[0]);
        String runnableClass = options.getRunnableClass();
        if (runnableClass == null) {
            throw new IllegalArgumentException("Missing runnable class name");
        }
        CConfiguration cConf = CConfiguration.create();
        SConfiguration sConf = SConfiguration.create();
        if (options.getExtraConfPath() != null) {
            cConf.addResource(new File(options.getExtraConfPath(), "cdap-site.xml").toURI().toURL());
            sConf.addResource(new File(options.getExtraConfPath(), "cdap-security.xml").toURI().toURL());
        }
        SecurityUtil.loginForMasterService(cConf);
        Configuration hConf = new Configuration();
        // Creates the master environment and load the MasterEnvironmentRunnable class from it.
        MasterEnvironment masterEnv = MasterEnvironments.setMasterEnvironment(MasterEnvironments.create(cConf, options.getEnvProvider()));
        MasterEnvironmentContext context = MasterEnvironments.createContext(cConf, hConf, masterEnv.getName());
        masterEnv.initialize(context);
        try {
            Class<?> cls = masterEnv.getClass().getClassLoader().loadClass(runnableClass);
            if (!MasterEnvironmentRunnable.class.isAssignableFrom(cls)) {
                throw new IllegalArgumentException("Runnable class " + runnableClass + " is not an instance of " + MasterEnvironmentRunnable.class);
            }
            RemoteClientFactory remoteClientFactory = new RemoteClientFactory(masterEnv.getDiscoveryServiceClientSupplier().get(), getInternalAuthenticator(cConf), getRemoteAuthenticator(cConf));
            MasterEnvironmentRunnableContext runnableContext = new DefaultMasterEnvironmentRunnableContext(context.getLocationFactory(), remoteClientFactory);
            @SuppressWarnings("unchecked") MasterEnvironmentRunnable runnable = masterEnv.createRunnable(runnableContext, (Class<? extends MasterEnvironmentRunnable>) cls);
            AtomicBoolean completed = new AtomicBoolean();
            Runtime.getRuntime().addShutdownHook(new Thread(() -> {
                if (!completed.get()) {
                    runnable.stop();
                    Uninterruptibles.awaitUninterruptibly(shutdownLatch, 30, TimeUnit.SECONDS);
                }
                Optional.ofNullable(tokenManager).ifPresent(TokenManager::stopAndWait);
            }));
            runnable.run(runnableArgs);
            completed.set(true);
        } finally {
            masterEnv.destroy();
        }
    } catch (Exception e) {
        LOG.error("Failed to execute with arguments {}", Arrays.toString(args), e);
        throw e;
    } finally {
        shutdownLatch.countDown();
    }
}

Example 3 with MasterEnvironment

use of io.cdap.cdap.master.spi.environment.MasterEnvironment in project cdap by caskdata.

the class MasterEnvironments method create.

/**
 * Creates a new instance of {@link MasterEnvironment}.
 *
 * @param cConf the CDAP configuration
 * @param envName the master environment name
 * @return a new, initialized instance
 * @throws NotFoundException if the master environment of the given name does not exist
 */
public static MasterEnvironment create(CConfiguration cConf, String envName) throws NotFoundException {
    MasterEnvironmentExtensionLoader loader = new MasterEnvironmentExtensionLoader(cConf);
    MasterEnvironment masterEnv = loader.get(envName);
    if (masterEnv == null) {
        throw new NotFoundException("Master environment of name " + envName + " does not exist");
    }
    return masterEnv;
}

Example 4 with MasterEnvironment

use of io.cdap.cdap.master.spi.environment.MasterEnvironment in project cdap by caskdata.

the class DistributedProgramContainerModule method addOnPremiseModules.

private void addOnPremiseModules(List<Module> modules) {
    CoreSecurityModule coreSecurityModule = CoreSecurityRuntimeModule.getDistributedModule(cConf);
    modules.add(new AuthenticationContextModules().getMasterModule());
    modules.add(coreSecurityModule);
    // If MasterEnvironment is not available, assuming it is the old hadoop stack with ZK, Kafka
    MasterEnvironment masterEnv = MasterEnvironments.getMasterEnvironment();
    if (masterEnv == null) {
        modules.add(new ZKClientModule());
        modules.add(new ZKDiscoveryModule());
        modules.add(new KafkaClientModule());
        modules.add(new KafkaLogAppenderModule());
        return;
    }
    if (coreSecurityModule.requiresZKClient()) {
        modules.add(new ZKClientModule());
    }
    modules.add(new AbstractModule() {

        @Override
        protected void configure() {
            bind(DiscoveryService.class).toProvider(new SupplierProviderBridge<>(masterEnv.getDiscoveryServiceSupplier()));
            bind(DiscoveryServiceClient.class).toProvider(new SupplierProviderBridge<>(masterEnv.getDiscoveryServiceClientSupplier()));
            bind(OwnerAdmin.class).to(DefaultOwnerAdmin.class);
        }
    });
    modules.add(new RemoteLogAppenderModule());
}

Example 5 with MasterEnvironment

use of io.cdap.cdap.master.spi.environment.MasterEnvironment in project cdap by caskdata.

the class TaskWorkerTwillRunnable method createInjector.

@VisibleForTesting
static Injector createInjector(CConfiguration cConf, Configuration hConf) {
    List<Module> modules = new ArrayList<>();
    CoreSecurityModule coreSecurityModule = CoreSecurityRuntimeModule.getDistributedModule(cConf);
    modules.add(new ConfigModule(cConf, hConf));
    modules.add(RemoteAuthenticatorModules.getDefaultModule());
    modules.add(new LocalLocationModule());
    modules.add(new IOModule());
    modules.add(new AuthenticationContextModules().getMasterWorkerModule());
    modules.add(coreSecurityModule);
    modules.add(new MessagingClientModule());
    modules.add(new SystemAppModule());
    modules.add(new MetricsClientRuntimeModule().getDistributedModules());
    // If MasterEnvironment is not available, assuming it is the old hadoop stack with ZK, Kafka
    MasterEnvironment masterEnv = MasterEnvironments.getMasterEnvironment();
    if (masterEnv == null) {
        modules.add(new ZKClientModule());
        modules.add(new ZKDiscoveryModule());
        modules.add(new KafkaClientModule());
        modules.add(new KafkaLogAppenderModule());
    } else {
        modules.add(new AbstractModule() {

            @Override
            protected void configure() {
                bind(DiscoveryService.class).toProvider(new SupplierProviderBridge<>(masterEnv.getDiscoveryServiceSupplier()));
                bind(DiscoveryServiceClient.class).toProvider(new SupplierProviderBridge<>(masterEnv.getDiscoveryServiceClientSupplier()));
            }
        });
        modules.add(new RemoteLogAppenderModule());
        if (coreSecurityModule.requiresZKClient()) {
            modules.add(new ZKClientModule());
        }
    }
    return Guice.createInjector(modules);
}