Examples with CConfiguration io.cdap.cdap.common.conf.CConfiguration used on opensource projects

Example 6 with CConfiguration

use of io.cdap.cdap.common.conf.CConfiguration in project cdap by caskdata.

the class TestBase method createCConf.

private static CConfiguration createCConf(File localDataDir) throws IOException {
    CConfiguration cConf = CConfiguration.create();
    // Setup defaults that can be overridden by user
    cConf.setBoolean(Constants.Explore.EXPLORE_ENABLED, true);
    cConf.setBoolean(Constants.Explore.START_ON_DEMAND, false);
    cConf.set(Constants.AppFabric.SYSTEM_ARTIFACTS_DIR, "");
    // Set this to artificially big value to effectively disable SystemProgramManagementService
    cConf.setLong(Constants.AppFabric.SYSTEM_PROGRAM_SCAN_INTERVAL_SECONDS, 3600 * 24 * 30);
    // The system properties are usually setup by TestConfiguration class using @ClassRule
    for (String key : System.getProperties().stringPropertyNames()) {
        if (key.startsWith(TestConfiguration.PROPERTY_PREFIX)) {
            String value = System.getProperty(key);
            cConf.set(key.substring(TestConfiguration.PROPERTY_PREFIX.length()), System.getProperty(key));
            LOG.info("Custom configuration set: {} = {}", key, value);
        }
    }
    // These configurations cannot be overridden by user
    // configure all services except for router to bind to localhost
    String localhost = InetAddress.getLoopbackAddress().getHostAddress();
    cConf.set(Constants.Service.MASTER_SERVICES_BIND_ADDRESS, localhost);
    cConf.set(Constants.MessagingSystem.HTTP_SERVER_BIND_ADDRESS, localhost);
    cConf.set(Constants.Transaction.Container.ADDRESS, localhost);
    cConf.set(Constants.Dataset.Executor.ADDRESS, localhost);
    cConf.set(Constants.Metrics.ADDRESS, localhost);
    cConf.set(Constants.MetricsProcessor.BIND_ADDRESS, localhost);
    cConf.set(Constants.LogSaver.ADDRESS, localhost);
    cConf.set(Constants.Security.AUTH_SERVER_BIND_ADDRESS, localhost);
    cConf.set(Constants.Explore.SERVER_ADDRESS, localhost);
    cConf.set(Constants.Metadata.SERVICE_BIND_ADDRESS, localhost);
    cConf.set(Constants.Preview.ADDRESS, localhost);
    cConf.set(Constants.SupportBundle.SERVICE_BIND_ADDRESS, localhost);
    cConf.set(Constants.CFG_LOCAL_DATA_DIR, localDataDir.getAbsolutePath());
    cConf.setBoolean(Constants.Dangerous.UNRECOVERABLE_RESET, true);
    cConf.set(Constants.Explore.LOCAL_DATA_DIR, TMP_FOLDER.newFolder("hive").getAbsolutePath());
    // Speed up test
    cConf.setLong(Constants.Scheduler.EVENT_POLL_DELAY_MILLIS, 100L);
    cConf.setLong(Constants.AppFabric.STATUS_EVENT_POLL_DELAY_MILLIS, 100L);
    // Set spark compat
    cConf.set(Constants.AppFabric.SPARK_COMPAT, getCurrentSparkCompat().getCompat());
    return cConf;
}

Example 7 with CConfiguration

use of io.cdap.cdap.common.conf.CConfiguration in project cdap by caskdata.

the class RemoteExecutionJobMain method doMain.

/**
 * The main method for the Application. It expects the first argument contains the program run id.
 *
 * @param args arguments provided to the application
 * @throws Exception if failed to the the job
 */
private void doMain(String[] args) throws Exception {
    Map<String, String> arguments = RuntimeArguments.fromPosixArray(args);
    if (!arguments.containsKey(RUN_ID)) {
        throw new IllegalArgumentException("Missing argument " + RUN_ID);
    }
    if (UserGroupInformation.isSecurityEnabled()) {
        String principal = arguments.get(ClusterProperties.KERBEROS_PRINCIPAL);
        String keytab = arguments.get(ClusterProperties.KERBEROS_KEYTAB);
        if (principal != null && keytab != null) {
            LOG.info("Login with kerberos principal {} and keytab {}", principal, keytab);
            UserGroupInformation.loginUserFromKeytab(principal, keytab);
        } else {
            LOG.warn("Security enabled but no kerberos principal and keytab are provided");
        }
    }
    // Stop the job when this process get terminated
    Runtime.getRuntime().addShutdownHook(new Thread(runtimeJob::requestStop));
    System.setProperty(Constants.Zookeeper.TWILL_ZK_SERVER_LOCALHOST, "false");
    RunId runId = RunIds.fromString(arguments.get(RUN_ID));
    CConfiguration cConf = CConfiguration.create();
    // Not support explore functionality in remote execution
    cConf.setBoolean(Constants.Explore.EXPLORE_ENABLED, false);
    // Namespace the HDFS on the current cluster to segregate multiple runs on the same cluster
    cConf.set(Constants.CFG_HDFS_NAMESPACE, "/twill-" + runId);
    try {
        RemoteExecutionRuntimeJobEnvironment jobEnv = initialize(cConf);
        runtimeJob.run(jobEnv);
    } finally {
        destroy();
    }
}

Example 8 with CConfiguration

use of io.cdap.cdap.common.conf.CConfiguration in project cdap by caskdata.

the class AbstractProgramTwillRunnable method doInitialize.

/**
 * Prepares this instance to execute a program.
 *
 * @param programOptionFile a json file containing the serialized {@link ProgramOptions}
 * @throws Exception if failed to initialize
 */
private void doInitialize(File programOptionFile) throws Exception {
    controllerFuture = new CompletableFuture<>();
    programCompletion = new CompletableFuture<>();
    // Setup process wide settings
    Thread.setDefaultUncaughtExceptionHandler(new UncaughtExceptionHandler());
    System.setSecurityManager(new ProgramContainerSecurityManager(System.getSecurityManager()));
    SLF4JBridgeHandler.removeHandlersForRootLogger();
    SLF4JBridgeHandler.install();
    // Create the ProgramOptions
    programOptions = createProgramOptions(programOptionFile);
    programRunId = programOptions.getProgramId().run(ProgramRunners.getRunId(programOptions));
    Arguments systemArgs = programOptions.getArguments();
    LoggingContextAccessor.setLoggingContext(LoggingContextHelper.getLoggingContextWithRunId(programRunId, systemArgs.asMap()));
    ClusterMode clusterMode = ProgramRunners.getClusterMode(programOptions);
    // Loads configurations
    Configuration hConf = new Configuration();
    if (clusterMode == ClusterMode.ON_PREMISE) {
        hConf.clear();
        hConf.addResource(new File(systemArgs.getOption(ProgramOptionConstants.HADOOP_CONF_FILE)).toURI().toURL());
    }
    UserGroupInformation.setConfiguration(hConf);
    CConfiguration cConf = CConfiguration.create();
    cConf.clear();
    cConf.addResource(new File(systemArgs.getOption(ProgramOptionConstants.CDAP_CONF_FILE)).toURI().toURL());
    maxStopSeconds = cConf.getLong(io.cdap.cdap.common.conf.Constants.AppFabric.PROGRAM_MAX_STOP_SECONDS);
    Injector injector = Guice.createInjector(createModule(cConf, hConf, programOptions, programRunId));
    // Initialize log appender
    logAppenderInitializer = injector.getInstance(LogAppenderInitializer.class);
    logAppenderInitializer.initialize();
    SystemArguments.setLogLevel(programOptions.getUserArguments(), logAppenderInitializer);
    // Setup the proxy selector for in active monitoring mode
    oldProxySelector = ProxySelector.getDefault();
    if (clusterMode == ClusterMode.ISOLATED) {
        RuntimeMonitors.setupMonitoring(injector, programOptions);
    }
    // Create list of core services. They'll will be started in the run method and shutdown when the run
    // method completed
    coreServices = createCoreServices(injector, programOptions);
    // Create the ProgramRunner
    programRunner = createProgramRunner(injector);
    // Create the Program instance
    Location programJarLocation = Locations.toLocation(new File(systemArgs.getOption(ProgramOptionConstants.PROGRAM_JAR)));
    ApplicationSpecification appSpec = readJsonFile(new File(systemArgs.getOption(ProgramOptionConstants.APP_SPEC_FILE)), ApplicationSpecification.class);
    // Expand the program jar for creating classloader
    ClassLoaderFolder classLoaderFolder = BundleJarUtil.prepareClassLoaderFolder(programJarLocation, () -> new File("expanded." + System.currentTimeMillis() + programJarLocation.getName()));
    program = Programs.create(cConf, programRunner, new ProgramDescriptor(programOptions.getProgramId(), appSpec), programJarLocation, classLoaderFolder.getDir());
}

Example 9 with CConfiguration

use of io.cdap.cdap.common.conf.CConfiguration in project cdap by caskdata.

the class DistributedProgramRunner method createContainerCConf.

/**
 * Creates the {@link CConfiguration} to be used in the program container.
 */
private CConfiguration createContainerCConf(CConfiguration cConf) {
    CConfiguration result = CConfiguration.copy(cConf);
    // don't have tephra retry in order to give CDAP more control over when to retry and how.
    result.set(TxConstants.Service.CFG_DATA_TX_CLIENT_RETRY_STRATEGY, "n-times");
    result.setInt(TxConstants.Service.CFG_DATA_TX_CLIENT_ATTEMPTS, 0);
    // Unset the hbase.client.retries.number and hbase.rpc.timeout so that program container
    // runs with default values for them from hbase-site/hbase-default.
    result.unset(Constants.HBase.CLIENT_RETRIES);
    result.unset(Constants.HBase.RPC_TIMEOUT);
    // Unset the runtime extension directory as the necessary extension jars should be shipped to the container
    // by the distributed ProgramRunner.
    result.unset(Constants.AppFabric.RUNTIME_EXT_DIR);
    // Set the CFG_LOCAL_DATA_DIR to a relative path as the data directory for the container should be relative to the
    // container directory
    result.set(Constants.CFG_LOCAL_DATA_DIR, "data");
    // Setup the configurations for isolated mode
    if (clusterMode == ClusterMode.ISOLATED) {
        // Disable logs collection through twill
        result.set(Constants.COLLECT_APP_CONTAINER_LOG_LEVEL, "OFF");
        // Disable implicit transaction
        result.set(Constants.AppFabric.PROGRAM_TRANSACTION_CONTROL, TransactionControl.EXPLICIT.name());
        // Disable transaction support
        result.setBoolean(Constants.Transaction.TX_ENABLED, false);
        // Disable explore
        result.set(Constants.Explore.EXPLORE_ENABLED, Boolean.FALSE.toString());
        // Always use NoSQL as storage
        result.set(Constants.Dataset.DATA_STORAGE_IMPLEMENTATION, Constants.Dataset.DATA_STORAGE_NOSQL);
        // The following services will be running in the edge node host.
        // Set the bind addresses for all of them to "${master.services.bind.address}"
        // Which the value of `"master.services.bind.address" will be set in the DefaultRuntimeJob when it
        // starts in the remote machine
        String masterBindAddrConf = "${" + Constants.Service.MASTER_SERVICES_BIND_ADDRESS + "}";
        result.set(Constants.MessagingSystem.HTTP_SERVER_BIND_ADDRESS, masterBindAddrConf);
        // Don't try and use the CDAP system certs for SSL
        result.unset(Constants.Security.SSL.INTERNAL_CERT_PATH);
    }
    return result;
}

Example 10 with CConfiguration

use of io.cdap.cdap.common.conf.CConfiguration in project cdap by caskdata.

the class DefaultRuntimeJob method createCConf.

/**
 * Create {@link CConfiguration} with the given {@link RuntimeJobEnvironment}.
 * Properties returned by the {@link RuntimeJobEnvironment#getProperties()} will be set into the returned
 * {@link CConfiguration} instance.
 */
private CConfiguration createCConf(RuntimeJobEnvironment runtimeJobEnv, ProgramOptions programOpts) throws IOException {
    CConfiguration cConf = CConfiguration.create();
    cConf.clear();
    cConf.addResource(new File(DistributedProgramRunner.CDAP_CONF_FILE_NAME).toURI().toURL());
    for (Map.Entry<String, String> entry : runtimeJobEnv.getProperties().entrySet()) {
        cConf.set(entry.getKey(), entry.getValue());
    }
    cConf.setBoolean(Constants.AppFabric.PROGRAM_REMOTE_RUNNER, true);
    String hostName = InetAddress.getLocalHost().getCanonicalHostName();
    cConf.set(Constants.Service.MASTER_SERVICES_BIND_ADDRESS, hostName);
    // set the password into the cConf so that it can be used in the distributed jobs launched by this process.
    if (SystemArguments.getRuntimeMonitorType(cConf, programOpts) == RuntimeMonitorType.SSH) {
        Path serviceProxySecretFile = Paths.get(Constants.RuntimeMonitor.SERVICE_PROXY_PASSWORD_FILE);
        if (Files.exists(serviceProxySecretFile)) {
            cConf.set(Constants.RuntimeMonitor.SERVICE_PROXY_PASSWORD, new String(Files.readAllBytes(serviceProxySecretFile), StandardCharsets.UTF_8));
        }
    }
    // Pass runtime token to distributed jobs
    Path tokenFile = Paths.get(Constants.Security.Authentication.RUNTIME_TOKEN_FILE);
    if (Files.exists(tokenFile)) {
        cConf.set(Constants.Security.Authentication.RUNTIME_TOKEN, new String(Files.readAllBytes(tokenFile), StandardCharsets.UTF_8));
    }
    return cConf;
}