Examples with SecretDisk io.cdap.cdap.master.spi.twill.SecretDisk used on opensource projects

Example 1 with SecretDisk

use of io.cdap.cdap.master.spi.twill.SecretDisk in project cdap by caskdata.

the class DistributedPreviewManager method run.

@Override
public void run() {
    TwillController activeController = null;
    for (TwillController controller : twillRunner.lookup(PreviewRunnerTwillApplication.NAME)) {
        // If detected more than one controller, terminate those extra controllers.
        if (activeController != null) {
            controller.terminate();
        } else {
            activeController = controller;
        }
    }
    // If there is no preview runner running, create one
    if (activeController == null) {
        try {
            Path tmpDir = new File(cConf.get(Constants.CFG_LOCAL_DATA_DIR), cConf.get(Constants.AppFabric.TEMP_DIR)).toPath();
            Files.createDirectories(tmpDir);
            Path runDir = Files.createTempDirectory(tmpDir, "preview");
            try {
                CConfiguration cConfCopy = CConfiguration.copy(cConf);
                Path cConfPath = runDir.resolve("cConf.xml");
                if (!cConf.getBoolean(Constants.Twill.Security.WORKER_MOUNT_SECRET)) {
                    // Unset the internal certificate path since certificate is stored cdap-security which
                    // is not going to be exposed to preview runner.
                    // TODO: CDAP-18768 this will break preview when certificate checking is enabled.
                    cConfCopy.unset(Constants.Security.SSL.INTERNAL_CERT_PATH);
                }
                try (Writer writer = Files.newBufferedWriter(cConfPath, StandardCharsets.UTF_8)) {
                    cConfCopy.writeXml(writer);
                }
                Path hConfPath = runDir.resolve("hConf.xml");
                try (Writer writer = Files.newBufferedWriter(hConfPath, StandardCharsets.UTF_8)) {
                    hConf.writeXml(writer);
                }
                Boolean artifactLocalizerEnabled = cConf.getBoolean(Constants.Preview.ARTIFACT_LOCALIZER_ENABLED);
                ResourceSpecification runnerResourceSpec = ResourceSpecification.Builder.with().setVirtualCores(cConf.getInt(Constants.Preview.CONTAINER_CORES)).setMemory(cConf.getInt(Constants.Preview.CONTAINER_MEMORY_MB), ResourceSpecification.SizeUnit.MEGA).setInstances(cConf.getInt(Constants.Preview.CONTAINER_COUNT)).build();
                Optional<ResourceSpecification> artifactLocalizerResourceSpec = Optional.empty();
                if (artifactLocalizerEnabled) {
                    artifactLocalizerResourceSpec = Optional.of(ResourceSpecification.Builder.with().setVirtualCores(cConf.getInt(Constants.ArtifactLocalizer.CONTAINER_CORES)).setMemory(cConf.getInt(Constants.ArtifactLocalizer.CONTAINER_MEMORY_MB), ResourceSpecification.SizeUnit.MEGA).setInstances(cConf.getInt(Constants.TaskWorker.CONTAINER_COUNT)).build());
                }
                LOG.info("Starting preview runners with {} instances and artifactLocalizer {}", runnerResourceSpec.getInstances(), artifactLocalizerEnabled ? "enabled" : "disabled");
                TwillPreparer twillPreparer = twillRunner.prepare(new PreviewRunnerTwillApplication(cConfPath.toUri(), hConfPath.toUri(), runnerResourceSpec, artifactLocalizerResourceSpec));
                String priorityClass = cConf.get(Constants.Preview.CONTAINER_PRIORITY_CLASS_NAME);
                if (priorityClass != null) {
                    twillPreparer = twillPreparer.setSchedulerQueue(priorityClass);
                }
                if (twillPreparer instanceof DependentTwillPreparer) {
                    if (artifactLocalizerEnabled) {
                        twillPreparer = ((DependentTwillPreparer) twillPreparer).dependentRunnableNames(PreviewRunnerTwillRunnable.class.getSimpleName(), ArtifactLocalizerTwillRunnable.class.getSimpleName());
                    }
                }
                if (twillPreparer instanceof StatefulTwillPreparer) {
                    int diskSize = cConf.getInt(Constants.Preview.CONTAINER_DISK_SIZE_GB);
                    twillPreparer = ((StatefulTwillPreparer) twillPreparer).withStatefulRunnable(PreviewRunnerTwillRunnable.class.getSimpleName(), false, new StatefulDisk("preview-runner-data", diskSize, cConf.get(Constants.CFG_LOCAL_DATA_DIR)));
                }
                if (twillPreparer instanceof SecureTwillPreparer) {
                    String twillUserIdentity = cConf.get(Constants.Twill.Security.IDENTITY_USER);
                    if (twillUserIdentity != null) {
                        SecurityContext securityContext = new SecurityContext.Builder().withIdentity(twillUserIdentity).build();
                        twillPreparer = ((SecureTwillPreparer) twillPreparer).withSecurityContext(PreviewRunnerTwillRunnable.class.getSimpleName(), securityContext);
                    }
                    if (artifactLocalizerEnabled) {
                        // Mount secret in ArtifactLocalizer sidecar which only runs trusted code,
                        // so requests originated by ArtifactLocalizer can run with system identity when internal auth
                        // is enabled.
                        String secretName = cConf.get(Constants.Twill.Security.MASTER_SECRET_DISK_NAME);
                        String secretPath = cConf.get(Constants.Twill.Security.MASTER_SECRET_DISK_PATH);
                        twillPreparer = ((SecureTwillPreparer) twillPreparer).withSecretDisk(ArtifactLocalizerTwillRunnable.class.getSimpleName(), new SecretDisk(secretName, secretPath));
                    }
                    if (cConf.getBoolean(Constants.Twill.Security.WORKER_MOUNT_SECRET)) {
                        String secretName = cConf.get(Constants.Twill.Security.WORKER_SECRET_DISK_NAME);
                        String secretPath = cConf.get(Constants.Twill.Security.WORKER_SECRET_DISK_PATH);
                        twillPreparer = ((SecureTwillPreparer) twillPreparer).withSecretDisk(PreviewRunnerTwillRunnable.class.getSimpleName(), new SecretDisk(secretName, secretPath));
                    }
                }
                activeController = twillPreparer.start(5, TimeUnit.MINUTES);
                activeController.onRunning(() -> deleteDir(runDir), Threads.SAME_THREAD_EXECUTOR);
                activeController.onTerminated(() -> deleteDir(runDir), Threads.SAME_THREAD_EXECUTOR);
            } catch (Exception e) {
                deleteDir(runDir);
                throw e;
            }
        } catch (Exception e) {
            LOG.warn("Failed to launch preview runner. It will be retried", e);
        }
    }
    controller = activeController;
}

Example 2 with SecretDisk

use of io.cdap.cdap.master.spi.twill.SecretDisk in project cdap by caskdata.

the class TaskWorkerServiceLauncher method run.

public void run() {
    TwillController activeController = null;
    for (TwillController controller : twillRunner.lookup(TaskWorkerTwillApplication.NAME)) {
        // If detected more than one controller, terminate those extra controllers.
        if (activeController != null) {
            controller.terminate();
        } else {
            activeController = controller;
        }
    }
    // If there is no task worker runner running, create one
    if (activeController == null) {
        try {
            Path tmpDir = new File(cConf.get(Constants.CFG_LOCAL_DATA_DIR), cConf.get(Constants.AppFabric.TEMP_DIR)).toPath();
            Files.createDirectories(tmpDir);
            Path runDir = Files.createTempDirectory(tmpDir, "task.worker.launcher");
            try {
                // Unset the internal certificate path since certificate is stored cdap-security which
                // is not exposed (i.e. mounted in k8s) to TaskWorkerService.
                CConfiguration cConfCopy = CConfiguration.copy(cConf);
                cConfCopy.unset(Constants.Security.SSL.INTERNAL_CERT_PATH);
                Path cConfPath = runDir.resolve("cConf.xml");
                try (Writer writer = Files.newBufferedWriter(cConfPath, StandardCharsets.UTF_8)) {
                    cConfCopy.writeXml(writer);
                }
                Path hConfPath = runDir.resolve("hConf.xml");
                try (Writer writer = Files.newBufferedWriter(hConfPath, StandardCharsets.UTF_8)) {
                    hConf.writeXml(writer);
                }
                ResourceSpecification taskworkerResourceSpec = ResourceSpecification.Builder.with().setVirtualCores(cConf.getInt(Constants.TaskWorker.CONTAINER_CORES)).setMemory(cConf.getInt(Constants.TaskWorker.CONTAINER_MEMORY_MB), ResourceSpecification.SizeUnit.MEGA).setInstances(cConf.getInt(Constants.TaskWorker.CONTAINER_COUNT)).build();
                ResourceSpecification artifactLocalizerResourceSpec = ResourceSpecification.Builder.with().setVirtualCores(cConf.getInt(Constants.ArtifactLocalizer.CONTAINER_CORES)).setMemory(cConf.getInt(Constants.ArtifactLocalizer.CONTAINER_MEMORY_MB), ResourceSpecification.SizeUnit.MEGA).setInstances(cConf.getInt(Constants.TaskWorker.CONTAINER_COUNT)).build();
                LOG.info("Starting TaskWorker pool with {} instances", taskworkerResourceSpec.getInstances());
                TwillPreparer twillPreparer = twillRunner.prepare(new TaskWorkerTwillApplication(cConfPath.toUri(), hConfPath.toUri(), taskworkerResourceSpec, artifactLocalizerResourceSpec));
                String priorityClass = cConf.get(Constants.TaskWorker.CONTAINER_PRIORITY_CLASS_NAME);
                if (priorityClass != null) {
                    twillPreparer = twillPreparer.setSchedulerQueue(priorityClass);
                }
                if (twillPreparer instanceof DependentTwillPreparer) {
                    twillPreparer = ((DependentTwillPreparer) twillPreparer).dependentRunnableNames(TaskWorkerTwillRunnable.class.getSimpleName(), ArtifactLocalizerTwillRunnable.class.getSimpleName());
                }
                if (twillPreparer instanceof StatefulTwillPreparer) {
                    int diskSize = cConf.getInt(Constants.TaskWorker.CONTAINER_DISK_SIZE_GB);
                    twillPreparer = ((StatefulTwillPreparer) twillPreparer).withStatefulRunnable(TaskWorkerTwillRunnable.class.getSimpleName(), false, new StatefulDisk(STATEFUL_DISK_NAME, diskSize, cConf.get(Constants.CFG_LOCAL_DATA_DIR)));
                    if (cConf.getBoolean(Constants.TaskWorker.CONTAINER_DISK_READONLY)) {
                        twillPreparer = ((StatefulTwillPreparer) twillPreparer).withReadonlyDisk(TaskWorkerTwillRunnable.class.getSimpleName(), STATEFUL_DISK_NAME);
                    }
                }
                if (twillPreparer instanceof SecureTwillPreparer) {
                    SecurityContext securityContext = createSecurityContext();
                    twillPreparer = ((SecureTwillPreparer) twillPreparer).withSecurityContext(TaskWorkerTwillRunnable.class.getSimpleName(), securityContext);
                    // Mount secret in ArtifactLocalizer sidecar which only run trusted code,
                    // so requests originated by ArtifactLocalizer can run with system identity when internal auth
                    // is enabled.
                    twillPreparer = ((SecureTwillPreparer) twillPreparer).withSecretDisk(ArtifactLocalizerTwillRunnable.class.getSimpleName(), new SecretDisk(cConf.get(Constants.Twill.Security.MASTER_SECRET_DISK_NAME), cConf.get(Constants.Twill.Security.MASTER_SECRET_DISK_PATH)));
                    if (cConf.getBoolean(Constants.Twill.Security.WORKER_MOUNT_SECRET)) {
                        String secretName = cConf.get(Constants.Twill.Security.WORKER_SECRET_DISK_NAME);
                        String secretPath = cConf.get(Constants.Twill.Security.WORKER_SECRET_DISK_PATH);
                        twillPreparer = ((SecureTwillPreparer) twillPreparer).withSecretDisk(TaskWorkerTwillRunnable.class.getSimpleName(), new SecretDisk(secretName, secretPath));
                    }
                }
                activeController = twillPreparer.start(5, TimeUnit.MINUTES);
                activeController.onRunning(() -> deleteDir(runDir), Threads.SAME_THREAD_EXECUTOR);
                activeController.onTerminated(() -> deleteDir(runDir), Threads.SAME_THREAD_EXECUTOR);
            } catch (Exception e) {
                deleteDir(runDir);
                throw e;
            }
        } catch (Exception e) {
            LOG.warn(String.format("Failed to launch TaskWorker pool, retry in %d", cConf.getInt(Constants.TaskWorker.POOL_CHECK_INTERVAL)), e);
        }
    }
    this.twillController = activeController;
}

Example 3 with SecretDisk

use of io.cdap.cdap.master.spi.twill.SecretDisk in project cdap by caskdata.

the class KubeTwillPreparer method addSecreteVolMountIfNeeded.

private List<V1VolumeMount> addSecreteVolMountIfNeeded(RuntimeSpecification runtimeSpec, List<V1VolumeMount> mounts) {
    List<V1VolumeMount> updatedMounts = new ArrayList<>(mounts);
    // Add secret disks as secret volume mounts
    if (secretDiskRunnables.containsKey(runtimeSpec.getName())) {
        for (SecretDisk secretDisk : secretDiskRunnables.get(runtimeSpec.getName()).getSecretDisks()) {
            String secretName = secretDisk.getName();
            String mountPath = secretDisk.getPath();
            updatedMounts.add(new V1VolumeMount().name(secretName).mountPath(mountPath).readOnly(true));
        }
    }
    return updatedMounts;
}

Example 4 with SecretDisk

use of io.cdap.cdap.master.spi.twill.SecretDisk in project cdap by caskdata.

the class DistributedProgramRunner method run.

@Override
public final ProgramController run(final Program program, ProgramOptions oldOptions) {
    validateOptions(program, oldOptions);
    CConfiguration cConf = CConfiguration.copy(this.cConf);
    // Reload config for log extension jar update (CDAP-15091)
    cConf.reloadConfiguration();
    File tempDir = DirUtils.createTempDir(new File(cConf.get(Constants.CFG_LOCAL_DATA_DIR), cConf.get(Constants.AppFabric.TEMP_DIR)).getAbsoluteFile());
    try {
        ProgramLaunchConfig launchConfig = new ProgramLaunchConfig();
        if (clusterMode == ClusterMode.ISOLATED) {
            // For isolated mode, the hadoop classes comes from the hadoop classpath in the target cluster directly
            launchConfig.addExtraClasspath(Collections.singletonList("$HADOOP_CLASSPATH"));
        }
        setupLaunchConfig(launchConfig, program, oldOptions, cConf, hConf, tempDir);
        // Add extra localize resources needed by the program runner
        final Map<String, LocalizeResource> localizeResources = new HashMap<>(launchConfig.getExtraResources());
        final List<String> additionalClassPaths = new ArrayList<>();
        addContainerJars(cConf, localizeResources, additionalClassPaths);
        addAdditionalLogAppenderJars(cConf, tempDir, localizeResources, SystemArguments.getProfileProvisioner(oldOptions.getArguments().asMap()));
        prepareHBaseDDLExecutorResources(tempDir, cConf, localizeResources);
        List<URI> configResources = localizeConfigs(createContainerCConf(cConf), createContainerHConf(this.hConf), tempDir, localizeResources);
        // Localize the program jar
        Location programJarLocation = program.getJarLocation();
        final String programJarName = programJarLocation.getName();
        localizeResources.put(programJarName, new LocalizeResource(programJarLocation.toURI(), false));
        // Localize the app spec
        localizeResources.put(APP_SPEC_FILE_NAME, new LocalizeResource(saveJsonFile(program.getApplicationSpecification(), ApplicationSpecification.class, File.createTempFile("appSpec", ".json", tempDir))));
        URI logbackURI = getLogBackURI(program);
        if (logbackURI != null) {
            // Localize the logback xml
            localizeResources.put(LOGBACK_FILE_NAME, new LocalizeResource(logbackURI, false));
        }
        // Update the ProgramOptions to carry program and runtime information necessary to reconstruct the program
        // and runs it in the remote container
        Map<String, String> extraSystemArgs = new HashMap<>(launchConfig.getExtraSystemArguments());
        extraSystemArgs.put(ProgramOptionConstants.PROGRAM_JAR, programJarName);
        extraSystemArgs.put(ProgramOptionConstants.HADOOP_CONF_FILE, HADOOP_CONF_FILE_NAME);
        extraSystemArgs.put(ProgramOptionConstants.CDAP_CONF_FILE, CDAP_CONF_FILE_NAME);
        extraSystemArgs.put(ProgramOptionConstants.APP_SPEC_FILE, APP_SPEC_FILE_NAME);
        ProgramOptions options = updateProgramOptions(oldOptions, localizeResources, DirUtils.createTempDir(tempDir), extraSystemArgs);
        ProgramRunId programRunId = program.getId().run(ProgramRunners.getRunId(options));
        // Localize the serialized program options
        localizeResources.put(PROGRAM_OPTIONS_FILE_NAME, new LocalizeResource(saveJsonFile(options, ProgramOptions.class, File.createTempFile("program.options", ".json", tempDir))));
        Callable<ProgramController> callable = () -> {
            ProgramTwillApplication twillApplication = new ProgramTwillApplication(programRunId, options, launchConfig.getRunnables(), launchConfig.getLaunchOrder(), localizeResources, createEventHandler(cConf, programRunId, options));
            TwillPreparer twillPreparer = twillRunner.prepare(twillApplication);
            // Also add the configuration files to container classpath so that the
            // TwillAppLifecycleEventHandler can get it. This can be removed when TWILL-246 is fixed.
            // Only ON_PREMISE mode will be using EventHandler
            twillPreparer.withResources(configResources);
            Map<String, String> userArgs = options.getUserArguments().asMap();
            // Setup log level
            twillPreparer.setLogLevels(transformLogLevels(SystemArguments.getLogLevels(userArgs)));
            // Set the configuration for the twill application
            Map<String, String> twillConfigs = new HashMap<>();
            if (DistributedProgramRunner.this instanceof LongRunningDistributedProgramRunner) {
                twillConfigs.put(Configs.Keys.YARN_ATTEMPT_FAILURES_VALIDITY_INTERVAL, cConf.get(Constants.AppFabric.YARN_ATTEMPT_FAILURES_VALIDITY_INTERVAL));
            } else {
                // For non long running program type, set the max attempts to 1 to avoid YARN retry.
                // If the AM container dies, the program execution will be marked as failure.
                // Note that this setting is only applicable to the Twill YARN application
                // (e.g. workflow, Spark client, MR client, etc), but not to the actual Spark / MR job.
                twillConfigs.put(Configs.Keys.YARN_MAX_APP_ATTEMPTS, Integer.toString(1));
            }
            // Add twill configurations coming from the runtime arguments
            twillConfigs.putAll(SystemArguments.getTwillApplicationConfigs(userArgs));
            twillPreparer.withConfiguration(twillConfigs);
            // Setup per runnable configurations
            for (Map.Entry<String, RunnableDefinition> entry : launchConfig.getRunnables().entrySet()) {
                String runnable = entry.getKey();
                RunnableDefinition runnableDefinition = entry.getValue();
                if (runnableDefinition.getMaxRetries() != null) {
                    twillPreparer.withMaxRetries(runnable, runnableDefinition.getMaxRetries());
                }
                twillPreparer.setLogLevels(runnable, transformLogLevels(runnableDefinition.getLogLevels()));
                twillPreparer.withConfiguration(runnable, runnableDefinition.getTwillRunnableConfigs());
                // Add cdap-security.xml if using secrets, and set the runnable identity.
                if (twillPreparer instanceof SecureTwillPreparer) {
                    String twillSystemIdentity = cConf.get(Constants.Twill.Security.IDENTITY_SYSTEM);
                    if (twillSystemIdentity != null) {
                        SecurityContext securityContext = new SecurityContext.Builder().withIdentity(twillSystemIdentity).build();
                        twillPreparer = ((SecureTwillPreparer) twillPreparer).withSecurityContext(runnable, securityContext);
                    }
                    String securityName = cConf.get(Constants.Twill.Security.MASTER_SECRET_DISK_NAME);
                    String securityPath = cConf.get(Constants.Twill.Security.MASTER_SECRET_DISK_PATH);
                    twillPreparer = ((SecureTwillPreparer) twillPreparer).withSecretDisk(runnable, new SecretDisk(securityName, securityPath));
                }
            }
            if (options.isDebug()) {
                twillPreparer.enableDebugging();
            }
            logProgramStart(program, options);
            // Add scheduler queue name if defined
            String schedulerQueueName = options.getArguments().getOption(Constants.AppFabric.APP_SCHEDULER_QUEUE);
            if (schedulerQueueName != null && !schedulerQueueName.isEmpty()) {
                LOG.info("Setting scheduler queue for app {} as {}", program.getId(), schedulerQueueName);
                twillPreparer.setSchedulerQueue(schedulerQueueName);
            }
            // Set JVM options based on configuration
            String jvmOpts = cConf.get(Constants.AppFabric.PROGRAM_JVM_OPTS);
            if (!Strings.isNullOrEmpty(jvmOpts)) {
                twillPreparer.addJVMOptions(jvmOpts);
            }
            if (logbackURI != null) {
                twillPreparer.addJVMOptions("-Dlogback.configurationFile=" + LOGBACK_FILE_NAME);
            }
            addLogHandler(twillPreparer, cConf);
            // Setup the environment for the container logback.xml
            twillPreparer.withEnv(Collections.singletonMap("CDAP_LOG_DIR", ApplicationConstants.LOG_DIR_EXPANSION_VAR));
            // Add dependencies
            Set<Class<?>> extraDependencies = addExtraDependencies(cConf, new HashSet<>(launchConfig.getExtraDependencies()));
            twillPreparer.withDependencies(extraDependencies);
            // Add the additional classes to the classpath that comes from the container jar setting
            twillPreparer.withClassPaths(additionalClassPaths);
            twillPreparer.withClassPaths(launchConfig.getExtraClasspath());
            twillPreparer.withEnv(launchConfig.getExtraEnv());
            // Add the YARN_APPLICATION_CLASSPATH so that yarn classpath are included in the twill container.
            // The Yarn app classpath goes last
            List<String> yarnAppClassPath = Arrays.asList(hConf.getTrimmedStrings(YarnConfiguration.YARN_APPLICATION_CLASSPATH, YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH));
            twillPreparer.withApplicationClassPaths(yarnAppClassPath).withClassPaths(yarnAppClassPath);
            twillPreparer.withBundlerClassAcceptor(launchConfig.getClassAcceptor()).withApplicationArguments(PROGRAM_OPTIONS_FILE_NAME).setClassLoader(MainClassLoader.class.getName());
            TwillController twillController;
            // Change the context classloader to the combine classloader of this ProgramRunner and
            // all the classloaders of the dependencies classes so that Twill can trace classes.
            ClassLoader oldClassLoader = ClassLoaders.setContextClassLoader(new CombineClassLoader(DistributedProgramRunner.this.getClass().getClassLoader(), extraDependencies.stream().map(Class::getClassLoader)::iterator));
            try {
                twillController = twillPreparer.start(cConf.getLong(Constants.AppFabric.PROGRAM_MAX_START_SECONDS), TimeUnit.SECONDS);
                // Block on the twill controller until it is in running state or terminated (due to failure)
                CountDownLatch latch = new CountDownLatch(1);
                twillController.onRunning(latch::countDown, Threads.SAME_THREAD_EXECUTOR);
                twillController.onTerminated(latch::countDown, Threads.SAME_THREAD_EXECUTOR);
                latch.await(cConf.getLong(Constants.AppFabric.PROGRAM_MAX_START_SECONDS), TimeUnit.SECONDS);
            } finally {
                ClassLoaders.setContextClassLoader(oldClassLoader);
            }
            return createProgramController(programRunId, addCleanupListener(twillController, program, tempDir));
        };
        return impersonator.doAs(programRunId, callable);
    } catch (Exception e) {
        deleteDirectory(tempDir);
        throw Throwables.propagate(e);
    }
}