Search in sources :

Example 1 with StatefulDisk

use of io.cdap.cdap.master.spi.twill.StatefulDisk in project cdap by caskdata.

the class DistributedPreviewManager method run.

@Override
public void run() {
    TwillController activeController = null;
    for (TwillController controller : twillRunner.lookup(PreviewRunnerTwillApplication.NAME)) {
        // If detected more than one controller, terminate those extra controllers.
        if (activeController != null) {
            controller.terminate();
        } else {
            activeController = controller;
        }
    }
    // If there is no preview runner running, create one
    if (activeController == null) {
        try {
            Path tmpDir = new File(cConf.get(Constants.CFG_LOCAL_DATA_DIR), cConf.get(Constants.AppFabric.TEMP_DIR)).toPath();
            Files.createDirectories(tmpDir);
            Path runDir = Files.createTempDirectory(tmpDir, "preview");
            try {
                CConfiguration cConfCopy = CConfiguration.copy(cConf);
                Path cConfPath = runDir.resolve("cConf.xml");
                if (!cConf.getBoolean(Constants.Twill.Security.WORKER_MOUNT_SECRET)) {
                    // Unset the internal certificate path since certificate is stored cdap-security which
                    // is not going to be exposed to preview runner.
                    // TODO: CDAP-18768 this will break preview when certificate checking is enabled.
                    cConfCopy.unset(Constants.Security.SSL.INTERNAL_CERT_PATH);
                }
                try (Writer writer = Files.newBufferedWriter(cConfPath, StandardCharsets.UTF_8)) {
                    cConfCopy.writeXml(writer);
                }
                Path hConfPath = runDir.resolve("hConf.xml");
                try (Writer writer = Files.newBufferedWriter(hConfPath, StandardCharsets.UTF_8)) {
                    hConf.writeXml(writer);
                }
                Boolean artifactLocalizerEnabled = cConf.getBoolean(Constants.Preview.ARTIFACT_LOCALIZER_ENABLED);
                ResourceSpecification runnerResourceSpec = ResourceSpecification.Builder.with().setVirtualCores(cConf.getInt(Constants.Preview.CONTAINER_CORES)).setMemory(cConf.getInt(Constants.Preview.CONTAINER_MEMORY_MB), ResourceSpecification.SizeUnit.MEGA).setInstances(cConf.getInt(Constants.Preview.CONTAINER_COUNT)).build();
                Optional<ResourceSpecification> artifactLocalizerResourceSpec = Optional.empty();
                if (artifactLocalizerEnabled) {
                    artifactLocalizerResourceSpec = Optional.of(ResourceSpecification.Builder.with().setVirtualCores(cConf.getInt(Constants.ArtifactLocalizer.CONTAINER_CORES)).setMemory(cConf.getInt(Constants.ArtifactLocalizer.CONTAINER_MEMORY_MB), ResourceSpecification.SizeUnit.MEGA).setInstances(cConf.getInt(Constants.TaskWorker.CONTAINER_COUNT)).build());
                }
                LOG.info("Starting preview runners with {} instances and artifactLocalizer {}", runnerResourceSpec.getInstances(), artifactLocalizerEnabled ? "enabled" : "disabled");
                TwillPreparer twillPreparer = twillRunner.prepare(new PreviewRunnerTwillApplication(cConfPath.toUri(), hConfPath.toUri(), runnerResourceSpec, artifactLocalizerResourceSpec));
                String priorityClass = cConf.get(Constants.Preview.CONTAINER_PRIORITY_CLASS_NAME);
                if (priorityClass != null) {
                    twillPreparer = twillPreparer.setSchedulerQueue(priorityClass);
                }
                if (twillPreparer instanceof DependentTwillPreparer) {
                    if (artifactLocalizerEnabled) {
                        twillPreparer = ((DependentTwillPreparer) twillPreparer).dependentRunnableNames(PreviewRunnerTwillRunnable.class.getSimpleName(), ArtifactLocalizerTwillRunnable.class.getSimpleName());
                    }
                }
                if (twillPreparer instanceof StatefulTwillPreparer) {
                    int diskSize = cConf.getInt(Constants.Preview.CONTAINER_DISK_SIZE_GB);
                    twillPreparer = ((StatefulTwillPreparer) twillPreparer).withStatefulRunnable(PreviewRunnerTwillRunnable.class.getSimpleName(), false, new StatefulDisk("preview-runner-data", diskSize, cConf.get(Constants.CFG_LOCAL_DATA_DIR)));
                }
                if (twillPreparer instanceof SecureTwillPreparer) {
                    String twillUserIdentity = cConf.get(Constants.Twill.Security.IDENTITY_USER);
                    if (twillUserIdentity != null) {
                        SecurityContext securityContext = new SecurityContext.Builder().withIdentity(twillUserIdentity).build();
                        twillPreparer = ((SecureTwillPreparer) twillPreparer).withSecurityContext(PreviewRunnerTwillRunnable.class.getSimpleName(), securityContext);
                    }
                    if (artifactLocalizerEnabled) {
                        // Mount secret in ArtifactLocalizer sidecar which only runs trusted code,
                        // so requests originated by ArtifactLocalizer can run with system identity when internal auth
                        // is enabled.
                        String secretName = cConf.get(Constants.Twill.Security.MASTER_SECRET_DISK_NAME);
                        String secretPath = cConf.get(Constants.Twill.Security.MASTER_SECRET_DISK_PATH);
                        twillPreparer = ((SecureTwillPreparer) twillPreparer).withSecretDisk(ArtifactLocalizerTwillRunnable.class.getSimpleName(), new SecretDisk(secretName, secretPath));
                    }
                    if (cConf.getBoolean(Constants.Twill.Security.WORKER_MOUNT_SECRET)) {
                        String secretName = cConf.get(Constants.Twill.Security.WORKER_SECRET_DISK_NAME);
                        String secretPath = cConf.get(Constants.Twill.Security.WORKER_SECRET_DISK_PATH);
                        twillPreparer = ((SecureTwillPreparer) twillPreparer).withSecretDisk(PreviewRunnerTwillRunnable.class.getSimpleName(), new SecretDisk(secretName, secretPath));
                    }
                }
                activeController = twillPreparer.start(5, TimeUnit.MINUTES);
                activeController.onRunning(() -> deleteDir(runDir), Threads.SAME_THREAD_EXECUTOR);
                activeController.onTerminated(() -> deleteDir(runDir), Threads.SAME_THREAD_EXECUTOR);
            } catch (Exception e) {
                deleteDir(runDir);
                throw e;
            }
        } catch (Exception e) {
            LOG.warn("Failed to launch preview runner. It will be retried", e);
        }
    }
    controller = activeController;
}
Also used : Path(java.nio.file.Path) DependentTwillPreparer(io.cdap.cdap.master.spi.twill.DependentTwillPreparer) StatefulTwillPreparer(io.cdap.cdap.master.spi.twill.StatefulTwillPreparer) ResourceSpecification(org.apache.twill.api.ResourceSpecification) SecretDisk(io.cdap.cdap.master.spi.twill.SecretDisk) CConfiguration(io.cdap.cdap.common.conf.CConfiguration) IOException(java.io.IOException) TwillController(org.apache.twill.api.TwillController) StatefulDisk(io.cdap.cdap.master.spi.twill.StatefulDisk) SecureTwillPreparer(io.cdap.cdap.master.spi.twill.SecureTwillPreparer) SecurityContext(io.cdap.cdap.master.spi.twill.SecurityContext) File(java.io.File) Writer(java.io.Writer) TwillPreparer(org.apache.twill.api.TwillPreparer) StatefulTwillPreparer(io.cdap.cdap.master.spi.twill.StatefulTwillPreparer) SecureTwillPreparer(io.cdap.cdap.master.spi.twill.SecureTwillPreparer) DependentTwillPreparer(io.cdap.cdap.master.spi.twill.DependentTwillPreparer)

Example 2 with StatefulDisk

use of io.cdap.cdap.master.spi.twill.StatefulDisk in project cdap by caskdata.

the class TaskWorkerServiceLauncher method run.

public void run() {
    TwillController activeController = null;
    for (TwillController controller : twillRunner.lookup(TaskWorkerTwillApplication.NAME)) {
        // If detected more than one controller, terminate those extra controllers.
        if (activeController != null) {
            controller.terminate();
        } else {
            activeController = controller;
        }
    }
    // If there is no task worker runner running, create one
    if (activeController == null) {
        try {
            Path tmpDir = new File(cConf.get(Constants.CFG_LOCAL_DATA_DIR), cConf.get(Constants.AppFabric.TEMP_DIR)).toPath();
            Files.createDirectories(tmpDir);
            Path runDir = Files.createTempDirectory(tmpDir, "task.worker.launcher");
            try {
                // Unset the internal certificate path since certificate is stored cdap-security which
                // is not exposed (i.e. mounted in k8s) to TaskWorkerService.
                CConfiguration cConfCopy = CConfiguration.copy(cConf);
                cConfCopy.unset(Constants.Security.SSL.INTERNAL_CERT_PATH);
                Path cConfPath = runDir.resolve("cConf.xml");
                try (Writer writer = Files.newBufferedWriter(cConfPath, StandardCharsets.UTF_8)) {
                    cConfCopy.writeXml(writer);
                }
                Path hConfPath = runDir.resolve("hConf.xml");
                try (Writer writer = Files.newBufferedWriter(hConfPath, StandardCharsets.UTF_8)) {
                    hConf.writeXml(writer);
                }
                ResourceSpecification taskworkerResourceSpec = ResourceSpecification.Builder.with().setVirtualCores(cConf.getInt(Constants.TaskWorker.CONTAINER_CORES)).setMemory(cConf.getInt(Constants.TaskWorker.CONTAINER_MEMORY_MB), ResourceSpecification.SizeUnit.MEGA).setInstances(cConf.getInt(Constants.TaskWorker.CONTAINER_COUNT)).build();
                ResourceSpecification artifactLocalizerResourceSpec = ResourceSpecification.Builder.with().setVirtualCores(cConf.getInt(Constants.ArtifactLocalizer.CONTAINER_CORES)).setMemory(cConf.getInt(Constants.ArtifactLocalizer.CONTAINER_MEMORY_MB), ResourceSpecification.SizeUnit.MEGA).setInstances(cConf.getInt(Constants.TaskWorker.CONTAINER_COUNT)).build();
                LOG.info("Starting TaskWorker pool with {} instances", taskworkerResourceSpec.getInstances());
                TwillPreparer twillPreparer = twillRunner.prepare(new TaskWorkerTwillApplication(cConfPath.toUri(), hConfPath.toUri(), taskworkerResourceSpec, artifactLocalizerResourceSpec));
                String priorityClass = cConf.get(Constants.TaskWorker.CONTAINER_PRIORITY_CLASS_NAME);
                if (priorityClass != null) {
                    twillPreparer = twillPreparer.setSchedulerQueue(priorityClass);
                }
                if (twillPreparer instanceof DependentTwillPreparer) {
                    twillPreparer = ((DependentTwillPreparer) twillPreparer).dependentRunnableNames(TaskWorkerTwillRunnable.class.getSimpleName(), ArtifactLocalizerTwillRunnable.class.getSimpleName());
                }
                if (twillPreparer instanceof StatefulTwillPreparer) {
                    int diskSize = cConf.getInt(Constants.TaskWorker.CONTAINER_DISK_SIZE_GB);
                    twillPreparer = ((StatefulTwillPreparer) twillPreparer).withStatefulRunnable(TaskWorkerTwillRunnable.class.getSimpleName(), false, new StatefulDisk(STATEFUL_DISK_NAME, diskSize, cConf.get(Constants.CFG_LOCAL_DATA_DIR)));
                    if (cConf.getBoolean(Constants.TaskWorker.CONTAINER_DISK_READONLY)) {
                        twillPreparer = ((StatefulTwillPreparer) twillPreparer).withReadonlyDisk(TaskWorkerTwillRunnable.class.getSimpleName(), STATEFUL_DISK_NAME);
                    }
                }
                if (twillPreparer instanceof SecureTwillPreparer) {
                    SecurityContext securityContext = createSecurityContext();
                    twillPreparer = ((SecureTwillPreparer) twillPreparer).withSecurityContext(TaskWorkerTwillRunnable.class.getSimpleName(), securityContext);
                    // Mount secret in ArtifactLocalizer sidecar which only run trusted code,
                    // so requests originated by ArtifactLocalizer can run with system identity when internal auth
                    // is enabled.
                    twillPreparer = ((SecureTwillPreparer) twillPreparer).withSecretDisk(ArtifactLocalizerTwillRunnable.class.getSimpleName(), new SecretDisk(cConf.get(Constants.Twill.Security.MASTER_SECRET_DISK_NAME), cConf.get(Constants.Twill.Security.MASTER_SECRET_DISK_PATH)));
                    if (cConf.getBoolean(Constants.Twill.Security.WORKER_MOUNT_SECRET)) {
                        String secretName = cConf.get(Constants.Twill.Security.WORKER_SECRET_DISK_NAME);
                        String secretPath = cConf.get(Constants.Twill.Security.WORKER_SECRET_DISK_PATH);
                        twillPreparer = ((SecureTwillPreparer) twillPreparer).withSecretDisk(TaskWorkerTwillRunnable.class.getSimpleName(), new SecretDisk(secretName, secretPath));
                    }
                }
                activeController = twillPreparer.start(5, TimeUnit.MINUTES);
                activeController.onRunning(() -> deleteDir(runDir), Threads.SAME_THREAD_EXECUTOR);
                activeController.onTerminated(() -> deleteDir(runDir), Threads.SAME_THREAD_EXECUTOR);
            } catch (Exception e) {
                deleteDir(runDir);
                throw e;
            }
        } catch (Exception e) {
            LOG.warn(String.format("Failed to launch TaskWorker pool, retry in %d", cConf.getInt(Constants.TaskWorker.POOL_CHECK_INTERVAL)), e);
        }
    }
    this.twillController = activeController;
}
Also used : Path(java.nio.file.Path) DependentTwillPreparer(io.cdap.cdap.master.spi.twill.DependentTwillPreparer) StatefulTwillPreparer(io.cdap.cdap.master.spi.twill.StatefulTwillPreparer) ResourceSpecification(org.apache.twill.api.ResourceSpecification) SecretDisk(io.cdap.cdap.master.spi.twill.SecretDisk) CConfiguration(io.cdap.cdap.common.conf.CConfiguration) IOException(java.io.IOException) TwillController(org.apache.twill.api.TwillController) StatefulDisk(io.cdap.cdap.master.spi.twill.StatefulDisk) SecureTwillPreparer(io.cdap.cdap.master.spi.twill.SecureTwillPreparer) SecurityContext(io.cdap.cdap.master.spi.twill.SecurityContext) File(java.io.File) Writer(java.io.Writer) TwillPreparer(org.apache.twill.api.TwillPreparer) StatefulTwillPreparer(io.cdap.cdap.master.spi.twill.StatefulTwillPreparer) SecureTwillPreparer(io.cdap.cdap.master.spi.twill.SecureTwillPreparer) DependentTwillPreparer(io.cdap.cdap.master.spi.twill.DependentTwillPreparer)

Example 3 with StatefulDisk

use of io.cdap.cdap.master.spi.twill.StatefulDisk in project cdap by caskdata.

the class KubeTwillPreparer method buildStatefulSet.

/**
 * Returns a {@link V1StatefulSet} object for the {@link TwillRunnable} represented by the
 * given {@link RuntimeSpecification}
 */
private V1StatefulSet buildStatefulSet(V1ObjectMeta metadata, Map<String, RuntimeSpecification> runtimeSpecs, Location runtimeConfigLocation, StatefulRunnable statefulRunnable) {
    List<StatefulDisk> disks = statefulRunnable.getStatefulDisks();
    int replicas = getMainRuntimeSpecification(runtimeSpecs).getResourceSpecification().getInstances();
    return new V1StatefulSetBuilder().withMetadata(metadata).withNewSpec().withSelector(new V1LabelSelector().matchLabels(metadata.getLabels())).withReplicas(replicas).withPodManagementPolicy(statefulRunnable.isOrderedStart() ? "OrderedReady" : "Parallel").addAllToVolumeClaimTemplates(disks.stream().map(this::createPVC).collect(Collectors.toList())).withNewTemplate().withMetadata(metadata).withSpec(createPodSpec(runtimeConfigLocation, runtimeSpecs, disks.stream().map(this::createDiskMount).toArray(V1VolumeMount[]::new))).endTemplate().endSpec().build();
}
Also used : StatefulDisk(io.cdap.cdap.master.spi.twill.StatefulDisk) V1StatefulSetBuilder(io.kubernetes.client.openapi.models.V1StatefulSetBuilder) V1LabelSelector(io.kubernetes.client.openapi.models.V1LabelSelector) V1VolumeMount(io.kubernetes.client.openapi.models.V1VolumeMount)

Aggregations

StatefulDisk (io.cdap.cdap.master.spi.twill.StatefulDisk)3 CConfiguration (io.cdap.cdap.common.conf.CConfiguration)2 DependentTwillPreparer (io.cdap.cdap.master.spi.twill.DependentTwillPreparer)2 SecretDisk (io.cdap.cdap.master.spi.twill.SecretDisk)2 SecureTwillPreparer (io.cdap.cdap.master.spi.twill.SecureTwillPreparer)2 SecurityContext (io.cdap.cdap.master.spi.twill.SecurityContext)2 StatefulTwillPreparer (io.cdap.cdap.master.spi.twill.StatefulTwillPreparer)2 File (java.io.File)2 IOException (java.io.IOException)2 Writer (java.io.Writer)2 Path (java.nio.file.Path)2 ResourceSpecification (org.apache.twill.api.ResourceSpecification)2 TwillController (org.apache.twill.api.TwillController)2 TwillPreparer (org.apache.twill.api.TwillPreparer)2 V1LabelSelector (io.kubernetes.client.openapi.models.V1LabelSelector)1 V1StatefulSetBuilder (io.kubernetes.client.openapi.models.V1StatefulSetBuilder)1 V1VolumeMount (io.kubernetes.client.openapi.models.V1VolumeMount)1