use of io.cdap.cdap.master.spi.twill.DependentTwillPreparer in project cdap by caskdata.
the class DistributedPreviewManager method run.
@Override
public void run() {
TwillController activeController = null;
for (TwillController controller : twillRunner.lookup(PreviewRunnerTwillApplication.NAME)) {
// If detected more than one controller, terminate those extra controllers.
if (activeController != null) {
controller.terminate();
} else {
activeController = controller;
}
}
// If there is no preview runner running, create one
if (activeController == null) {
try {
Path tmpDir = new File(cConf.get(Constants.CFG_LOCAL_DATA_DIR), cConf.get(Constants.AppFabric.TEMP_DIR)).toPath();
Files.createDirectories(tmpDir);
Path runDir = Files.createTempDirectory(tmpDir, "preview");
try {
CConfiguration cConfCopy = CConfiguration.copy(cConf);
Path cConfPath = runDir.resolve("cConf.xml");
if (!cConf.getBoolean(Constants.Twill.Security.WORKER_MOUNT_SECRET)) {
// Unset the internal certificate path since certificate is stored cdap-security which
// is not going to be exposed to preview runner.
// TODO: CDAP-18768 this will break preview when certificate checking is enabled.
cConfCopy.unset(Constants.Security.SSL.INTERNAL_CERT_PATH);
}
try (Writer writer = Files.newBufferedWriter(cConfPath, StandardCharsets.UTF_8)) {
cConfCopy.writeXml(writer);
}
Path hConfPath = runDir.resolve("hConf.xml");
try (Writer writer = Files.newBufferedWriter(hConfPath, StandardCharsets.UTF_8)) {
hConf.writeXml(writer);
}
Boolean artifactLocalizerEnabled = cConf.getBoolean(Constants.Preview.ARTIFACT_LOCALIZER_ENABLED);
ResourceSpecification runnerResourceSpec = ResourceSpecification.Builder.with().setVirtualCores(cConf.getInt(Constants.Preview.CONTAINER_CORES)).setMemory(cConf.getInt(Constants.Preview.CONTAINER_MEMORY_MB), ResourceSpecification.SizeUnit.MEGA).setInstances(cConf.getInt(Constants.Preview.CONTAINER_COUNT)).build();
Optional<ResourceSpecification> artifactLocalizerResourceSpec = Optional.empty();
if (artifactLocalizerEnabled) {
artifactLocalizerResourceSpec = Optional.of(ResourceSpecification.Builder.with().setVirtualCores(cConf.getInt(Constants.ArtifactLocalizer.CONTAINER_CORES)).setMemory(cConf.getInt(Constants.ArtifactLocalizer.CONTAINER_MEMORY_MB), ResourceSpecification.SizeUnit.MEGA).setInstances(cConf.getInt(Constants.TaskWorker.CONTAINER_COUNT)).build());
}
LOG.info("Starting preview runners with {} instances and artifactLocalizer {}", runnerResourceSpec.getInstances(), artifactLocalizerEnabled ? "enabled" : "disabled");
TwillPreparer twillPreparer = twillRunner.prepare(new PreviewRunnerTwillApplication(cConfPath.toUri(), hConfPath.toUri(), runnerResourceSpec, artifactLocalizerResourceSpec));
String priorityClass = cConf.get(Constants.Preview.CONTAINER_PRIORITY_CLASS_NAME);
if (priorityClass != null) {
twillPreparer = twillPreparer.setSchedulerQueue(priorityClass);
}
if (twillPreparer instanceof DependentTwillPreparer) {
if (artifactLocalizerEnabled) {
twillPreparer = ((DependentTwillPreparer) twillPreparer).dependentRunnableNames(PreviewRunnerTwillRunnable.class.getSimpleName(), ArtifactLocalizerTwillRunnable.class.getSimpleName());
}
}
if (twillPreparer instanceof StatefulTwillPreparer) {
int diskSize = cConf.getInt(Constants.Preview.CONTAINER_DISK_SIZE_GB);
twillPreparer = ((StatefulTwillPreparer) twillPreparer).withStatefulRunnable(PreviewRunnerTwillRunnable.class.getSimpleName(), false, new StatefulDisk("preview-runner-data", diskSize, cConf.get(Constants.CFG_LOCAL_DATA_DIR)));
}
if (twillPreparer instanceof SecureTwillPreparer) {
String twillUserIdentity = cConf.get(Constants.Twill.Security.IDENTITY_USER);
if (twillUserIdentity != null) {
SecurityContext securityContext = new SecurityContext.Builder().withIdentity(twillUserIdentity).build();
twillPreparer = ((SecureTwillPreparer) twillPreparer).withSecurityContext(PreviewRunnerTwillRunnable.class.getSimpleName(), securityContext);
}
if (artifactLocalizerEnabled) {
// Mount secret in ArtifactLocalizer sidecar which only runs trusted code,
// so requests originated by ArtifactLocalizer can run with system identity when internal auth
// is enabled.
String secretName = cConf.get(Constants.Twill.Security.MASTER_SECRET_DISK_NAME);
String secretPath = cConf.get(Constants.Twill.Security.MASTER_SECRET_DISK_PATH);
twillPreparer = ((SecureTwillPreparer) twillPreparer).withSecretDisk(ArtifactLocalizerTwillRunnable.class.getSimpleName(), new SecretDisk(secretName, secretPath));
}
if (cConf.getBoolean(Constants.Twill.Security.WORKER_MOUNT_SECRET)) {
String secretName = cConf.get(Constants.Twill.Security.WORKER_SECRET_DISK_NAME);
String secretPath = cConf.get(Constants.Twill.Security.WORKER_SECRET_DISK_PATH);
twillPreparer = ((SecureTwillPreparer) twillPreparer).withSecretDisk(PreviewRunnerTwillRunnable.class.getSimpleName(), new SecretDisk(secretName, secretPath));
}
}
activeController = twillPreparer.start(5, TimeUnit.MINUTES);
activeController.onRunning(() -> deleteDir(runDir), Threads.SAME_THREAD_EXECUTOR);
activeController.onTerminated(() -> deleteDir(runDir), Threads.SAME_THREAD_EXECUTOR);
} catch (Exception e) {
deleteDir(runDir);
throw e;
}
} catch (Exception e) {
LOG.warn("Failed to launch preview runner. It will be retried", e);
}
}
controller = activeController;
}
use of io.cdap.cdap.master.spi.twill.DependentTwillPreparer in project cdap by caskdata.
the class SystemWorkerServiceLauncher method run.
public void run() {
TwillController activeController = null;
for (TwillController controller : twillRunner.lookup(SystemWorkerTwillApplication.NAME)) {
// If detected more than one controller, terminate those extra controllers.
if (activeController != null) {
controller.terminate();
} else {
activeController = controller;
}
}
// If there is no system worker runner running, create one
if (activeController == null) {
try {
Path tmpDir = new File(cConf.get(Constants.CFG_LOCAL_DATA_DIR), cConf.get(Constants.AppFabric.TEMP_DIR)).toPath();
Files.createDirectories(tmpDir);
Path runDir = Files.createTempDirectory(tmpDir, "system.worker.launcher");
try {
CConfiguration cConfCopy = CConfiguration.copy(cConf);
Path cConfPath = runDir.resolve("cConf.xml");
try (Writer writer = Files.newBufferedWriter(cConfPath, StandardCharsets.UTF_8)) {
cConfCopy.writeXml(writer);
}
Path hConfPath = runDir.resolve("hConf.xml");
try (Writer writer = Files.newBufferedWriter(hConfPath, StandardCharsets.UTF_8)) {
hConf.writeXml(writer);
}
ResourceSpecification systemResourceSpec = ResourceSpecification.Builder.with().setVirtualCores(cConf.getInt(Constants.SystemWorker.CONTAINER_CORES)).setMemory(cConf.getInt(Constants.SystemWorker.CONTAINER_MEMORY_MB), ResourceSpecification.SizeUnit.MEGA).setInstances(cConf.getInt(Constants.SystemWorker.CONTAINER_COUNT)).build();
ResourceSpecification artifactLocalizerResourceSpec = ResourceSpecification.Builder.with().setVirtualCores(cConf.getInt(Constants.ArtifactLocalizer.CONTAINER_CORES)).setMemory(cConf.getInt(Constants.ArtifactLocalizer.CONTAINER_MEMORY_MB), ResourceSpecification.SizeUnit.MEGA).setInstances(cConf.getInt(Constants.TaskWorker.CONTAINER_COUNT)).build();
LOG.info("Starting SystemWorker pool with {} instances", systemResourceSpec.getInstances());
TwillPreparer twillPreparer = twillRunner.prepare(new SystemWorkerTwillApplication(cConfPath.toUri(), hConfPath.toUri(), systemResourceSpec, artifactLocalizerResourceSpec));
String priorityClass = cConf.get(Constants.TaskWorker.CONTAINER_PRIORITY_CLASS_NAME);
if (priorityClass != null) {
twillPreparer = twillPreparer.setSchedulerQueue(priorityClass);
}
if (twillPreparer instanceof DependentTwillPreparer) {
twillPreparer = ((DependentTwillPreparer) twillPreparer).dependentRunnableNames(SystemWorkerTwillRunnable.class.getSimpleName(), ArtifactLocalizerTwillRunnable.class.getSimpleName());
}
activeController = twillPreparer.start(5, TimeUnit.MINUTES);
activeController.onRunning(() -> deleteDir(runDir), Threads.SAME_THREAD_EXECUTOR);
activeController.onTerminated(() -> deleteDir(runDir), Threads.SAME_THREAD_EXECUTOR);
} catch (Exception e) {
deleteDir(runDir);
throw e;
}
} catch (Exception e) {
LOG.warn(String.format("Failed to launch SystemWorker pool, retry in %d", cConf.getInt(Constants.TaskWorker.POOL_CHECK_INTERVAL)), e);
}
}
this.twillController = activeController;
}
use of io.cdap.cdap.master.spi.twill.DependentTwillPreparer in project cdap by caskdata.
the class TaskWorkerServiceLauncher method run.
public void run() {
TwillController activeController = null;
for (TwillController controller : twillRunner.lookup(TaskWorkerTwillApplication.NAME)) {
// If detected more than one controller, terminate those extra controllers.
if (activeController != null) {
controller.terminate();
} else {
activeController = controller;
}
}
// If there is no task worker runner running, create one
if (activeController == null) {
try {
Path tmpDir = new File(cConf.get(Constants.CFG_LOCAL_DATA_DIR), cConf.get(Constants.AppFabric.TEMP_DIR)).toPath();
Files.createDirectories(tmpDir);
Path runDir = Files.createTempDirectory(tmpDir, "task.worker.launcher");
try {
// Unset the internal certificate path since certificate is stored cdap-security which
// is not exposed (i.e. mounted in k8s) to TaskWorkerService.
CConfiguration cConfCopy = CConfiguration.copy(cConf);
cConfCopy.unset(Constants.Security.SSL.INTERNAL_CERT_PATH);
Path cConfPath = runDir.resolve("cConf.xml");
try (Writer writer = Files.newBufferedWriter(cConfPath, StandardCharsets.UTF_8)) {
cConfCopy.writeXml(writer);
}
Path hConfPath = runDir.resolve("hConf.xml");
try (Writer writer = Files.newBufferedWriter(hConfPath, StandardCharsets.UTF_8)) {
hConf.writeXml(writer);
}
ResourceSpecification taskworkerResourceSpec = ResourceSpecification.Builder.with().setVirtualCores(cConf.getInt(Constants.TaskWorker.CONTAINER_CORES)).setMemory(cConf.getInt(Constants.TaskWorker.CONTAINER_MEMORY_MB), ResourceSpecification.SizeUnit.MEGA).setInstances(cConf.getInt(Constants.TaskWorker.CONTAINER_COUNT)).build();
ResourceSpecification artifactLocalizerResourceSpec = ResourceSpecification.Builder.with().setVirtualCores(cConf.getInt(Constants.ArtifactLocalizer.CONTAINER_CORES)).setMemory(cConf.getInt(Constants.ArtifactLocalizer.CONTAINER_MEMORY_MB), ResourceSpecification.SizeUnit.MEGA).setInstances(cConf.getInt(Constants.TaskWorker.CONTAINER_COUNT)).build();
LOG.info("Starting TaskWorker pool with {} instances", taskworkerResourceSpec.getInstances());
TwillPreparer twillPreparer = twillRunner.prepare(new TaskWorkerTwillApplication(cConfPath.toUri(), hConfPath.toUri(), taskworkerResourceSpec, artifactLocalizerResourceSpec));
String priorityClass = cConf.get(Constants.TaskWorker.CONTAINER_PRIORITY_CLASS_NAME);
if (priorityClass != null) {
twillPreparer = twillPreparer.setSchedulerQueue(priorityClass);
}
if (twillPreparer instanceof DependentTwillPreparer) {
twillPreparer = ((DependentTwillPreparer) twillPreparer).dependentRunnableNames(TaskWorkerTwillRunnable.class.getSimpleName(), ArtifactLocalizerTwillRunnable.class.getSimpleName());
}
if (twillPreparer instanceof StatefulTwillPreparer) {
int diskSize = cConf.getInt(Constants.TaskWorker.CONTAINER_DISK_SIZE_GB);
twillPreparer = ((StatefulTwillPreparer) twillPreparer).withStatefulRunnable(TaskWorkerTwillRunnable.class.getSimpleName(), false, new StatefulDisk(STATEFUL_DISK_NAME, diskSize, cConf.get(Constants.CFG_LOCAL_DATA_DIR)));
if (cConf.getBoolean(Constants.TaskWorker.CONTAINER_DISK_READONLY)) {
twillPreparer = ((StatefulTwillPreparer) twillPreparer).withReadonlyDisk(TaskWorkerTwillRunnable.class.getSimpleName(), STATEFUL_DISK_NAME);
}
}
if (twillPreparer instanceof SecureTwillPreparer) {
SecurityContext securityContext = createSecurityContext();
twillPreparer = ((SecureTwillPreparer) twillPreparer).withSecurityContext(TaskWorkerTwillRunnable.class.getSimpleName(), securityContext);
// Mount secret in ArtifactLocalizer sidecar which only run trusted code,
// so requests originated by ArtifactLocalizer can run with system identity when internal auth
// is enabled.
twillPreparer = ((SecureTwillPreparer) twillPreparer).withSecretDisk(ArtifactLocalizerTwillRunnable.class.getSimpleName(), new SecretDisk(cConf.get(Constants.Twill.Security.MASTER_SECRET_DISK_NAME), cConf.get(Constants.Twill.Security.MASTER_SECRET_DISK_PATH)));
if (cConf.getBoolean(Constants.Twill.Security.WORKER_MOUNT_SECRET)) {
String secretName = cConf.get(Constants.Twill.Security.WORKER_SECRET_DISK_NAME);
String secretPath = cConf.get(Constants.Twill.Security.WORKER_SECRET_DISK_PATH);
twillPreparer = ((SecureTwillPreparer) twillPreparer).withSecretDisk(TaskWorkerTwillRunnable.class.getSimpleName(), new SecretDisk(secretName, secretPath));
}
}
activeController = twillPreparer.start(5, TimeUnit.MINUTES);
activeController.onRunning(() -> deleteDir(runDir), Threads.SAME_THREAD_EXECUTOR);
activeController.onTerminated(() -> deleteDir(runDir), Threads.SAME_THREAD_EXECUTOR);
} catch (Exception e) {
deleteDir(runDir);
throw e;
}
} catch (Exception e) {
LOG.warn(String.format("Failed to launch TaskWorker pool, retry in %d", cConf.getInt(Constants.TaskWorker.POOL_CHECK_INTERVAL)), e);
}
}
this.twillController = activeController;
}
Aggregations