Examples with JobStatus com.google.cloud.dataproc.v1beta2.JobStatus used on opensource projects

Example 11 with JobStatus

use of com.google.cloud.dataproc.v1beta2.JobStatus in project halyard by spinnaker.

the class GoogleProviderUtils method ensureSshKeysExist.

private static void ensureSshKeysExist() {
    File sshKeyFile = new File(getSshKeyFile());
    if (!sshKeyFile.exists()) {
        if (!sshKeyFile.getParentFile().exists()) {
            sshKeyFile.getParentFile().mkdirs();
        }
        log.info("Generating a new ssh key file...");
        JobExecutor jobExecutor = DaemonTaskHandler.getJobExecutor();
        List<String> command = new ArrayList<>();
        command.add("ssh-keygen");
        // no password
        command.add("-N");
        command.add("");
        // rsa key
        command.add("-t");
        command.add("rsa");
        // path to keyfile
        command.add("-f");
        command.add(getSshKeyFile());
        // username sshing into machine
        command.add("-C");
        command.add("ubuntu");
        JobRequest request = new JobRequest().setTokenizedCommand(command);
        JobStatus status;
        try {
            status = jobExecutor.backoffWait(jobExecutor.startJob(request));
        } catch (InterruptedException e) {
            throw new DaemonTaskInterrupted(e);
        }
        if (status.getResult() == JobStatus.Result.FAILURE) {
            throw new HalException(FATAL, "ssh-keygen failed: " + status.getStdErr());
        }
        try {
            File sshPublicKeyFile = new File(getSshPublicKeyFile());
            String sshKeyContents = IOUtils.toString(new FileInputStream(sshPublicKeyFile));
            if (!sshKeyContents.startsWith("ubuntu:")) {
                sshKeyContents = "ubuntu:" + sshKeyContents;
                FileUtils.writeByteArrayToFile(sshPublicKeyFile, sshKeyContents.getBytes());
            }
        } catch (IOException e) {
            throw new HalException(FATAL, "Cannot reformat ssh key to match google key format expectation: " + e.getMessage(), e);
        }
        command = new ArrayList<>();
        command.add("chmod");
        command.add("400");
        command.add(getSshKeyFile());
        request = new JobRequest().setTokenizedCommand(command);
        try {
            status = jobExecutor.backoffWait(jobExecutor.startJob(request));
        } catch (InterruptedException e) {
            throw new DaemonTaskInterrupted(e);
        }
        if (status.getResult() == JobStatus.Result.FAILURE) {
            throw new HalException(FATAL, "chmod failed: " + status.getStdErr() + status.getStdOut());
        }
    }
}

Example 12 with JobStatus

use of com.google.cloud.dataproc.v1beta2.JobStatus in project halyard by spinnaker.

the class GoogleProviderUtils method openSshTunnel.

private static Proxy openSshTunnel(String ip, int port, String keyFile) throws InterruptedException {
    JobExecutor jobExecutor = DaemonTaskHandler.getJobExecutor();
    List<String> command = new ArrayList<>();
    // Make sure we don't have an entry for this host already (GCP recycles IPs).
    command.add("ssh-keygen");
    command.add("-R");
    command.add(ip);
    JobRequest request = new JobRequest().setTokenizedCommand(command);
    JobStatus status = jobExecutor.backoffWait(jobExecutor.startJob(request));
    if (status.getResult() != JobStatus.Result.SUCCESS) {
        if (status.getStdErr().contains("No such file")) {
            log.info("No ssh known_hosts file exists yet");
        } else {
            throw new HalException(FATAL, "Unable to remove old host entry " + status.getStdErr());
        }
    }
    int localPort = SocketUtils.findAvailableTcpPort();
    command.clear();
    command.add("ssh");
    command.add("ubuntu@" + ip);
    command.add("-o");
    command.add("StrictHostKeyChecking=no");
    command.add("-i");
    command.add(keyFile);
    command.add("-N");
    command.add("-L");
    command.add(String.format("%d:localhost:%d", localPort, port));
    request = new JobRequest().setTokenizedCommand(command);
    String jobId = jobExecutor.startJob(request);
    status = jobExecutor.updateJob(jobId);
    while (status == null) {
        DaemonTaskHandler.safeSleep(TimeUnit.SECONDS.toMillis(1));
        status = jobExecutor.updateJob(jobId);
    }
    return new Proxy().setJobId(jobId).setPort(localPort);
}

Example 13 with JobStatus

use of com.google.cloud.dataproc.v1beta2.JobStatus in project halyard by spinnaker.

the class KubernetesV2Executor method apply.

public void apply(String manifest) {
    manifest = kubernetesV2Utils.prettify(manifest);
    List<String> command = kubernetesV2Utils.kubectlPrefix(account);
    command.add("apply");
    command.add("-f");
    // read from stdin
    command.add("-");
    JobRequest request = new JobRequest().setTokenizedCommand(command);
    ByteArrayOutputStream stdout = new ByteArrayOutputStream();
    ByteArrayOutputStream stderr = new ByteArrayOutputStream();
    String jobId = executor.startJob(request, System.getenv(), new ByteArrayInputStream(manifest.getBytes()), stdout, stderr);
    JobStatus status;
    try {
        status = executor.backoffWait(jobId);
    } catch (InterruptedException e) {
        throw new DaemonTaskInterrupted(e);
    }
    if (status.getState() != JobStatus.State.COMPLETED) {
        throw new HalException(Problem.Severity.FATAL, String.join("\n", "Unterminated deployment of manifest:", manifest, stderr.toString(), stdout.toString()));
    }
    if (status.getResult() != JobStatus.Result.SUCCESS) {
        throw new HalException(Problem.Severity.FATAL, String.join("\n", "Failed to deploy manifest:", manifest, stderr.toString(), stdout.toString()));
    }
}

Example 14 with JobStatus

use of com.google.cloud.dataproc.v1beta2.JobStatus in project halyard by spinnaker.

the class KubernetesV2Executor method exists.

private boolean exists(String namespace, String kind, String name) {
    log.info("Checking for " + kind + "/" + name);
    List<String> command = kubernetesV2Utils.kubectlPrefix(account);
    if (StringUtils.isNotEmpty(namespace)) {
        command.add("-n");
        command.add(namespace);
    }
    command.add("get");
    command.add(kind);
    command.add(name);
    JobRequest request = new JobRequest().setTokenizedCommand(command);
    String jobId = executor.startJob(request);
    JobStatus status;
    try {
        status = executor.backoffWait(jobId);
    } catch (InterruptedException e) {
        throw new DaemonTaskInterrupted(e);
    }
    if (status.getState() != JobStatus.State.COMPLETED) {
        throw new HalException(Problem.Severity.FATAL, String.join("\n", "Unterminated check for " + kind + "/" + name + " in " + namespace, status.getStdErr(), status.getStdOut()));
    }
    if (status.getResult() == JobStatus.Result.SUCCESS) {
        return true;
    } else if (status.getStdErr().contains("NotFound")) {
        return false;
    } else {
        throw new HalException(Problem.Severity.FATAL, String.join("\n", "Failed check for " + kind + "/" + name + " in " + namespace, status.getStdErr(), status.getStdOut()));
    }
}

Example 15 with JobStatus

use of com.google.cloud.dataproc.v1beta2.JobStatus in project halyard by spinnaker.

the class KubernetesV1ProviderUtils method openProxy.

static Proxy openProxy(JobExecutor jobExecutor, AccountDeploymentDetails<KubernetesAccount> details) {
    KubernetesAccount account = details.getAccount();
    Proxy proxy = proxyMap.getOrDefault(Proxy.buildKey(details.getDeploymentName()), new Proxy());
    String jobId = proxy.jobId;
    if (StringUtils.isEmpty(jobId) || !jobExecutor.jobExists(jobId)) {
        DaemonTaskHandler.newStage("Connecting to the Kubernetes cluster in account \"" + account.getName() + "\"");
        List<String> command = kubectlAccountCommand(details);
        command.add("proxy");
        // select a random port
        command.add("--port=0");
        JobRequest request = new JobRequest().setTokenizedCommand(command);
        proxy.jobId = jobExecutor.startJob(request);
        JobStatus status = jobExecutor.updateJob(proxy.jobId);
        while (status == null) {
            DaemonTaskHandler.safeSleep(TimeUnit.SECONDS.toMillis(2));
            status = jobExecutor.updateJob(proxy.jobId);
        }
        // This should be a long-running job.
        if (status.getState() == JobStatus.State.COMPLETED) {
            throw new HalException(Severity.FATAL, "Unable to establish a proxy against account " + account.getName() + ":\n" + status.getStdOut() + "\n" + status.getStdErr());
        }
        String connectionMessage = status.getStdOut();
        Pattern portPattern = Pattern.compile(":(\\d+)");
        Matcher matcher = portPattern.matcher(connectionMessage);
        if (matcher.find()) {
            proxy.setPort(Integer.valueOf(matcher.group(1)));
            proxyMap.put(Proxy.buildKey(details.getDeploymentName()), proxy);
            DaemonTaskHandler.message("Connected to kubernetes cluster for account " + account.getName() + " on port " + proxy.getPort());
            DaemonTaskHandler.message("View the kube ui on http://localhost:" + proxy.getPort() + "/ui/");
        } else {
            throw new HalException(Severity.FATAL, "Could not parse connection information from:\n" + connectionMessage + "(" + status.getStdErr() + ")");
        }
    }
    return proxy;
}