Search in sources :

Example 11 with AzureCloudException

use of com.microsoft.azure.vmagent.exceptions.AzureCloudException in project azure-vm-agents-plugin by jenkinsci.

the class AzureVMCloud method doProvision.

public void doProvision(final int numberOfNewAgents, List<PlannedNode> plannedNodes, final AzureVMAgentTemplate template, final boolean isProvisionOutside) {
    Callable<AzureVMDeploymentInfo> callableTask = new Callable<AzureVMDeploymentInfo>() {

        @Override
        public AzureVMDeploymentInfo call() throws AzureCloudException {
            try {
                return template.provisionAgents(new StreamTaskListener(System.out, Charset.defaultCharset()), numberOfNewAgents);
            } catch (AzureCloudException e) {
                throw e;
            } catch (Exception e) {
                throw AzureCloudException.create(e);
            }
        }
    };
    final Future<AzureVMDeploymentInfo> deploymentFuture = getThreadPool().submit(callableTask);
    for (int i = 0; i < numberOfNewAgents; i++) {
        final int index = i;
        final ProvisioningActivity.Id provisioningId = new ProvisioningActivity.Id(this.name, template.getTemplateName());
        plannedNodes.add(new TrackedPlannedNode(provisioningId, template.getNoOfParallelJobs(), Computer.threadPoolForRemoting.submit(new Callable<Node>() {

            @Override
            public Node call() throws AzureCloudException {
                // Wait for the future to complete
                try {
                    // Only lock for pool maintaining.
                    PoolLock.provisionLock(template);
                    if (isProvisionOutside) {
                        CloudStatistics.ProvisioningListener.get().onStarted(provisioningId);
                    }
                    AzureVMDeploymentInfo info;
                    try {
                        info = deploymentFuture.get();
                    } catch (InterruptedException | ExecutionException e) {
                        handleFailure(template, null, e, FailureStage.DEPLOYMENT);
                        throw AzureCloudException.create(e);
                    }
                    final String deploymentName = info.getDeploymentName();
                    final String vmBaseName = info.getVmBaseName();
                    final String vmName = String.format("%s%d", vmBaseName, index);
                    AzureVMAgent agent;
                    try {
                        agent = createProvisionedAgent(provisioningId, template, vmName, deploymentName);
                    } catch (AzureCloudException e) {
                        LOGGER.log(Level.SEVERE, String.format("Failure creating provisioned agent '%s'", vmName), e);
                        handleFailure(template, vmName, e, FailureStage.PROVISIONING);
                        throw e;
                    }
                    try {
                        LOGGER.log(Level.INFO, "Adding agent {0} to Jenkins nodes", agent.getNodeName());
                        // Place the node in blocked state while it starts.
                        try {
                            agent.blockCleanUpAction();
                            Jenkins.get().addNode(agent);
                            Computer computer = agent.toComputer();
                            if (agent.getAgentLaunchMethod().equalsIgnoreCase("SSH") && computer != null) {
                                computer.connect(false).get();
                            } else if (agent.getAgentLaunchMethod().equalsIgnoreCase("JNLP")) {
                                // Wait until node is online
                                waitUntilJNLPNodeIsOnline(agent);
                            }
                        } finally {
                            // Place node in default state, now can be
                            // dealt with by the cleanup task.
                            agent.clearCleanUpAction();
                        }
                    } catch (Exception e) {
                        LOGGER.log(Level.SEVERE, String.format("Failure to in post-provisioning for '%s'", vmName), e);
                        handleFailure(template, vmName, e, FailureStage.POSTPROVISIONING);
                        // Remove the node from jenkins
                        try {
                            Jenkins.get().removeNode(agent);
                        } catch (IOException nodeRemoveEx) {
                            LOGGER.log(Level.SEVERE, String.format("Failure removing Jenkins node for '%s'", vmName), nodeRemoveEx);
                        // Do not throw to avoid it being recorded
                        }
                        throw AzureCloudException.create(e);
                    }
                    if (isProvisionOutside) {
                        CloudStatistics.ProvisioningListener.get().onComplete(provisioningId, agent);
                    }
                    template.getTemplateProvisionStrategy().success();
                    return agent;
                } catch (AzureCloudException e) {
                    if (isProvisionOutside) {
                        CloudStatistics.ProvisioningListener.get().onFailure(provisioningId, e);
                    }
                    throw e;
                } finally {
                    PoolLock.provisionUnlock(template);
                }
            }

            private void handleFailure(AzureVMAgentTemplate template, String vmName, Exception e, FailureStage stage) {
                // Attempt to terminate whatever was created if any
                if (vmName != null) {
                    try {
                        getServiceDelegate().terminateVirtualMachine(vmName, template.getResourceGroupName());
                    } catch (AzureCloudException terminateEx) {
                        LOGGER.log(Level.SEVERE, String.format("Failure terminating previous failed agent '%s'", vmName), terminateEx);
                    // Do not throw to avoid it being recorded
                    }
                }
                template.retrieveAzureCloudReference().adjustVirtualMachineCount(-1, template.getMaxVirtualMachinesLimit());
                // Update the template status given this new issue.
                template.handleTemplateProvisioningFailure(e.getMessage(), stage);
            }
        })));
    }
}
Also used : StreamTaskListener(hudson.util.StreamTaskListener) TrackedPlannedNode(org.jenkinsci.plugins.cloudstats.TrackedPlannedNode) Node(hudson.model.Node) TrackedPlannedNode(org.jenkinsci.plugins.cloudstats.TrackedPlannedNode) PlannedNode(hudson.slaves.NodeProvisioner.PlannedNode) AzureCloudException(com.microsoft.azure.vmagent.exceptions.AzureCloudException) IOException(java.io.IOException) Callable(java.util.concurrent.Callable) ServletException(javax.servlet.ServletException) AzureCloudException(com.microsoft.azure.vmagent.exceptions.AzureCloudException) IOException(java.io.IOException) ExecutionException(java.util.concurrent.ExecutionException) ProvisioningActivity(org.jenkinsci.plugins.cloudstats.ProvisioningActivity) SlaveComputer(hudson.slaves.SlaveComputer) Computer(hudson.model.Computer) FailureStage(com.microsoft.azure.vmagent.util.FailureStage)

Example 12 with AzureCloudException

use of com.microsoft.azure.vmagent.exceptions.AzureCloudException in project azure-vm-agents-plugin by jenkinsci.

the class AzureVMCloudRetensionStrategy method check.

protected long check(final AzureVMComputer agentNode, ExecutionEngine executionEngine) {
    // Determine whether we can recycle this machine.
    // The CRS is the way that nodes that are currently operating "correctly"
    // can be retained/reclaimed.  Any failure modes need to be dealt with through
    // the clean up task.
    boolean canRecycle = true;
    // Node must be idle
    canRecycle &= agentNode.isIdle();
    // The node must also be online.  This also implies not temporarily disconnected
    // (like by a user).
    canRecycle &= agentNode.isOnline();
    // The configured idle time must be > 0 (which means leave forever)
    canRecycle &= idleTerminationMillis > 0;
    // The number of ms it's been idle must be greater than the current idle time.
    canRecycle &= idleTerminationMillis < (System.currentTimeMillis() - agentNode.getIdleStartMilliseconds());
    if (agentNode.getNode() == null) {
        return 1;
    }
    final AzureVMAgent agent = agentNode.getNode();
    if (canRecycle) {
        LOGGER.log(Level.INFO, "Idle timeout reached for agent: {0}, action: {1}", new Object[] { agentNode.getName(), agent.isShutdownOnIdle() ? "shutdown" : "delete" });
        Callable<Void> task = () -> {
            // Block cleanup while we execute so the cleanup task doesn't try to take it
            // away (node will go offline).  Also blocks cleanup in case of shutdown.
            agent.blockCleanUpAction();
            if (agent.isShutdownOnIdle()) {
                LOGGER.log(Level.INFO, "Going to idleTimeout agent: {0}", agentNode.getName());
                agent.shutdown(Messages._Idle_Timeout_Shutdown());
            } else {
                agent.deprovision(Messages._Idle_Timeout_Delete());
            }
            return null;
        };
        try {
            final int maxRetries = 3;
            final int waitInterval = 30;
            final int defaultTimeoutInSeconds = 30 * 60;
            executionEngine.executeAsync(task, new LinearRetryForAllExceptions(maxRetries, waitInterval, defaultTimeoutInSeconds));
        } catch (AzureCloudException ae) {
            LOGGER.log(Level.WARNING, String.format("Could not terminate or shutdown %s", agentNode.getName()), ae);
            // If we have an exception, set the agent for deletion.
            // It's unlikely we'll be able to shut it down properly ever.
            AzureVMAgent node = agentNode.getNode();
            if (node != null) {
                node.setCleanUpAction(CleanUpAction.DELETE, Messages._Failed_Initial_Shutdown_Or_Delete());
            }
        } catch (Exception e) {
            LOGGER.log(Level.WARNING, String.format("Exception occurred while calling timeout on node %s", agentNode.getName()), e);
            // If we have an exception, set the agent for deletion.
            // It's unlikely we'll be able to shut it down properly ever.
            AzureVMAgent node = agentNode.getNode();
            if (node != null) {
                node.setCleanUpAction(CleanUpAction.DELETE, Messages._Failed_Initial_Shutdown_Or_Delete());
            }
        }
    }
    return 1;
}
Also used : LinearRetryForAllExceptions(com.microsoft.azure.vmagent.retry.LinearRetryForAllExceptions) AzureCloudException(com.microsoft.azure.vmagent.exceptions.AzureCloudException) AzureCloudException(com.microsoft.azure.vmagent.exceptions.AzureCloudException)

Example 13 with AzureCloudException

use of com.microsoft.azure.vmagent.exceptions.AzureCloudException in project azure-vm-agents-plugin by jenkinsci.

the class AzureVMComputer method doDoDelete.

protected HttpResponse doDoDelete(ExecutionEngine executionEngine) {
    checkPermission(DELETE);
    this.setAcceptingTasks(false);
    final AzureVMAgent agent = getNode();
    if (agent != null) {
        Callable<Void> task = () -> {
            LOGGER.log(Level.INFO, "AzureVMComputer: doDoDelete called for agent {0}", agent.getNodeName());
            try {
                // Deprovision
                agent.deprovision(Messages._User_Delete());
            } catch (Exception e) {
                LOGGER.log(Level.INFO, "AzureVMComputer: doDoDelete: Exception occurred while deleting agent", e);
                throw AzureCloudException.create("AzureVMComputer: doDoDelete: Exception occurred while deleting agent", e);
            }
            return null;
        };
        try {
            executionEngine.executeAsync(task, new NoRetryStrategy());
        } catch (AzureCloudException exception) {
            // No need to throw exception back, just log and move on.
            LOGGER.log(Level.INFO, "AzureVMComputer: execute: failed to shutdown/delete " + agent.getDisplayName(), exception);
        }
    }
    return new HttpRedirect("..");
}
Also used : NoRetryStrategy(com.microsoft.azure.vmagent.retry.NoRetryStrategy) HttpRedirect(org.kohsuke.stapler.HttpRedirect) AzureCloudException(com.microsoft.azure.vmagent.exceptions.AzureCloudException) AzureCloudException(com.microsoft.azure.vmagent.exceptions.AzureCloudException)

Example 14 with AzureCloudException

use of com.microsoft.azure.vmagent.exceptions.AzureCloudException in project azure-vm-agents-plugin by jenkinsci.

the class AzureVMManagementServiceDelegate method terminateVirtualMachine.

/**
 * Terminates a virtual machine.
 *
 * @param vmName            VM name
 * @param resourceGroupName Resource group containing the VM
 */
public void terminateVirtualMachine(final String vmName, final String resourceGroupName, ExecutionEngine executionEngine) throws AzureCloudException {
    try {
        if (virtualMachineExists(vmName, resourceGroupName)) {
            List<URI> diskUrisToRemove = new ArrayList<>();
            List<String> diskIdToRemove = new ArrayList<>();
            if (!azureClient.virtualMachines().getByResourceGroup(resourceGroupName, vmName).isManagedDiskEnabled()) {
                // Mark OS disk for removal
                diskUrisToRemove.add(new URI(azureClient.virtualMachines().getByResourceGroup(resourceGroupName, vmName).osUnmanagedDiskVhdUri()));
            } else {
                diskIdToRemove.add(azureClient.virtualMachines().getByResourceGroup(resourceGroupName, vmName).osDiskId());
            }
            // TODO: Remove data disks or add option to do so?
            // Remove the VM
            LOGGER.log(Level.INFO, "Removing virtual machine {0}", vmName);
            azureClient.virtualMachines().deleteByResourceGroup(resourceGroupName, vmName);
            // Now remove the disks
            for (URI diskUri : diskUrisToRemove) {
                this.removeStorageBlob(diskUri, resourceGroupName);
            }
            for (String id : diskIdToRemove) {
                LOGGER.log(Level.INFO, "Removing managed disk with id: {0}", id);
                azureClient.disks().deleteById(id);
            }
            // If used managed Disk with custom vhd, we need to delete the temporary image.
            if (!diskIdToRemove.isEmpty()) {
                removeImage(azureClient, vmName, resourceGroupName);
            }
        }
    } catch (Exception e) {
        LOGGER.log(Level.WARNING, "Exception while deleting VM", e);
        // Check if VM is already deleted: if VM is already deleted then just ignore exception.
        if (!Constants.ERROR_CODE_RESOURCE_NF.equalsIgnoreCase(e.getMessage())) {
            throw AzureCloudException.create(e);
        }
    } finally {
        LOGGER.log(Level.INFO, "Clean operation starting for {0} NIC and IP", vmName);
        executionEngine.executeAsync((Callable<Void>) () -> {
            removeIPName(resourceGroupName, vmName);
            return null;
        }, new NoRetryStrategy());
    }
}
Also used : NoRetryStrategy(com.microsoft.azure.vmagent.retry.NoRetryStrategy) URI(java.net.URI) AzureCloudException(com.microsoft.azure.vmagent.exceptions.AzureCloudException) FormException(hudson.model.Descriptor.FormException) ManagementException(com.azure.core.management.exception.ManagementException) IOException(java.io.IOException)

Aggregations

AzureCloudException (com.microsoft.azure.vmagent.exceptions.AzureCloudException)14 IOException (java.io.IOException)11 ManagementException (com.azure.core.management.exception.ManagementException)7 FormException (hudson.model.Descriptor.FormException)7 VirtualMachine (com.azure.resourcemanager.compute.models.VirtualMachine)4 Computer (hudson.model.Computer)3 ExecutionException (java.util.concurrent.ExecutionException)3 ServletException (javax.servlet.ServletException)3 AzureResourceManager (com.azure.resourcemanager.AzureResourceManager)2 StorageAccount (com.azure.resourcemanager.storage.models.StorageAccount)2 StandardUsernamePasswordCredentials (com.cloudbees.plugins.credentials.common.StandardUsernamePasswordCredentials)2 NoRetryStrategy (com.microsoft.azure.vmagent.retry.NoRetryStrategy)2 SlaveComputer (hudson.slaves.SlaveComputer)2 URI (java.net.URI)2 Callable (java.util.concurrent.Callable)2 GalleryImageVersion (com.azure.resourcemanager.compute.models.GalleryImageVersion)1 OperatingSystemTypes (com.azure.resourcemanager.compute.models.OperatingSystemTypes)1 PowerState (com.azure.resourcemanager.compute.models.PowerState)1 PurchasePlan (com.azure.resourcemanager.compute.models.PurchasePlan)1 VirtualMachineCustomImage (com.azure.resourcemanager.compute.models.VirtualMachineCustomImage)1