use of com.azure.resourcemanager.storage.models.StorageAccountKey in project azure-vm-agents-plugin by jenkinsci.
the class IntegrationTest method containerExists.
protected boolean containerExists(String storageURI) {
try {
BlobUrlParts blobUrlParts = BlobUrlParts.parse(storageURI);
String storageAccountName = blobUrlParts.getAccountName();
final String containerName = blobUrlParts.getBlobContainerName();
StorageAccount storageAccount = azureClient.storageAccounts().getByResourceGroup(testEnv.azureResourceGroup, storageAccountName);
List<StorageAccountKey> storageKeys = storageAccount.getKeys();
if (storageKeys.isEmpty()) {
return false;
} else {
String storageAccountKey = storageKeys.get(0).value();
BlobServiceClient account = new BlobServiceClientBuilder().credential(new StorageSharedKeyCredential(storageAccountName, storageAccountKey)).endpoint(storageAccount.endPoints().primary().blob()).buildClient();
BlobContainerClient blobClient = account.getBlobContainerClient(containerName);
return blobClient.exists();
}
} catch (Exception e) {
LOGGER.log(Level.SEVERE, e.getMessage(), e);
return false;
}
}
use of com.azure.resourcemanager.storage.models.StorageAccountKey in project azure-vm-agents-plugin by jenkinsci.
the class AzureVMManagementServiceDelegate method createDeployment.
public AzureVMDeploymentInfo createDeployment(AzureVMAgentTemplate template, int numberOfAgents, AzureVMAgentCleanUpTask.DeploymentRegistrar deploymentRegistrar) throws AzureCloudException, IOException {
InputStream embeddedTemplate = null;
String scriptUri = null;
try {
LOGGER.log(Level.INFO, "Initializing deployment for {0} agentTemplate(s) {1}", new Object[] { numberOfAgents, template.getTemplateName() });
Map<String, Object> properties = AzureVMAgentTemplate.getTemplateProperties(template);
final Date timestamp = new Date(System.currentTimeMillis());
final String deploymentName = AzureUtil.getDeploymentName(template.getTemplateName(), timestamp);
final String vmBaseName = AzureUtil.getVMBaseName(template.getTemplateName(), deploymentName, (String) properties.get("osType"), numberOfAgents);
final String locationName = AzureUtil.getLocationNameByLabel(template.getLocation());
final String storageAccountName = template.getStorageAccountName();
final String storageAccountType = template.getStorageAccountType();
final String diskType = template.getDiskType();
final boolean ephemeralOSDisk = template.isEphemeralOSDisk();
final int osDiskSize = template.getOsDiskSize();
final AzureVMAgentTemplate.AvailabilityTypeClass availabilityType = template.getAvailabilityType();
final String availabilitySet = availabilityType != null ? availabilityType.getAvailabilitySet() : null;
if (!template.getResourceGroupName().matches(Constants.DEFAULT_RESOURCE_GROUP_PATTERN)) {
LOGGER.log(Level.SEVERE, "ResourceGroup Name {0} is invalid. It should be 1-64 alphanumeric characters", new Object[] { template.getResourceGroupName() });
throw new Exception("ResourceGroup Name is invalid");
}
LOGGER.log(Level.INFO, "Creating a new deployment {0} with VM base name {1} for {2} VM(s)", new Object[] { deploymentName, vmBaseName, numberOfAgents });
final String resourceGroupName = template.getResourceGroupName();
final String resourceGroupReferenceType = template.getResourceGroupReferenceType();
String cloudName = template.retrieveAzureCloudReference().getCloudName();
if (Constants.RESOURCE_GROUP_REFERENCE_TYPE_NEW.equals(resourceGroupReferenceType)) {
createAzureResourceGroup(azureClient, locationName, resourceGroupName, cloudName);
}
// For blob endpoint url in arm template, it's different based on different environments
// So create StorageAccount and get suffix
List<AzureTagPair> cloudTags = template.retrieveAzureCloudReference().getCloudTags();
List<AzureTagPair> templateTags = template.getTags();
final List<AzureTagPair> tags = concat(cloudTags, templateTags);
createStorageAccount(azureClient, storageAccountType, storageAccountName, locationName, resourceGroupName, template.getTemplateName(), tags);
StorageAccount storageAccount = getStorageAccount(azureClient, storageAccountName, resourceGroupName);
String blobEndpointSuffix = getBlobEndpointSuffixForTemplate(storageAccount);
boolean isBasic = template.isTopLevelType(Constants.IMAGE_TOP_LEVEL_BASIC);
ImageReferenceType referenceType = template.getImageReference().getType();
final boolean preInstallSshInWindows = properties.get("osType").equals(Constants.OS_TYPE_WINDOWS) && properties.get("agentLaunchMethod").equals(Constants.LAUNCH_METHOD_SSH) && (isBasic || referenceType == ImageReferenceType.REFERENCE || template.isPreInstallSsh());
final boolean useCustomScriptExtension = preInstallSshInWindows || properties.get("osType").equals(Constants.OS_TYPE_WINDOWS) && !StringUtils.isBlank((String) properties.get("initScript")) && properties.get("agentLaunchMethod").equals(Constants.LAUNCH_METHOD_JNLP);
// check if a custom image id has been provided otherwise work with publisher and offer
boolean useManagedDisk = diskType.equals(Constants.DISK_MANAGED);
String msg;
String templateLocation;
boolean useCustomImage = !isBasic && referenceType == ImageReferenceType.CUSTOM;
if (useCustomScriptExtension) {
if (useManagedDisk) {
msg = "Use embedded deployment template (with script and managed) {0}";
if (useCustomImage) {
templateLocation = EMBEDDED_TEMPLATE_IMAGE_WITH_SCRIPT_MANAGED_FILENAME;
} else {
templateLocation = (referenceType == ImageReferenceType.CUSTOM_IMAGE || referenceType == ImageReferenceType.GALLERY) ? EMBEDDED_TEMPLATE_IMAGE_ID_WITH_SCRIPT_MANAGED_FILENAME : EMBEDDED_TEMPLATE_WITH_SCRIPT_MANAGED_FILENAME;
}
} else {
msg = "Use embedded deployment template (with script) {0}";
templateLocation = useCustomImage ? EMBEDDED_TEMPLATE_IMAGE_WITH_SCRIPT_FILENAME : EMBEDDED_TEMPLATE_WITH_SCRIPT_FILENAME;
}
} else {
if (useManagedDisk) {
msg = "Use embedded deployment template (with managed) {0}";
if (useCustomImage) {
templateLocation = EMBEDDED_TEMPLATE_IMAGE_WITH_MANAGED_FILENAME;
} else {
templateLocation = (referenceType == ImageReferenceType.CUSTOM_IMAGE || referenceType == ImageReferenceType.GALLERY) ? EMBEDDED_TEMPLATE_IMAGE_ID_WITH_MANAGED_FILENAME : EMBEDDED_TEMPLATE_WITH_MANAGED_FILENAME;
}
} else {
msg = "Use embedded deployment template {0}";
templateLocation = useCustomImage ? EMBEDDED_TEMPLATE_IMAGE_FILENAME : EMBEDDED_TEMPLATE_FILENAME;
}
}
LOGGER.log(Level.INFO, msg, templateLocation);
embeddedTemplate = AzureVMManagementServiceDelegate.class.getResourceAsStream(templateLocation);
final JsonNode tmp = MAPPER.readTree(embeddedTemplate);
// Add count variable for loop....
final ObjectNode count = MAPPER.createObjectNode();
count.put("type", "int");
count.put("defaultValue", numberOfAgents);
((ObjectNode) tmp.get("parameters")).replace("count", count);
putVariable(tmp, "vmName", vmBaseName);
putVariable(tmp, "location", locationName);
putVariable(tmp, "jenkinsTag", Constants.AZURE_JENKINS_TAG_VALUE);
putVariable(tmp, "resourceTag", deploymentRegistrar.getDeploymentTag().get());
putVariable(tmp, "cloudTag", cloudName);
putVariable(tmp, "osDiskStorageAccountType", template.getOsDiskStorageAccountType());
// getId()->Custom Managed Image, here we need the last one: Image Reference
if (!isBasic) {
if (referenceType == ImageReferenceType.REFERENCE) {
boolean isImageParameterValid = checkImageParameter(template);
if (isImageParameterValid) {
String imageVersion = StringUtils.isNotEmpty(template.getImageReference().getVersion()) ? template.getImageReference().getVersion() : "latest";
VirtualMachineImage image = azureClient.virtualMachineImages().getImage(locationName, template.getImageReference().getPublisher(), template.getImageReference().getOffer(), template.getImageReference().getSku(), imageVersion);
if (image != null) {
PurchasePlan plan = image.plan();
if (plan != null) {
ArrayNode resources = (ArrayNode) tmp.get("resources");
for (JsonNode resource : resources) {
String type = resource.get("type").asText();
if (type.contains("virtualMachine")) {
ObjectNode planNode = MAPPER.createObjectNode();
planNode.put("name", plan.name());
planNode.put("publisher", plan.publisher());
planNode.put("product", plan.product());
((ObjectNode) resource).replace("plan", planNode);
}
}
}
} else {
LOGGER.log(Level.SEVERE, "Failed to find the image with publisher:{0} offer:{1} sku:{2} " + "version:{3} when trying to add purchase plan to ARM template", new Object[] { template.getImageReference().getPublisher(), template.getImageReference().getOffer(), template.getImageReference().getSku(), imageVersion });
}
}
} else if (referenceType == ImageReferenceType.CUSTOM_IMAGE) {
String id = template.getId();
VirtualMachineCustomImage customImage = azureClient.virtualMachineCustomImages().getById(id);
if (customImage != null) {
Map<String, String> imageTags = customImage.tags();
if (imageTags != null) {
String planInfo = imageTags.get("PlanInfo");
String planProduct = imageTags.get("PlanProduct");
String planPublisher = imageTags.get("PlanPublisher");
if (StringUtils.isNotBlank(planInfo) && StringUtils.isNotBlank(planProduct) && StringUtils.isNotBlank(planPublisher)) {
ArrayNode resources = (ArrayNode) tmp.get("resources");
for (JsonNode resource : resources) {
String type = resource.get("type").asText();
if (type.contains("virtualMachine")) {
ObjectNode planNode = MAPPER.createObjectNode();
planNode.put("name", planInfo);
planNode.put("publisher", planPublisher);
planNode.put("product", planProduct);
((ObjectNode) resource).replace("plan", planNode);
}
}
}
}
}
}
}
boolean msiEnabled = template.isEnableMSI();
boolean uamiEnabled = template.isEnableUAMI();
boolean osDiskSizeChanged = osDiskSize > 0;
boolean availabilitySetEnabled = availabilitySet != null;
boolean isSpecializedImage = false;
if (template.getImageReference() != null) {
isSpecializedImage = template.getImageReference().getGalleryImageSpecialized();
}
if (msiEnabled || uamiEnabled || osDiskSizeChanged || availabilitySetEnabled || isSpecializedImage) {
ArrayNode resources = (ArrayNode) tmp.get("resources");
for (JsonNode resource : resources) {
String type = resource.get("type").asText();
if (type.contains("virtualMachine")) {
// https://docs.microsoft.com/en-us/rest/api/compute/virtualmachines/createorupdate#resourceidentitytype
if (msiEnabled && uamiEnabled) {
String uamiID = template.getUamiID();
ObjectNode identityNode = MAPPER.createObjectNode();
identityNode.put("type", "SystemAssigned, UserAssigned");
ObjectNode resourceId = MAPPER.createObjectNode();
resourceId.replace(uamiID, MAPPER.createObjectNode());
identityNode.replace("userAssignedIdentities", resourceId);
((ObjectNode) resource).replace("identity", identityNode);
} else if (msiEnabled) {
ObjectNode identityNode = MAPPER.createObjectNode();
identityNode.put("type", "systemAssigned");
((ObjectNode) resource).replace("identity", identityNode);
} else if (uamiEnabled) {
String uamiID = template.getUamiID();
ObjectNode identityNode = MAPPER.createObjectNode();
identityNode.put("type", "UserAssigned");
ObjectNode resourceId = MAPPER.createObjectNode();
resourceId.replace(uamiID, MAPPER.createObjectNode());
identityNode.replace("userAssignedIdentities", resourceId);
((ObjectNode) resource).replace("identity", identityNode);
}
if (osDiskSizeChanged) {
JsonNode jsonNode = resource.get("properties").get("storageProfile").get("osDisk");
((ObjectNode) jsonNode).replace("diskSizeGB", new IntNode(osDiskSize));
}
if (availabilitySetEnabled) {
ObjectNode availabilitySetNode = MAPPER.createObjectNode();
availabilitySetNode.put("id", String.format("[resourceId('Microsoft.Compute/availabilitySets', '%s')]", availabilitySet));
JsonNode propertiesNode = resource.get("properties");
((ObjectNode) propertiesNode).replace("availabilitySet", availabilitySetNode);
}
if (isSpecializedImage) {
// For specialized image remove the osProfile from the properties of the VirtualMachine resource
JsonNode propertiesNode = resource.get("properties");
((ObjectNode) propertiesNode).remove("osProfile");
}
}
}
}
ArrayNode resources = (ArrayNode) tmp.get("resources");
for (JsonNode resource : resources) {
injectCustomTag(resource, tags);
}
copyVariableIfNotBlank(tmp, properties, "imageId");
copyVariableIfNotBlank(tmp, properties, "imagePublisher");
copyVariableIfNotBlank(tmp, properties, "imageOffer");
copyVariableIfNotBlank(tmp, properties, "imageSku");
copyVariableIfNotBlank(tmp, properties, "imageVersion");
copyVariableIfNotBlank(tmp, properties, "osType");
putVariable(tmp, "ephemeralOSDisk", Boolean.toString(ephemeralOSDisk));
putVariableIfNotBlank(tmp, "image", template.getImageReference().getUri());
String imageId = getImageId(properties);
// Gallery Image is a special case for custom image, reuse the logic of custom image by replacing the imageId here
if (referenceType == ImageReferenceType.GALLERY) {
GalleryImageVersion galleryImageVersion;
String galleryImageVersionStr = template.getImageReference().getGalleryImageVersion();
String galleryImageDefinition = template.getImageReference().getGalleryImageDefinition();
String gallerySubscriptionId = template.getImageReference().getGallerySubscriptionId();
String galleryResourceGroup = template.getImageReference().getGalleryResourceGroup();
String galleryName = template.getImageReference().getGalleryName();
if (StringUtils.isBlank(galleryImageVersionStr) || StringUtils.isBlank(galleryImageDefinition) || StringUtils.isBlank(galleryResourceGroup) || StringUtils.isBlank(galleryName)) {
throw AzureCloudException.create("One of gallery name, gallery image version, image definition and image resource group " + "is blank.");
}
AzureResourceManager client = AzureResourceManagerCache.get(azureCredentialsId, gallerySubscriptionId);
if (Constants.VERSION_LATEST.equals(galleryImageVersionStr)) {
galleryImageVersion = getGalleryImageLatestVersion(galleryResourceGroup, galleryName, galleryImageDefinition, client);
} else {
galleryImageVersion = client.galleryImageVersions().getByGalleryImage(galleryResourceGroup, galleryName, galleryImageDefinition, galleryImageVersionStr);
}
if (galleryImageVersion == null) {
throw AzureCloudException.create("Can not find the right version for the gallery image.");
}
imageId = galleryImageVersion.id();
LOGGER.log(Level.INFO, "Create VM with gallery image id {0}", new Object[] { imageId });
putVariableIfNotBlank(tmp, "imageId", imageId);
}
if (imageId != null) {
addTagToVm(tmp, "JenkinsImageId", imageId);
}
// add variables for that and upload the init script to the storage account
if (useCustomScriptExtension) {
String rootUrl = fixEmpty(Jenkins.get().getRootUrl());
if (rootUrl == null) {
throw AzureCloudException.create("Jenkins URL must be set");
}
putVariable(tmp, "jenkinsServerURL", rootUrl);
// Calculate the client secrets. The secrets are based off the machine name,
ArrayNode clientSecretsNode = ((ObjectNode) tmp.get("variables")).putArray("clientSecrets");
for (int i = 0; i < numberOfAgents; i++) {
clientSecretsNode.add(JnlpAgentReceiver.SLAVE_SECRET.mac(String.format("%s%d", vmBaseName, i)));
}
// Upload the startup script to blob storage
String scriptName = String.format("%s%s", deploymentName, "init.ps1");
String initScript;
if (preInstallSshInWindows) {
initScript = loadScript(PRE_INSTALL_SSH_FILENAME);
} else {
initScript = (String) properties.get("initScript");
}
scriptUri = uploadCustomScript(template, scriptName, initScript);
putVariable(tmp, "startupScriptURI", scriptUri);
putVariable(tmp, "startupScriptName", scriptName);
List<StorageAccountKey> storageKeys = azureClient.storageAccounts().getByResourceGroup(template.getResourceGroupName(), storageAccountName).getKeys();
if (storageKeys.isEmpty()) {
throw AzureCloudException.create("Exception occurred while fetching the storage account key");
}
String storageAccountKey = storageKeys.get(0).value();
final ObjectNode storageAccountKeyNode = MAPPER.createObjectNode();
storageAccountKeyNode.put("type", "secureString");
storageAccountKeyNode.put("defaultValue", storageAccountKey);
// Add the storage account key
((ObjectNode) tmp.get("parameters")).replace("storageAccountKey", storageAccountKeyNode);
}
putVariable(tmp, "vmSize", template.getVirtualMachineSize());
// Grab the username/pass
StandardUsernamePasswordCredentials creds = template.getVMCredentials();
putVariable(tmp, "adminUsername", creds.getUsername());
putVariableIfNotBlank(tmp, "storageAccountName", storageAccountName);
putVariableIfNotBlank(tmp, "storageAccountType", storageAccountType);
putVariableIfNotBlank(tmp, "blobEndpointSuffix", blobEndpointSuffix);
// then subnet name can't be either (based on verification rules)
if (!isBasic && StringUtils.isNotBlank((String) properties.get("virtualNetworkName"))) {
copyVariableIfNotBlank(tmp, properties, "virtualNetworkName");
copyVariable(tmp, properties, "subnetName");
if (StringUtils.isNotBlank((String) properties.get("virtualNetworkResourceGroupName"))) {
copyVariable(tmp, properties, "virtualNetworkResourceGroupName");
} else {
putVariable(tmp, "virtualNetworkResourceGroupName", resourceGroupName);
}
} else {
addDefaultVNetResourceNode(tmp, resourceGroupName, tags);
}
if (template.isSpotInstance()) {
addSpotInstance(tmp);
}
if (!(Boolean) properties.get("usePrivateIP")) {
addPublicIPResourceNode(tmp, tags);
}
if (template.isAcceleratedNetworking()) {
addAcceleratedNetworking(tmp);
}
if (StringUtils.isNotBlank((String) properties.get("nsgName"))) {
addNSGNode(tmp, (String) properties.get("nsgName"));
}
final ObjectNode parameters = MAPPER.createObjectNode();
defineParameter(tmp, "adminPassword", "secureString");
putParameter(parameters, "adminPassword", creds.getPassword().getPlainText());
// Register the deployment for cleanup
deploymentRegistrar.registerDeployment(cloudName, template.getResourceGroupName(), deploymentName, scriptUri);
// Create the deployment
String templateJson = tmp.toString();
LOGGER.log(Level.FINE, templateJson);
azureClient.deployments().define(deploymentName).withExistingResourceGroup(template.getResourceGroupName()).withTemplate(templateJson).withParameters(parameters.toString()).withMode(DeploymentMode.INCREMENTAL).beginCreate();
return new AzureVMDeploymentInfo(deploymentName, vmBaseName, numberOfAgents);
} catch (Exception e) {
LOGGER.log(Level.SEVERE, String.format("Unable to deploy %d %s", numberOfAgents, template.getTemplateName()), e);
// Pass the info off to the template so that it can be queued for update.
template.handleTemplateProvisioningFailure(e.getMessage(), FailureStage.PROVISIONING);
try {
removeStorageBlob(new URI(scriptUri), template.getResourceGroupName());
} catch (Exception ex) {
LOGGER.log(Level.WARNING, "Delete initScript failed: {0}", scriptUri);
}
throw AzureCloudException.create(e);
} finally {
if (embeddedTemplate != null) {
embeddedTemplate.close();
}
}
}
use of com.azure.resourcemanager.storage.models.StorageAccountKey in project azure-vm-agents-plugin by jenkinsci.
the class IntegrationTest method downloadFromAzure.
protected String downloadFromAzure(String resourceGroup, String storageAccountName, String containerName, String fileName) throws IOException {
StorageAccount storageAccount = azureClient.storageAccounts().getByResourceGroup(resourceGroup, storageAccountName);
List<StorageAccountKey> storageKeys = storageAccount.getKeys();
String storageAccountKey = storageKeys.get(0).value();
BlobServiceClient account = new BlobServiceClientBuilder().credential(new StorageSharedKeyCredential(storageAccountName, storageAccountKey)).endpoint(storageAccount.endPoints().primary().blob()).buildClient();
BlobContainerClient blobClient = account.getBlobContainerClient(containerName);
if (!blobClient.exists()) {
blobClient.create();
}
BlobClient blob = blobClient.getBlobClient(fileName);
ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
blob.download(byteArrayOutputStream);
return byteArrayOutputStream.toString(StandardCharsets.UTF_8.name());
}
use of com.azure.resourcemanager.storage.models.StorageAccountKey in project azure-vm-agents-plugin by jenkinsci.
the class IntegrationTest method blobExists.
protected boolean blobExists(String storageURI) {
try {
BlobUrlParts blobUrlParts = BlobUrlParts.parse(storageURI);
String storageAccountName = blobUrlParts.getAccountName();
final String containerName = blobUrlParts.getBlobContainerName();
final String blobName = blobUrlParts.getBlobName();
StorageAccount storageAccount = azureClient.storageAccounts().getByResourceGroup(testEnv.azureResourceGroup, storageAccountName);
List<StorageAccountKey> storageKeys = storageAccount.getKeys();
if (storageKeys.isEmpty()) {
return false;
} else {
String storageAccountKey = storageKeys.get(0).value();
BlobServiceClient account = new BlobServiceClientBuilder().credential(new StorageSharedKeyCredential(storageAccountName, storageAccountKey)).endpoint(storageAccount.endPoints().primary().blob()).buildClient();
BlobContainerClient blobClient = account.getBlobContainerClient(containerName);
return blobClient.getBlobClient(blobName).exists();
}
} catch (Exception e) {
LOGGER.log(Level.SEVERE, e.getMessage(), e);
return false;
}
}
use of com.azure.resourcemanager.storage.models.StorageAccountKey in project azure-vm-agents-plugin by jenkinsci.
the class IntegrationTest method uploadFile.
protected String uploadFile(StorageAccount storageAccount, String uploadFileName, String writtenData, String containerName) throws Exception {
List<StorageAccountKey> storageKeys = storageAccount.getKeys();
if (storageKeys.isEmpty()) {
throw new Exception("Can't find key");
}
String storageAccountKey = storageKeys.get(0).value();
BlobServiceClient account = new BlobServiceClientBuilder().credential(new StorageSharedKeyCredential(testEnv.azureStorageAccountName, storageAccountKey)).endpoint(storageAccount.endPoints().primary().blob()).buildClient();
BlobContainerClient container = account.getBlobContainerClient(containerName);
if (!container.exists()) {
container.create();
}
BlobClient blob = container.getBlobClient(uploadFileName);
ByteArrayInputStream stream = new ByteArrayInputStream(writtenData.getBytes(StandardCharsets.UTF_8));
blob.upload(stream, writtenData.length());
return blob.getBlobUrl();
}
Aggregations