use of com.cloud.agent.resource.kvm.storage.utils.QemuImg in project cosmic by MissionCriticalCloud.
the class KvmStorageProcessor method backupSnapshot.
@Override
public Answer backupSnapshot(final CopyCommand cmd) {
final DataTO srcData = cmd.getSrcTO();
final DataTO destData = cmd.getDestTO();
final SnapshotObjectTO snapshot = (SnapshotObjectTO) srcData;
final PrimaryDataStoreTO primaryStore = (PrimaryDataStoreTO) snapshot.getDataStore();
final SnapshotObjectTO destSnapshot = (SnapshotObjectTO) destData;
final DataStoreTO imageStore = destData.getDataStore();
final NfsTO nfsImageStore = (NfsTO) imageStore;
final String secondaryStoragePoolUrl = nfsImageStore.getUrl();
// NOTE: snapshot name is encoded in snapshot path
final int index = snapshot.getPath().lastIndexOf("/");
// -1 means the snapshot is created from existing vm snapshot
final boolean isCreatedFromVmSnapshot = index == -1;
final String snapshotName = snapshot.getPath().substring(index + 1);
String descName = snapshotName;
final String volumePath = snapshot.getVolume().getPath();
final String snapshotDestPath;
final String snapshotRelPath;
final String vmName = snapshot.getVmName();
KvmStoragePool secondaryStoragePool = null;
Connect conn = null;
KvmPhysicalDisk snapshotDisk = null;
KvmStoragePool primaryPool = null;
try {
conn = LibvirtConnection.getConnectionByVmName(vmName);
secondaryStoragePool = this.storagePoolMgr.getStoragePoolByUri(secondaryStoragePoolUrl);
final String ssPmountPath = secondaryStoragePool.getLocalPath();
snapshotRelPath = destSnapshot.getPath();
snapshotDestPath = ssPmountPath + File.separator + snapshotRelPath;
snapshotDisk = this.storagePoolMgr.getPhysicalDisk(primaryStore.getPoolType(), primaryStore.getUuid(), volumePath);
primaryPool = snapshotDisk.getPool();
long size = 0;
if (primaryPool.getType() == StoragePoolType.RBD) {
final String rbdSnapshot = snapshotDisk.getPath() + "@" + snapshotName;
final String snapshotFile = snapshotDestPath + "/" + snapshotName;
try {
this.logger.debug("Attempting to backup RBD snapshot " + rbdSnapshot);
final File snapDir = new File(snapshotDestPath);
this.logger.debug("Attempting to create " + snapDir.getAbsolutePath() + " recursively for snapshot storage");
FileUtils.forceMkdir(snapDir);
final QemuImgFile srcFile = new QemuImgFile(KvmPhysicalDisk.rbdStringBuilder(primaryPool.getSourceHost(), primaryPool.getSourcePort(), primaryPool.getAuthUserName(), primaryPool.getAuthSecret(), rbdSnapshot));
srcFile.setFormat(snapshotDisk.getFormat());
final QemuImgFile destFile = new QemuImgFile(snapshotFile);
destFile.setFormat(PhysicalDiskFormat.QCOW2);
this.logger.debug("Backing up RBD snapshot " + rbdSnapshot + " to " + snapshotFile);
final QemuImg q = new QemuImg(cmd.getWaitInMillSeconds());
q.convert(srcFile, destFile);
final File snapFile = new File(snapshotFile);
if (snapFile.exists()) {
size = snapFile.length();
}
this.logger.debug("Finished backing up RBD snapshot " + rbdSnapshot + " to " + snapshotFile + " Snapshot size: " + size);
} catch (final FileNotFoundException e) {
this.logger.error("Failed to open " + snapshotDestPath + ". The error was: " + e.getMessage());
return new CopyCmdAnswer(e.toString());
} catch (final IOException e) {
this.logger.error("Failed to create " + snapshotDestPath + ". The error was: " + e.getMessage());
return new CopyCmdAnswer(e.toString());
} catch (final QemuImgException e) {
this.logger.error("Failed to backup the RBD snapshot from " + rbdSnapshot + " to " + snapshotFile + " the error was: " + e.getMessage());
return new CopyCmdAnswer(e.toString());
}
} else {
final Script command = new Script(this.manageSnapshotPath, cmd.getWaitInMillSeconds(), this.logger);
command.add("-b", snapshotDisk.getPath());
command.add("-n", snapshotName);
command.add("-p", snapshotDestPath);
if (isCreatedFromVmSnapshot) {
descName = UUID.randomUUID().toString();
}
command.add("-t", descName);
final String result = command.execute();
if (result != null) {
this.logger.debug("Failed to backup snaptshot: " + result);
return new CopyCmdAnswer(result);
}
final File snapFile = new File(snapshotDestPath + "/" + descName);
if (snapFile.exists()) {
size = snapFile.length();
}
}
final SnapshotObjectTO newSnapshot = new SnapshotObjectTO();
newSnapshot.setPath(snapshotRelPath + File.separator + descName);
newSnapshot.setPhysicalSize(size);
return new CopyCmdAnswer(newSnapshot);
} catch (final LibvirtException | CloudRuntimeException e) {
this.logger.debug("Failed to backup snapshot: ", e);
return new CopyCmdAnswer(e.toString());
} finally {
if (isCreatedFromVmSnapshot) {
this.logger.debug("Ignoring removal of vm snapshot on primary as this snapshot is created from vm snapshot");
} else {
try {
/* Delete the snapshot on primary */
DomainState state = null;
Domain vm = null;
if (vmName != null) {
try {
vm = this.resource.getDomain(conn, vmName);
state = vm.getInfo().state;
} catch (final LibvirtException e) {
this.logger.trace("Ignoring libvirt error.", e);
}
}
final KvmStoragePool primaryStorage = this.storagePoolMgr.getStoragePool(primaryStore.getPoolType(), primaryStore.getUuid());
if (state == DomainState.VIR_DOMAIN_RUNNING && !primaryStorage.isExternalSnapshot()) {
final DomainSnapshot snap = vm.snapshotLookupByName(snapshotName);
snap.delete(0);
/*
* libvirt on RHEL6 doesn't handle resume event emitted from
* qemu
*/
vm = this.resource.getDomain(conn, vmName);
state = vm.getInfo().state;
if (state == DomainInfo.DomainState.VIR_DOMAIN_PAUSED) {
vm.resume();
}
} else {
if (primaryPool.getType() != StoragePoolType.RBD) {
final Script command = new Script(this.manageSnapshotPath, this.cmdsTimeout, this.logger);
command.add("-d", snapshotDisk.getPath());
command.add("-n", snapshotName);
final String result = command.execute();
if (result != null) {
this.logger.debug("Failed to delete snapshot on primary: " + result);
// return new CopyCmdAnswer("Failed to backup snapshot: " + result);
}
}
}
} catch (final Exception ex) {
this.logger.debug("Failed to delete snapshots on primary", ex);
}
}
try {
if (secondaryStoragePool != null) {
secondaryStoragePool.delete();
}
} catch (final Exception ex) {
this.logger.debug("Failed to delete secondary storage", ex);
}
}
}
use of com.cloud.agent.resource.kvm.storage.utils.QemuImg in project cosmic by MissionCriticalCloud.
the class KvmStorageProcessor method createTemplateFromVolume.
@Override
public Answer createTemplateFromVolume(final CopyCommand cmd) {
final DataTO srcData = cmd.getSrcTO();
final DataTO destData = cmd.getDestTO();
final int wait = cmd.getWaitInMillSeconds();
final TemplateObjectTO template = (TemplateObjectTO) destData;
final DataStoreTO imageStore = template.getDataStore();
final VolumeObjectTO volume = (VolumeObjectTO) srcData;
final PrimaryDataStoreTO primaryStore = (PrimaryDataStoreTO) volume.getDataStore();
if (!(imageStore instanceof NfsTO)) {
return new CopyCmdAnswer("unsupported protocol");
}
final NfsTO nfsImageStore = (NfsTO) imageStore;
KvmStoragePool secondaryStorage = null;
final KvmStoragePool primary;
try {
final String templateFolder = template.getPath();
secondaryStorage = this.storagePoolMgr.getStoragePoolByUri(nfsImageStore.getUrl());
primary = this.storagePoolMgr.getStoragePool(primaryStore.getPoolType(), primaryStore.getUuid());
final KvmPhysicalDisk disk = this.storagePoolMgr.getPhysicalDisk(primaryStore.getPoolType(), primaryStore.getUuid(), volume.getPath());
final String tmpltPath = secondaryStorage.getLocalPath() + File.separator + templateFolder;
this.storageLayer.mkdirs(tmpltPath);
final String templateName = UUID.randomUUID().toString();
if (primary.getType() != StoragePoolType.RBD) {
final Script command = new Script(this.createTmplPath, wait, this.logger);
command.add("-f", disk.getPath());
command.add("-t", tmpltPath);
command.add("-n", templateName + ".qcow2");
final String result = command.execute();
if (result != null) {
this.logger.debug("failed to create template: " + result);
return new CopyCmdAnswer(result);
}
} else {
this.logger.debug("Converting RBD disk " + disk.getPath() + " into template " + templateName);
final QemuImgFile srcFile = new QemuImgFile(KvmPhysicalDisk.rbdStringBuilder(primary.getSourceHost(), primary.getSourcePort(), primary.getAuthUserName(), primary.getAuthSecret(), disk.getPath()));
srcFile.setFormat(PhysicalDiskFormat.RAW);
final QemuImgFile destFile = new QemuImgFile(tmpltPath + "/" + templateName + ".qcow2");
destFile.setFormat(PhysicalDiskFormat.QCOW2);
final QemuImg q = new QemuImg(cmd.getWaitInMillSeconds());
try {
q.convert(srcFile, destFile);
} catch (final QemuImgException e) {
final String message = "Failed to create new template while converting " + srcFile.getFileName() + " to " + destFile.getFileName() + " the error was: " + e.getMessage();
throw new QemuImgException(message);
}
final File templateProp = new File(tmpltPath + "/template.properties");
if (!templateProp.exists()) {
templateProp.createNewFile();
}
String templateContent = "filename=" + templateName + ".qcow2" + System.getProperty("line.separator");
final DateFormat dateFormat = new SimpleDateFormat("MM_dd_yyyy");
final Date date = new Date();
templateContent += "snapshot.name=" + dateFormat.format(date) + System.getProperty("line.separator");
try (final FileOutputStream templFo = new FileOutputStream(templateProp)) {
templFo.write(templateContent.getBytes());
templFo.flush();
} catch (final IOException e) {
throw e;
}
}
final Map<String, Object> params = new HashMap<>();
params.put(StorageLayer.InstanceConfigKey, this.storageLayer);
final Processor qcow2Processor = new QCOW2Processor();
qcow2Processor.configure("QCOW2 Processor", params);
final TemplateFormatInfo info = qcow2Processor.process(tmpltPath, null, templateName);
final TemplateLocation loc = new TemplateLocation(this.storageLayer, tmpltPath);
loc.create(1, true, templateName);
loc.addFormat(info);
loc.save();
final TemplateObjectTO newTemplate = new TemplateObjectTO();
newTemplate.setPath(templateFolder + File.separator + templateName + ".qcow2");
newTemplate.setSize(info.virtualSize);
newTemplate.setPhysicalSize(info.size);
newTemplate.setFormat(ImageFormat.QCOW2);
newTemplate.setName(templateName);
return new CopyCmdAnswer(newTemplate);
} catch (final QemuImgException e) {
this.logger.error(e.getMessage());
return new CopyCmdAnswer(e.toString());
} catch (final Exception e) {
this.logger.debug("Failed to createTemplateFromVolume: ", e);
return new CopyCmdAnswer(e.toString());
} finally {
if (secondaryStorage != null) {
secondaryStorage.delete();
}
}
}
use of com.cloud.agent.resource.kvm.storage.utils.QemuImg in project cosmic by MissionCriticalCloud.
the class LibvirtStorageAdaptor method copyPhysicalDisk.
@Override
public KvmPhysicalDisk copyPhysicalDisk(final KvmPhysicalDisk disk, final String name, final KvmStoragePool destPool, final int timeout) {
final KvmStoragePool srcPool = disk.getPool();
final PhysicalDiskFormat sourceFormat = disk.getFormat();
final String sourcePath = disk.getPath();
KvmPhysicalDisk newDisk;
this.logger.debug("copyPhysicalDisk: disk size:" + disk.getSize() + ", virtualsize:" + disk.getVirtualSize() + " format:" + disk.getFormat());
if (destPool.getType() != StoragePoolType.RBD) {
if (disk.getFormat() == PhysicalDiskFormat.TAR) {
newDisk = destPool.createPhysicalDisk(name, PhysicalDiskFormat.DIR, StorageProvisioningType.THIN, disk.getVirtualSize());
} else {
newDisk = destPool.createPhysicalDisk(name, StorageProvisioningType.THIN, disk.getVirtualSize());
}
} else {
newDisk = new KvmPhysicalDisk(destPool.getSourceDir() + "/" + name, name, destPool);
newDisk.setFormat(PhysicalDiskFormat.RAW);
newDisk.setSize(disk.getVirtualSize());
newDisk.setVirtualSize(disk.getSize());
}
final String destPath = newDisk.getPath();
final PhysicalDiskFormat destFormat = newDisk.getFormat();
final QemuImg qemu = new QemuImg(timeout);
QemuImgFile srcFile = null;
QemuImgFile destFile = null;
if (srcPool.getType() != StoragePoolType.RBD && destPool.getType() != StoragePoolType.RBD) {
if (sourceFormat == PhysicalDiskFormat.TAR && destFormat == PhysicalDiskFormat.DIR) {
// LXC template
Script.runSimpleBashScript("cp " + sourcePath + " " + destPath);
} else if (sourceFormat == PhysicalDiskFormat.TAR) {
Script.runSimpleBashScript("tar -x -f " + sourcePath + " -C " + destPath, timeout);
} else if (sourceFormat == PhysicalDiskFormat.DIR) {
Script.runSimpleBashScript("mkdir -p " + destPath);
Script.runSimpleBashScript("chmod 755 " + destPath);
Script.runSimpleBashScript("cp -p -r " + sourcePath + "/* " + destPath, timeout);
} else {
srcFile = new QemuImgFile(sourcePath, sourceFormat);
try {
final Map<String, String> info = qemu.info(srcFile);
final String backingFile = info.get("backing_file");
// qcow2 templates can just be copied into place
if (sourceFormat.equals(destFormat) && backingFile == null) {
final String result = Script.runSimpleBashScript("cp -f " + sourcePath + " " + destPath, timeout);
if (result != null) {
throw new CloudRuntimeException("Failed to create disk: " + result);
}
} else {
destFile = new QemuImgFile(destPath, destFormat);
try {
qemu.convert(srcFile, destFile);
final Map<String, String> destInfo = qemu.info(destFile);
final Long virtualSize = Long.parseLong(destInfo.get("virtual_size"));
newDisk.setVirtualSize(virtualSize);
newDisk.setSize(virtualSize);
} catch (final QemuImgException e) {
this.logger.error("Failed to convert " + srcFile.getFileName() + " to " + destFile.getFileName() + " the error was: " + e.getMessage());
newDisk = null;
}
}
} catch (final QemuImgException e) {
this.logger.error("Failed to fetch the information of file " + srcFile.getFileName() + " the error was: " + e.getMessage());
newDisk = null;
}
}
} else if (srcPool.getType() != StoragePoolType.RBD && destPool.getType() == StoragePoolType.RBD) {
/**
* Using qemu-img we copy the QCOW2 disk to RAW (on RBD) directly. To do so it's mandatory that librbd on the
* system is at least 0.67.7 (Ceph Dumpling)
*/
this.logger.debug("The source image is not RBD, but the destination is. We will convert into RBD format 2");
try {
srcFile = new QemuImgFile(sourcePath, sourceFormat);
final String rbdDestPath = destPool.getSourceDir() + "/" + name;
final String rbdDestFile = KvmPhysicalDisk.rbdStringBuilder(destPool.getSourceHost(), destPool.getSourcePort(), destPool.getAuthUserName(), destPool.getAuthSecret(), rbdDestPath);
destFile = new QemuImgFile(rbdDestFile, destFormat);
this.logger.debug("Starting copy from source image " + srcFile.getFileName() + " to RBD image " + rbdDestPath);
qemu.convert(srcFile, destFile);
this.logger.debug("Succesfully converted source image " + srcFile.getFileName() + " to RBD image " + rbdDestPath);
/* We have to stat the RBD image to see how big it became afterwards */
final Rados r = new Rados(destPool.getAuthUserName());
r.confSet("mon_host", destPool.getSourceHost() + ":" + destPool.getSourcePort());
r.confSet("key", destPool.getAuthSecret());
r.confSet("client_mount_timeout", "30");
r.connect();
this.logger.debug("Succesfully connected to Ceph cluster at " + r.confGet("mon_host"));
final IoCTX io = r.ioCtxCreate(destPool.getSourceDir());
final Rbd rbd = new Rbd(io);
final RbdImage image = rbd.open(name);
final RbdImageInfo rbdInfo = image.stat();
newDisk.setSize(rbdInfo.size);
newDisk.setVirtualSize(rbdInfo.size);
this.logger.debug("After copy the resulting RBD image " + rbdDestPath + " is " + rbdInfo.size + " bytes long");
rbd.close(image);
r.ioCtxDestroy(io);
} catch (final QemuImgException e) {
this.logger.error("Failed to convert from " + srcFile.getFileName() + " to " + destFile.getFileName() + " the error was: " + e.getMessage());
newDisk = null;
} catch (final RadosException e) {
this.logger.error("A Ceph RADOS operation failed (" + e.getReturnValue() + "). The error was: " + e.getMessage());
newDisk = null;
} catch (final RbdException e) {
this.logger.error("A Ceph RBD operation failed (" + e.getReturnValue() + "). The error was: " + e.getMessage());
newDisk = null;
}
} else {
/**
* We let Qemu-Img do the work here. Although we could work with librbd and have that do the cloning it doesn't
* benefit us. It's better to keep the current code in place which works
*/
srcFile = new QemuImgFile(KvmPhysicalDisk.rbdStringBuilder(srcPool.getSourceHost(), srcPool.getSourcePort(), srcPool.getAuthUserName(), srcPool.getAuthSecret(), sourcePath));
srcFile.setFormat(sourceFormat);
destFile = new QemuImgFile(destPath);
destFile.setFormat(destFormat);
try {
qemu.convert(srcFile, destFile);
} catch (final QemuImgException e) {
this.logger.error("Failed to convert " + srcFile.getFileName() + " to " + destFile.getFileName() + " the error was: " + e.getMessage());
newDisk = null;
}
}
if (newDisk == null) {
throw new CloudRuntimeException("Failed to copy " + disk.getPath() + " to " + name);
}
return newDisk;
}
use of com.cloud.agent.resource.kvm.storage.utils.QemuImg in project cosmic by MissionCriticalCloud.
the class LibvirtStorageAdaptor method createPhysicalDiskByQemuImg.
private KvmPhysicalDisk createPhysicalDiskByQemuImg(final String name, final KvmStoragePool pool, final PhysicalDiskFormat format, final StorageProvisioningType provisioningType, final long size) {
final String volPath = pool.getLocalPath() + "/" + name;
long virtualSize = 0;
long actualSize = 0;
final int timeout = 0;
final QemuImgFile destFile = new QemuImgFile(volPath);
destFile.setFormat(format);
destFile.setSize(size);
final QemuImg qemu = new QemuImg(timeout);
final Map<String, String> options = new HashMap<>();
if (pool.getType() == StoragePoolType.NetworkFilesystem) {
options.put("preallocation", PreallocationType.getPreallocationType(provisioningType).toString().toLowerCase());
}
try {
qemu.create(destFile, options);
final Map<String, String> info = qemu.info(destFile);
virtualSize = Long.parseLong(info.get("virtual_size"));
actualSize = new File(destFile.getFileName()).length();
} catch (final QemuImgException e) {
this.logger.error("Failed to create " + volPath + " due to a failed executing of qemu-img: " + e.getMessage());
throw new CloudRuntimeException(e.toString());
}
final KvmPhysicalDisk disk = new KvmPhysicalDisk(volPath, name, pool);
disk.setFormat(format);
disk.setSize(actualSize);
disk.setVirtualSize(virtualSize);
return disk;
}
use of com.cloud.agent.resource.kvm.storage.utils.QemuImg in project cosmic by MissionCriticalCloud.
the class LibvirtStorageAdaptor method createDiskFromTemplate.
@Override
public KvmPhysicalDisk createDiskFromTemplate(final KvmPhysicalDisk template, final String name, final PhysicalDiskFormat format, final StorageProvisioningType provisioningType, final long size, final KvmStoragePool destPool, final int timeout) {
this.logger.info("Creating volume " + name + " from template " + template.getName() + " in pool " + destPool.getUuid() + " (" + destPool.getType().toString() + ") with size " + size);
KvmPhysicalDisk disk = null;
if (destPool.getType() == StoragePoolType.RBD) {
disk = createDiskFromTemplateOnRbd(template, name, size, destPool, timeout);
} else {
try {
disk = destPool.createPhysicalDisk(name, format, provisioningType, template.getVirtualSize());
if (disk == null) {
throw new CloudRuntimeException("Failed to create disk from template " + template.getName());
}
if (template.getFormat() == PhysicalDiskFormat.TAR) {
Script.runSimpleBashScript("tar -x -f " + template.getPath() + " -C " + disk.getPath(), timeout);
} else if (template.getFormat() == PhysicalDiskFormat.DIR) {
Script.runSimpleBashScript("mkdir -p " + disk.getPath());
Script.runSimpleBashScript("chmod 755 " + disk.getPath());
Script.runSimpleBashScript("tar -x -f " + template.getPath() + "/*.tar -C " + disk.getPath(), timeout);
} else if (format == PhysicalDiskFormat.QCOW2) {
final QemuImg qemu = new QemuImg(timeout);
final QemuImgFile destFile = new QemuImgFile(disk.getPath(), format);
if (size > template.getVirtualSize()) {
destFile.setSize(size);
} else {
destFile.setSize(template.getVirtualSize());
}
final Map<String, String> options = new HashMap<>();
options.put("preallocation", PreallocationType.getPreallocationType(provisioningType).toString().toLowerCase());
switch(provisioningType) {
case THIN:
final QemuImgFile backingFile = new QemuImgFile(template.getPath(), template.getFormat());
qemu.create(destFile, backingFile, options);
break;
case SPARSE:
case FAT:
final QemuImgFile srcFile = new QemuImgFile(template.getPath(), template.getFormat());
qemu.convert(srcFile, destFile, options);
break;
default:
break;
}
} else if (format == PhysicalDiskFormat.RAW) {
final QemuImgFile sourceFile = new QemuImgFile(template.getPath(), template.getFormat());
final QemuImgFile destFile = new QemuImgFile(disk.getPath(), PhysicalDiskFormat.RAW);
if (size > template.getVirtualSize()) {
destFile.setSize(size);
} else {
destFile.setSize(template.getVirtualSize());
}
final QemuImg qemu = new QemuImg(timeout);
final Map<String, String> options = new HashMap<>();
qemu.convert(sourceFile, destFile, options);
}
} catch (final QemuImgException e) {
this.logger.error("Failed to create " + disk.getPath() + " due to a failed executing of qemu-img: " + e.getMessage());
}
}
return disk;
}
Aggregations