Examples with FormatInfo com.cloud.storage.template.Processor.FormatInfo used on opensource projects

Example 6 with FormatInfo

use of com.cloud.storage.template.Processor.FormatInfo in project cloudstack by apache.

the class LibvirtComputingResourceTest method testCreatePrivateTemplateFromSnapshotCommand.

@Test
public void testCreatePrivateTemplateFromSnapshotCommand() {
    final StoragePool pool = Mockito.mock(StoragePool.class);
    final String secondaryStoragePoolURL = "nfs:/127.0.0.1/storage/secondary";
    final Long dcId = 1l;
    final Long accountId = 1l;
    final Long volumeId = 1l;
    final String backedUpSnapshotUuid = "/run/9a0afe7c-26a7-4585-bf87-abf82ae106d9/";
    final String backedUpSnapshotName = "snap";
    final String origTemplateInstallPath = "/install/path/";
    final Long newTemplateId = 2l;
    final String templateName = "templ";
    final int wait = 0;
    final CreatePrivateTemplateFromSnapshotCommand command = new CreatePrivateTemplateFromSnapshotCommand(pool, secondaryStoragePoolURL, dcId, accountId, volumeId, backedUpSnapshotUuid, backedUpSnapshotName, origTemplateInstallPath, newTemplateId, templateName, wait);
    final String templatePath = "/template/path";
    final String localPath = "/mnt/local";
    final String tmplName = "ce97bbc1-34fe-4259-9202-74bbce2562ab";
    final KVMStoragePoolManager storagePoolMgr = Mockito.mock(KVMStoragePoolManager.class);
    final KVMStoragePool secondaryPool = Mockito.mock(KVMStoragePool.class);
    final KVMStoragePool snapshotPool = Mockito.mock(KVMStoragePool.class);
    final KVMPhysicalDisk snapshot = Mockito.mock(KVMPhysicalDisk.class);
    final StorageLayer storage = Mockito.mock(StorageLayer.class);
    final LibvirtUtilitiesHelper libvirtUtilitiesHelper = Mockito.mock(LibvirtUtilitiesHelper.class);
    final TemplateLocation location = Mockito.mock(TemplateLocation.class);
    final Processor qcow2Processor = Mockito.mock(Processor.class);
    final FormatInfo info = Mockito.mock(FormatInfo.class);
    when(libvirtComputingResource.getStoragePoolMgr()).thenReturn(storagePoolMgr);
    String snapshotPath = command.getSnapshotUuid();
    final int index = snapshotPath.lastIndexOf("/");
    snapshotPath = snapshotPath.substring(0, index);
    when(storagePoolMgr.getStoragePoolByURI(command.getSecondaryStorageUrl() + snapshotPath)).thenReturn(snapshotPool);
    when(storagePoolMgr.getStoragePoolByURI(command.getSecondaryStorageUrl())).thenReturn(secondaryPool);
    when(snapshotPool.getPhysicalDisk(command.getSnapshotName())).thenReturn(snapshot);
    when(secondaryPool.getLocalPath()).thenReturn(localPath);
    when(libvirtComputingResource.getStorage()).thenReturn(storage);
    when(libvirtComputingResource.createTmplPath()).thenReturn(templatePath);
    when(libvirtComputingResource.getCmdsTimeout()).thenReturn(1);
    final String templateFolder = command.getAccountId() + File.separator + command.getNewTemplateId();
    final String templateInstallFolder = "template/tmpl/" + templateFolder;
    final String tmplPath = secondaryPool.getLocalPath() + File.separator + templateInstallFolder;
    when(libvirtComputingResource.getLibvirtUtilitiesHelper()).thenReturn(libvirtUtilitiesHelper);
    when(libvirtUtilitiesHelper.buildTemplateLocation(storage, tmplPath)).thenReturn(location);
    when(libvirtUtilitiesHelper.generateUUIDName()).thenReturn(tmplName);
    try {
        when(libvirtUtilitiesHelper.buildQCOW2Processor(storage)).thenReturn(qcow2Processor);
        when(qcow2Processor.process(tmplPath, null, tmplName)).thenReturn(info);
    } catch (final ConfigurationException e) {
        fail(e.getMessage());
    } catch (final InternalErrorException e) {
        fail(e.getMessage());
    }
    final LibvirtRequestWrapper wrapper = LibvirtRequestWrapper.getInstance();
    assertNotNull(wrapper);
    final Answer answer = wrapper.execute(command, libvirtComputingResource);
    assertTrue(answer.getResult());
    verify(libvirtComputingResource, times(1)).getStoragePoolMgr();
    verify(storagePoolMgr, times(1)).getStoragePoolByURI(command.getSecondaryStorageUrl() + snapshotPath);
    verify(storagePoolMgr, times(1)).getStoragePoolByURI(command.getSecondaryStorageUrl());
}

Example 7 with FormatInfo

use of com.cloud.storage.template.Processor.FormatInfo in project cloudstack by apache.

the class LibvirtComputingResourceTest method testCreatePrivateTemplateFromSnapshotCommandIOException.

@SuppressWarnings("unchecked")
@Test
public void testCreatePrivateTemplateFromSnapshotCommandIOException() {
    final StoragePool pool = Mockito.mock(StoragePool.class);
    final String secondaryStoragePoolURL = "nfs:/127.0.0.1/storage/secondary";
    final Long dcId = 1l;
    final Long accountId = 1l;
    final Long volumeId = 1l;
    final String backedUpSnapshotUuid = "/run/9a0afe7c-26a7-4585-bf87-abf82ae106d9/";
    final String backedUpSnapshotName = "snap";
    final String origTemplateInstallPath = "/install/path/";
    final Long newTemplateId = 2l;
    final String templateName = "templ";
    final int wait = 0;
    final CreatePrivateTemplateFromSnapshotCommand command = new CreatePrivateTemplateFromSnapshotCommand(pool, secondaryStoragePoolURL, dcId, accountId, volumeId, backedUpSnapshotUuid, backedUpSnapshotName, origTemplateInstallPath, newTemplateId, templateName, wait);
    final String templatePath = "/template/path";
    final String localPath = "/mnt/local";
    final String tmplName = "ce97bbc1-34fe-4259-9202-74bbce2562ab";
    final KVMStoragePoolManager storagePoolMgr = Mockito.mock(KVMStoragePoolManager.class);
    final KVMStoragePool secondaryPool = Mockito.mock(KVMStoragePool.class);
    final KVMStoragePool snapshotPool = Mockito.mock(KVMStoragePool.class);
    final KVMPhysicalDisk snapshot = Mockito.mock(KVMPhysicalDisk.class);
    final StorageLayer storage = Mockito.mock(StorageLayer.class);
    final LibvirtUtilitiesHelper libvirtUtilitiesHelper = Mockito.mock(LibvirtUtilitiesHelper.class);
    final TemplateLocation location = Mockito.mock(TemplateLocation.class);
    final Processor qcow2Processor = Mockito.mock(Processor.class);
    final FormatInfo info = Mockito.mock(FormatInfo.class);
    when(libvirtComputingResource.getStoragePoolMgr()).thenReturn(storagePoolMgr);
    String snapshotPath = command.getSnapshotUuid();
    final int index = snapshotPath.lastIndexOf("/");
    snapshotPath = snapshotPath.substring(0, index);
    when(storagePoolMgr.getStoragePoolByURI(command.getSecondaryStorageUrl() + snapshotPath)).thenReturn(snapshotPool);
    when(storagePoolMgr.getStoragePoolByURI(command.getSecondaryStorageUrl())).thenReturn(secondaryPool);
    when(snapshotPool.getPhysicalDisk(command.getSnapshotName())).thenReturn(snapshot);
    when(secondaryPool.getLocalPath()).thenReturn(localPath);
    when(libvirtComputingResource.getStorage()).thenReturn(storage);
    when(libvirtComputingResource.createTmplPath()).thenReturn(templatePath);
    when(libvirtComputingResource.getCmdsTimeout()).thenReturn(1);
    final String templateFolder = command.getAccountId() + File.separator + command.getNewTemplateId();
    final String templateInstallFolder = "template/tmpl/" + templateFolder;
    final String tmplPath = secondaryPool.getLocalPath() + File.separator + templateInstallFolder;
    when(libvirtComputingResource.getLibvirtUtilitiesHelper()).thenReturn(libvirtUtilitiesHelper);
    when(libvirtUtilitiesHelper.buildTemplateLocation(storage, tmplPath)).thenReturn(location);
    when(libvirtUtilitiesHelper.generateUUIDName()).thenReturn(tmplName);
    try {
        when(libvirtUtilitiesHelper.buildQCOW2Processor(storage)).thenReturn(qcow2Processor);
        when(qcow2Processor.process(tmplPath, null, tmplName)).thenReturn(info);
        when(location.create(1, true, tmplName)).thenThrow(IOException.class);
    } catch (final ConfigurationException e) {
        fail(e.getMessage());
    } catch (final InternalErrorException e) {
        fail(e.getMessage());
    } catch (final IOException e) {
        fail(e.getMessage());
    }
    final LibvirtRequestWrapper wrapper = LibvirtRequestWrapper.getInstance();
    assertNotNull(wrapper);
    final Answer answer = wrapper.execute(command, libvirtComputingResource);
    assertFalse(answer.getResult());
    verify(libvirtComputingResource, times(1)).getStoragePoolMgr();
    verify(storagePoolMgr, times(1)).getStoragePoolByURI(command.getSecondaryStorageUrl() + snapshotPath);
    verify(storagePoolMgr, times(1)).getStoragePoolByURI(command.getSecondaryStorageUrl());
}

Example 8 with FormatInfo

use of com.cloud.storage.template.Processor.FormatInfo in project cloudstack by apache.

the class LibvirtCreatePrivateTemplateFromVolumeCommandWrapper method execute.

@Override
public Answer execute(final CreatePrivateTemplateFromVolumeCommand command, final LibvirtComputingResource libvirtComputingResource) {
    final String secondaryStorageURL = command.getSecondaryStorageUrl();
    KVMStoragePool secondaryStorage = null;
    KVMStoragePool primary = null;
    final KVMStoragePoolManager storagePoolMgr = libvirtComputingResource.getStoragePoolMgr();
    try {
        final String templateFolder = command.getAccountId() + File.separator + command.getTemplateId() + File.separator;
        final String templateInstallFolder = "/template/tmpl/" + templateFolder;
        secondaryStorage = storagePoolMgr.getStoragePoolByURI(secondaryStorageURL);
        try {
            primary = storagePoolMgr.getStoragePool(command.getPool().getType(), command.getPrimaryStoragePoolNameLabel());
        } catch (final CloudRuntimeException e) {
            if (e.getMessage().contains("not found")) {
                primary = storagePoolMgr.createStoragePool(command.getPool().getUuid(), command.getPool().getHost(), command.getPool().getPort(), command.getPool().getPath(), command.getPool().getUserInfo(), command.getPool().getType());
            } else {
                return new CreatePrivateTemplateAnswer(command, false, e.getMessage());
            }
        }
        final KVMPhysicalDisk disk = primary.getPhysicalDisk(command.getVolumePath());
        final String tmpltPath = secondaryStorage.getLocalPath() + File.separator + templateInstallFolder;
        final StorageLayer storage = libvirtComputingResource.getStorage();
        storage.mkdirs(tmpltPath);
        if (primary.getType() != StoragePoolType.RBD) {
            final String createTmplPath = libvirtComputingResource.createTmplPath();
            final int cmdsTimeout = libvirtComputingResource.getCmdsTimeout();
            final Script scriptCommand = new Script(createTmplPath, cmdsTimeout, s_logger);
            scriptCommand.add("-f", disk.getPath());
            scriptCommand.add("-t", tmpltPath);
            scriptCommand.add("-n", command.getUniqueName() + ".qcow2");
            final String result = scriptCommand.execute();
            if (result != null) {
                s_logger.debug("failed to create template: " + result);
                return new CreatePrivateTemplateAnswer(command, false, result);
            }
        } else {
            s_logger.debug("Converting RBD disk " + disk.getPath() + " into template " + command.getUniqueName());
            final QemuImgFile srcFile = new QemuImgFile(KVMPhysicalDisk.RBDStringBuilder(primary.getSourceHost(), primary.getSourcePort(), primary.getAuthUserName(), primary.getAuthSecret(), disk.getPath()));
            srcFile.setFormat(PhysicalDiskFormat.RAW);
            final QemuImgFile destFile = new QemuImgFile(tmpltPath + "/" + command.getUniqueName() + ".qcow2");
            destFile.setFormat(PhysicalDiskFormat.QCOW2);
            final QemuImg q = new QemuImg(0);
            try {
                q.convert(srcFile, destFile);
            } catch (final QemuImgException | LibvirtException e) {
                s_logger.error("Failed to create new template while converting " + srcFile.getFileName() + " to " + destFile.getFileName() + " the error was: " + e.getMessage());
            }
            final File templateProp = new File(tmpltPath + "/template.properties");
            if (!templateProp.exists()) {
                templateProp.createNewFile();
            }
            String templateContent = "filename=" + command.getUniqueName() + ".qcow2" + System.getProperty("line.separator");
            final DateFormat dateFormat = new SimpleDateFormat("MM_dd_yyyy");
            final Date date = new Date();
            templateContent += "snapshot.name=" + dateFormat.format(date) + System.getProperty("line.separator");
            try (FileOutputStream templFo = new FileOutputStream(templateProp)) {
                templFo.write(templateContent.getBytes("UTF-8"));
                templFo.flush();
            } catch (final IOException ex) {
                s_logger.error("CreatePrivateTemplateAnswer:Exception:" + ex.getMessage());
            }
        }
        final Map<String, Object> params = new HashMap<String, Object>();
        params.put(StorageLayer.InstanceConfigKey, storage);
        final Processor qcow2Processor = new QCOW2Processor();
        qcow2Processor.configure("QCOW2 Processor", params);
        final FormatInfo info = qcow2Processor.process(tmpltPath, null, command.getUniqueName());
        final TemplateLocation loc = new TemplateLocation(storage, tmpltPath);
        loc.create(1, true, command.getUniqueName());
        loc.addFormat(info);
        loc.save();
        return new CreatePrivateTemplateAnswer(command, true, null, templateInstallFolder + command.getUniqueName() + ".qcow2", info.virtualSize, info.size, command.getUniqueName(), ImageFormat.QCOW2);
    } catch (final InternalErrorException e) {
        return new CreatePrivateTemplateAnswer(command, false, e.toString());
    } catch (final IOException e) {
        return new CreatePrivateTemplateAnswer(command, false, e.toString());
    } catch (final ConfigurationException e) {
        return new CreatePrivateTemplateAnswer(command, false, e.toString());
    } catch (final CloudRuntimeException e) {
        return new CreatePrivateTemplateAnswer(command, false, e.toString());
    } finally {
        if (secondaryStorage != null) {
            storagePoolMgr.deleteStoragePool(secondaryStorage.getType(), secondaryStorage.getUuid());
        }
    }
}

Example 9 with FormatInfo

use of com.cloud.storage.template.Processor.FormatInfo in project cloudstack by apache.

the class NfsSecondaryStorageResource method postUpload.

public String postUpload(String uuid, String filename, long processTimeout) {
    UploadEntity uploadEntity = uploadEntityStateMap.get(uuid);
    int installTimeoutPerGig = 180 * 60 * 1000;
    String resourcePath = uploadEntity.getInstallPathPrefix();
    // template download
    String finalResourcePath = uploadEntity.getTmpltPath();
    UploadEntity.ResourceType resourceType = uploadEntity.getResourceType();
    String fileSavedTempLocation = uploadEntity.getInstallPathPrefix() + "/" + filename;
    String dummyFileName = "dummy." + uploadEntity.getFormat().getFileExtension();
    if (ImageStoreUtil.isCompressedExtension(filename)) {
        String uploadedFileExtension = FilenameUtils.getExtension(filename);
        dummyFileName += "." + uploadedFileExtension;
    }
    String formatError = ImageStoreUtil.checkTemplateFormat(fileSavedTempLocation, dummyFileName);
    if (StringUtils.isNotBlank(formatError)) {
        String errorString = "File type mismatch between uploaded file and selected format. Selected file format: " + uploadEntity.getFormat() + ". Received: " + formatError;
        s_logger.error(errorString);
        return errorString;
    }
    int imgSizeGigs = getSizeInGB(_storage.getSize(fileSavedTempLocation));
    int maxSize = uploadEntity.getMaxSizeInGB();
    if (imgSizeGigs > maxSize) {
        String errorMessage = "Maximum file upload size exceeded. Physical file size: " + imgSizeGigs + "GB. Maximum allowed size: " + maxSize + "GB.";
        s_logger.error(errorMessage);
        return errorMessage;
    }
    // add one just in case
    imgSizeGigs++;
    long timeout = (long) imgSizeGigs * installTimeoutPerGig;
    Script scr = new Script(getScriptLocation(resourceType), timeout, s_logger);
    scr.add("-s", Integer.toString(imgSizeGigs));
    scr.add("-S", Long.toString(UploadEntity.s_maxTemplateSize));
    if (uploadEntity.getDescription() != null && uploadEntity.getDescription().length() > 1) {
        scr.add("-d", uploadEntity.getDescription());
    }
    if (uploadEntity.isHvm()) {
        scr.add("-h");
    }
    String checkSum = uploadEntity.getChksum();
    if (StringUtils.isNotBlank(checkSum)) {
        scr.add("-c", checkSum);
    }
    // add options common to ISO and template
    String extension = uploadEntity.getFormat().getFileExtension();
    String templateName = "";
    if (extension.equals("iso")) {
        templateName = uploadEntity.getUuid().trim().replace(" ", "_");
    } else {
        try {
            templateName = UUID.nameUUIDFromBytes((uploadEntity.getFilename() + System.currentTimeMillis()).getBytes("UTF-8")).toString();
        } catch (UnsupportedEncodingException e) {
            templateName = uploadEntity.getUuid().trim().replace(" ", "_");
        }
    }
    // run script to mv the temporary template file to the final template
    // file
    String templateFilename = templateName + "." + extension;
    uploadEntity.setTemplatePath(finalResourcePath + "/" + templateFilename);
    scr.add("-n", templateFilename);
    scr.add("-t", resourcePath);
    // this is the temporary
    scr.add("-f", fileSavedTempLocation);
    // template file downloaded
    if (uploadEntity.getChksum() != null && uploadEntity.getChksum().length() > 1) {
        scr.add("-c", uploadEntity.getChksum());
    }
    // cleanup
    scr.add("-u");
    String result;
    result = scr.execute();
    if (result != null) {
        return result;
    }
    // Set permissions for the downloaded template
    File downloadedTemplate = new File(resourcePath + "/" + templateFilename);
    _storage.setWorldReadableAndWriteable(downloadedTemplate);
    // Set permissions for template/volume.properties
    String propertiesFile = resourcePath;
    if (resourceType == UploadEntity.ResourceType.TEMPLATE) {
        propertiesFile += "/template.properties";
    } else {
        propertiesFile += "/volume.properties";
    }
    File templateProperties = new File(propertiesFile);
    _storage.setWorldReadableAndWriteable(templateProperties);
    TemplateLocation loc = new TemplateLocation(_storage, resourcePath);
    try {
        loc.create(uploadEntity.getEntityId(), true, uploadEntity.getFilename());
    } catch (IOException e) {
        s_logger.warn("Something is wrong with template location " + resourcePath, e);
        loc.purge();
        return "Unable to upload due to " + e.getMessage();
    }
    Map<String, Processor> processors = _dlMgr.getProcessors();
    for (Processor processor : processors.values()) {
        FormatInfo info = null;
        try {
            info = processor.process(resourcePath, null, templateName, processTimeout * 1000);
        } catch (InternalErrorException e) {
            s_logger.error("Template process exception ", e);
            return e.toString();
        }
        if (info != null) {
            loc.addFormat(info);
            uploadEntity.setVirtualSize(info.virtualSize);
            uploadEntity.setPhysicalSize(info.size);
            if (info.ovfInformationTO != null) {
                uploadEntity.setOvfInformationTO(info.ovfInformationTO);
            }
            break;
        }
    }
    if (!loc.save()) {
        s_logger.warn("Cleaning up because we're unable to save the formats");
        loc.purge();
    }
    uploadEntity.setStatus(UploadEntity.Status.COMPLETED);
    uploadEntityStateMap.put(uploadEntity.getUuid(), uploadEntity);
    return null;
}

Example 10 with FormatInfo

use of com.cloud.storage.template.Processor.FormatInfo in project cloudstack by apache.

the class NfsSecondaryStorageResource method copySnapshotToTemplateFromNfsToNfsXenserver.

protected Answer copySnapshotToTemplateFromNfsToNfsXenserver(CopyCommand cmd, SnapshotObjectTO srcData, NfsTO srcDataStore, TemplateObjectTO destData, NfsTO destDataStore) {
    String srcMountPoint = getRootDir(srcDataStore.getUrl(), _nfsVersion);
    String snapshotPath = srcData.getPath();
    int index = snapshotPath.lastIndexOf("/");
    String snapshotName = snapshotPath.substring(index + 1);
    if (!snapshotName.startsWith("VHD-") && !snapshotName.endsWith(".vhd")) {
        snapshotName = snapshotName + ".vhd";
    }
    snapshotPath = snapshotPath.substring(0, index);
    snapshotPath = srcMountPoint + File.separator + snapshotPath;
    String destMountPoint = getRootDir(destDataStore.getUrl(), _nfsVersion);
    String destPath = destMountPoint + File.separator + destData.getPath();
    String errMsg = null;
    try {
        _storage.mkdir(destPath);
        String templateUuid = UUID.randomUUID().toString();
        String templateName = templateUuid + ".vhd";
        Script command = new Script(createTemplateFromSnapshotXenScript, cmd.getWait() * 1000, s_logger);
        command.add("-p", snapshotPath);
        command.add("-s", snapshotName);
        command.add("-n", templateName);
        command.add("-t", destPath);
        String result = command.execute();
        if (result != null && !result.equalsIgnoreCase("")) {
            return new CopyCmdAnswer(result);
        }
        Map<String, Object> params = new HashMap<String, Object>();
        params.put(StorageLayer.InstanceConfigKey, _storage);
        Processor processor = new VhdProcessor();
        processor.configure("Vhd Processor", params);
        FormatInfo info = processor.process(destPath, null, templateUuid);
        TemplateLocation loc = new TemplateLocation(_storage, destPath);
        loc.create(1, true, templateUuid);
        loc.addFormat(info);
        loc.save();
        TemplateProp prop = loc.getTemplateInfo();
        TemplateObjectTO newTemplate = new TemplateObjectTO();
        newTemplate.setPath(destData.getPath() + File.separator + templateName);
        newTemplate.setFormat(ImageFormat.VHD);
        newTemplate.setSize(prop.getSize());
        newTemplate.setPhysicalSize(prop.getPhysicalSize());
        newTemplate.setName(templateUuid);
        return new CopyCmdAnswer(newTemplate);
    } catch (ConfigurationException e) {
        s_logger.debug("Failed to create template from snapshot: " + e.toString());
        errMsg = e.toString();
    } catch (InternalErrorException e) {
        s_logger.debug("Failed to create template from snapshot: " + e.toString());
        errMsg = e.toString();
    } catch (IOException e) {
        s_logger.debug("Failed to create template from snapshot: " + e.toString());
        errMsg = e.toString();
    }
    return new CopyCmdAnswer(errMsg);
}