Examples with Processor com.cloud.storage.template.Processor used on opensource projects

Example 11 with Processor

use of com.cloud.storage.template.Processor in project cloudstack by apache.

the class DownloadManagerImpl method configure.

@Override
@SuppressWarnings("unchecked")
public boolean configure(String name, Map<String, Object> params) throws ConfigurationException {
    _name = name;
    String value = null;
    _storage = (StorageLayer) params.get(StorageLayer.InstanceConfigKey);
    if (_storage == null) {
        value = (String) params.get(StorageLayer.ClassConfigKey);
        if (value == null) {
            throw new ConfigurationException("Unable to find the storage layer");
        }
        Class<StorageLayer> clazz;
        try {
            clazz = (Class<StorageLayer>) Class.forName(value);
            _storage = clazz.newInstance();
        } catch (ClassNotFoundException e) {
            throw new ConfigurationException("Unable to instantiate " + value);
        } catch (InstantiationException e) {
            throw new ConfigurationException("Unable to instantiate " + value);
        } catch (IllegalAccessException e) {
            throw new ConfigurationException("Unable to instantiate " + value);
        }
    }
    String inSystemVM = (String) params.get("secondary.storage.vm");
    if (inSystemVM != null && "true".equalsIgnoreCase(inSystemVM)) {
        s_logger.info("DownloadManager: starting additional services since we are inside system vm");
        _nfsVersion = NfsSecondaryStorageResource.retrieveNfsVersionFromParams(params);
        startAdditionalServices();
        blockOutgoingOnPrivate();
    }
    value = (String) params.get("install.timeout.pergig");
    installTimeoutPerGig = NumbersUtil.parseInt(value, 15 * 60) * 1000;
    value = (String) params.get("install.numthreads");
    final int numInstallThreads = NumbersUtil.parseInt(value, 10);
    String scriptsDir = (String) params.get("template.scripts.dir");
    if (scriptsDir == null) {
        scriptsDir = "scripts/storage/secondary";
    }
    listTmpltScr = Script.findScript(scriptsDir, "listvmtmplt.sh");
    if (listTmpltScr == null) {
        throw new ConfigurationException("Unable to find the listvmtmplt.sh");
    }
    s_logger.info("listvmtmplt.sh found in " + listTmpltScr);
    createTmpltScr = Script.findScript(scriptsDir, "createtmplt.sh");
    if (createTmpltScr == null) {
        throw new ConfigurationException("Unable to find createtmplt.sh");
    }
    s_logger.info("createtmplt.sh found in " + createTmpltScr);
    listVolScr = Script.findScript(scriptsDir, "listvolume.sh");
    if (listVolScr == null) {
        throw new ConfigurationException("Unable to find the listvolume.sh");
    }
    s_logger.info("listvolume.sh found in " + listVolScr);
    createVolScr = Script.findScript(scriptsDir, "createvolume.sh");
    if (createVolScr == null) {
        throw new ConfigurationException("Unable to find createvolume.sh");
    }
    s_logger.info("createvolume.sh found in " + createVolScr);
    _processors = new HashMap<String, Processor>();
    Processor processor = new VhdProcessor();
    processor.configure("VHD Processor", params);
    _processors.put("VHD Processor", processor);
    processor = new IsoProcessor();
    processor.configure("ISO Processor", params);
    _processors.put("ISO Processor", processor);
    processor = new QCOW2Processor();
    processor.configure("QCOW2 Processor", params);
    _processors.put("QCOW2 Processor", processor);
    processor = new OVAProcessor();
    processor.configure("OVA Processor", params);
    _processors.put("OVA Processor", processor);
    processor = new VmdkProcessor();
    processor.configure("VMDK Processor", params);
    _processors.put("VMDK Processor", processor);
    processor = new RawImageProcessor();
    processor.configure("Raw Image Processor", params);
    _processors.put("Raw Image Processor", processor);
    processor = new TARProcessor();
    processor.configure("TAR Processor", params);
    _processors.put("TAR Processor", processor);
    _templateDir = (String) params.get("public.templates.root.dir");
    if (_templateDir == null) {
        _templateDir = TemplateConstants.DEFAULT_TMPLT_ROOT_DIR;
    }
    _templateDir += File.separator + TemplateConstants.DEFAULT_TMPLT_FIRST_LEVEL_DIR;
    _volumeDir = TemplateConstants.DEFAULT_VOLUME_ROOT_DIR + File.separator;
    // Add more processors here.
    threadPool = Executors.newFixedThreadPool(numInstallThreads);
    return true;
}

Example 12 with Processor

use of com.cloud.storage.template.Processor in project cloudstack by apache.

the class DownloadManagerImpl method gatherTemplateInfo.

@Override
public Map<String, TemplateProp> gatherTemplateInfo(String rootDir) {
    Map<String, TemplateProp> result = new HashMap<String, TemplateProp>();
    String templateDir = rootDir + File.separator + _templateDir;
    if (!_storage.exists(templateDir)) {
        _storage.mkdirs(templateDir);
    }
    List<String> publicTmplts = listTemplates(templateDir);
    for (String tmplt : publicTmplts) {
        String path = tmplt.substring(0, tmplt.lastIndexOf(File.separator));
        TemplateLocation loc = new TemplateLocation(_storage, path);
        try {
            if (!loc.load()) {
                s_logger.warn("Post download installation was not completed for " + path);
                // loc.purge();
                _storage.cleanup(path, templateDir);
                continue;
            }
        } catch (IOException e) {
            s_logger.warn("Unable to load template location " + path, e);
            continue;
        }
        TemplateProp tInfo = loc.getTemplateInfo();
        if ((tInfo.getSize() == tInfo.getPhysicalSize()) && (tInfo.getInstallPath().endsWith(ImageFormat.OVA.getFileExtension()))) {
            try {
                Processor processor = _processors.get("OVA Processor");
                OVAProcessor vmdkProcessor = (OVAProcessor) processor;
                long vSize = vmdkProcessor.getTemplateVirtualSize(path, tInfo.getInstallPath().substring(tInfo.getInstallPath().lastIndexOf(File.separator) + 1));
                tInfo.setSize(vSize);
                loc.updateVirtualSize(vSize);
                loc.save();
            } catch (Exception e) {
                s_logger.error("Unable to get the virtual size of the template: " + tInfo.getInstallPath() + " due to " + e.getMessage());
            }
        }
        result.put(tInfo.getTemplateName(), tInfo);
        s_logger.debug("Added template name: " + tInfo.getTemplateName() + ", path: " + tmplt);
    }
    /*
        for (String tmplt : isoTmplts) {
            String tmp[];
            tmp = tmplt.split("/");
            String tmpltName = tmp[tmp.length - 2];
            tmplt = tmplt.substring(tmplt.lastIndexOf("iso/"));
            TemplateInfo tInfo = new TemplateInfo(tmpltName, tmplt, false);
            s_logger.debug("Added iso template name: " + tmpltName + ", path: " + tmplt);
            result.put(tmpltName, tInfo);
        }
         */
    return result;
}

Example 13 with Processor

use of com.cloud.storage.template.Processor in project CloudStack-archive by CloudStack-extras.

the class LibvirtComputingResource method execute.

protected CreatePrivateTemplateAnswer execute(final CreatePrivateTemplateFromSnapshotCommand cmd) {
    String templateFolder = cmd.getAccountId() + File.separator + cmd.getNewTemplateId();
    String templateInstallFolder = "template/tmpl/" + templateFolder;
    String tmplName = UUID.randomUUID().toString();
    String tmplFileName = tmplName + ".qcow2";
    KVMStoragePool secondaryPool = null;
    KVMStoragePool snapshotPool = null;
    try {
        String snapshotPath = cmd.getSnapshotUuid();
        int index = snapshotPath.lastIndexOf("/");
        snapshotPath = snapshotPath.substring(0, index);
        snapshotPool = _storagePoolMgr.getStoragePoolByURI(cmd.getSecondaryStorageUrl() + snapshotPath);
        KVMPhysicalDisk snapshot = snapshotPool.getPhysicalDisk(cmd.getSnapshotName());
        secondaryPool = _storagePoolMgr.getStoragePoolByURI(cmd.getSecondaryStorageUrl());
        String templatePath = secondaryPool.getLocalPath() + File.separator + templateInstallFolder;
        _storage.mkdirs(templatePath);
        String tmplPath = templateInstallFolder + File.separator + tmplFileName;
        Script command = new Script(_createTmplPath, _cmdsTimeout, s_logger);
        command.add("-t", templatePath);
        command.add("-n", tmplFileName);
        command.add("-f", snapshot.getPath());
        command.execute();
        Map<String, Object> params = new HashMap<String, Object>();
        params.put(StorageLayer.InstanceConfigKey, _storage);
        Processor qcow2Processor = new QCOW2Processor();
        qcow2Processor.configure("QCOW2 Processor", params);
        FormatInfo info = qcow2Processor.process(templatePath, null, tmplName);
        TemplateLocation loc = new TemplateLocation(_storage, templatePath);
        loc.create(1, true, tmplName);
        loc.addFormat(info);
        loc.save();
        return new CreatePrivateTemplateAnswer(cmd, true, "", tmplPath, info.virtualSize, info.size, tmplName, info.format);
    } catch (ConfigurationException e) {
        return new CreatePrivateTemplateAnswer(cmd, false, e.getMessage());
    } catch (InternalErrorException e) {
        return new CreatePrivateTemplateAnswer(cmd, false, e.getMessage());
    } catch (IOException e) {
        return new CreatePrivateTemplateAnswer(cmd, false, e.getMessage());
    } catch (CloudRuntimeException e) {
        return new CreatePrivateTemplateAnswer(cmd, false, e.getMessage());
    } finally {
        if (secondaryPool != null) {
            secondaryPool.delete();
        }
        if (snapshotPool != null) {
            snapshotPool.delete();
        }
    }
}

Example 14 with Processor

use of com.cloud.storage.template.Processor in project cloudstack by apache.

the class LibvirtCreatePrivateTemplateFromSnapshotCommandWrapper method execute.

@Override
public Answer execute(final CreatePrivateTemplateFromSnapshotCommand command, final LibvirtComputingResource libvirtComputingResource) {
    final LibvirtUtilitiesHelper libvirtUtilitiesHelper = libvirtComputingResource.getLibvirtUtilitiesHelper();
    final String templateFolder = command.getAccountId() + File.separator + command.getNewTemplateId();
    final String templateInstallFolder = "template/tmpl/" + templateFolder;
    final String tmplName = libvirtUtilitiesHelper.generateUUIDName();
    final String tmplFileName = tmplName + ".qcow2";
    KVMStoragePool secondaryPool = null;
    KVMStoragePool snapshotPool = null;
    final KVMStoragePoolManager storagePoolMgr = libvirtComputingResource.getStoragePoolMgr();
    try {
        String snapshotPath = command.getSnapshotUuid();
        final int index = snapshotPath.lastIndexOf("/");
        snapshotPath = snapshotPath.substring(0, index);
        snapshotPool = storagePoolMgr.getStoragePoolByURI(command.getSecondaryStorageUrl() + snapshotPath);
        secondaryPool = storagePoolMgr.getStoragePoolByURI(command.getSecondaryStorageUrl());
        final KVMPhysicalDisk snapshot = snapshotPool.getPhysicalDisk(command.getSnapshotName());
        final String templatePath = secondaryPool.getLocalPath() + File.separator + templateInstallFolder;
        final StorageLayer storage = libvirtComputingResource.getStorage();
        storage.mkdirs(templatePath);
        final String tmplPath = templateInstallFolder + File.separator + tmplFileName;
        final String createTmplPath = libvirtComputingResource.createTmplPath();
        final int cmdsTimeout = libvirtComputingResource.getCmdsTimeout();
        final Script scriptCommand = new Script(createTmplPath, cmdsTimeout, s_logger);
        scriptCommand.add("-t", templatePath);
        scriptCommand.add("-n", tmplFileName);
        scriptCommand.add("-f", snapshot.getPath());
        scriptCommand.execute();
        final Processor qcow2Processor = libvirtUtilitiesHelper.buildQCOW2Processor(storage);
        final FormatInfo info = qcow2Processor.process(templatePath, null, tmplName);
        final TemplateLocation loc = libvirtUtilitiesHelper.buildTemplateLocation(storage, templatePath);
        loc.create(1, true, tmplName);
        loc.addFormat(info);
        loc.save();
        return new CreatePrivateTemplateAnswer(command, true, "", tmplPath, info.virtualSize, info.size, tmplName, info.format);
    } catch (final ConfigurationException e) {
        return new CreatePrivateTemplateAnswer(command, false, e.getMessage());
    } catch (final InternalErrorException e) {
        return new CreatePrivateTemplateAnswer(command, false, e.getMessage());
    } catch (final IOException e) {
        return new CreatePrivateTemplateAnswer(command, false, e.getMessage());
    } catch (final CloudRuntimeException e) {
        return new CreatePrivateTemplateAnswer(command, false, e.getMessage());
    } finally {
        if (secondaryPool != null) {
            storagePoolMgr.deleteStoragePool(secondaryPool.getType(), secondaryPool.getUuid());
        }
        if (snapshotPool != null) {
            storagePoolMgr.deleteStoragePool(snapshotPool.getType(), snapshotPool.getUuid());
        }
    }
}

Example 15 with Processor

use of com.cloud.storage.template.Processor in project cloudstack by apache.

the class NfsSecondaryStorageResource method postUpload.

public String postUpload(String uuid, String filename) {
    UploadEntity uploadEntity = uploadEntityStateMap.get(uuid);
    int installTimeoutPerGig = 180 * 60 * 1000;
    String resourcePath = uploadEntity.getInstallPathPrefix();
    // template download
    String finalResourcePath = uploadEntity.getTmpltPath();
    UploadEntity.ResourceType resourceType = uploadEntity.getResourceType();
    String fileSavedTempLocation = uploadEntity.getInstallPathPrefix() + "/" + filename;
    String uploadedFileExtension = FilenameUtils.getExtension(filename);
    String userSelectedFormat = uploadEntity.getFormat().toString();
    if (uploadedFileExtension.equals("zip") || uploadedFileExtension.equals("bz2") || uploadedFileExtension.equals("gz")) {
        userSelectedFormat += "." + uploadedFileExtension;
    }
    String formatError = ImageStoreUtil.checkTemplateFormat(fileSavedTempLocation, userSelectedFormat);
    if (StringUtils.isNotBlank(formatError)) {
        String errorString = "File type mismatch between uploaded file and selected format. Selected file format: " + userSelectedFormat + ". Received: " + formatError;
        s_logger.error(errorString);
        return errorString;
    }
    int imgSizeGigs = getSizeInGB(_storage.getSize(fileSavedTempLocation));
    int maxSize = uploadEntity.getMaxSizeInGB();
    if (imgSizeGigs > maxSize) {
        String errorMessage = "Maximum file upload size exceeded. Physical file size: " + imgSizeGigs + "GB. Maximum allowed size: " + maxSize + "GB.";
        s_logger.error(errorMessage);
        return errorMessage;
    }
    // add one just in case
    imgSizeGigs++;
    long timeout = (long) imgSizeGigs * installTimeoutPerGig;
    Script scr = new Script(getScriptLocation(resourceType), timeout, s_logger);
    scr.add("-s", Integer.toString(imgSizeGigs));
    scr.add("-S", Long.toString(UploadEntity.s_maxTemplateSize));
    if (uploadEntity.getDescription() != null && uploadEntity.getDescription().length() > 1) {
        scr.add("-d", uploadEntity.getDescription());
    }
    if (uploadEntity.isHvm()) {
        scr.add("-h");
    }
    String checkSum = uploadEntity.getChksum();
    if (StringUtils.isNotBlank(checkSum)) {
        scr.add("-c", checkSum);
    }
    // add options common to ISO and template
    String extension = uploadEntity.getFormat().getFileExtension();
    String templateName = "";
    if (extension.equals("iso")) {
        templateName = uploadEntity.getUuid().trim().replace(" ", "_");
    } else {
        try {
            templateName = UUID.nameUUIDFromBytes((uploadEntity.getFilename() + System.currentTimeMillis()).getBytes("UTF-8")).toString();
        } catch (UnsupportedEncodingException e) {
            templateName = uploadEntity.getUuid().trim().replace(" ", "_");
        }
    }
    // run script to mv the temporary template file to the final template
    // file
    String templateFilename = templateName + "." + extension;
    uploadEntity.setTemplatePath(finalResourcePath + "/" + templateFilename);
    scr.add("-n", templateFilename);
    scr.add("-t", resourcePath);
    // this is the temporary
    scr.add("-f", fileSavedTempLocation);
    // template file downloaded
    if (uploadEntity.getChksum() != null && uploadEntity.getChksum().length() > 1) {
        scr.add("-c", uploadEntity.getChksum());
    }
    // cleanup
    scr.add("-u");
    String result;
    result = scr.execute();
    if (result != null) {
        return result;
    }
    // Set permissions for the downloaded template
    File downloadedTemplate = new File(resourcePath + "/" + templateFilename);
    _storage.setWorldReadableAndWriteable(downloadedTemplate);
    // Set permissions for template/volume.properties
    String propertiesFile = resourcePath;
    if (resourceType == UploadEntity.ResourceType.TEMPLATE) {
        propertiesFile += "/template.properties";
    } else {
        propertiesFile += "/volume.properties";
    }
    File templateProperties = new File(propertiesFile);
    _storage.setWorldReadableAndWriteable(templateProperties);
    TemplateLocation loc = new TemplateLocation(_storage, resourcePath);
    try {
        loc.create(uploadEntity.getEntityId(), true, uploadEntity.getFilename());
    } catch (IOException e) {
        s_logger.warn("Something is wrong with template location " + resourcePath, e);
        loc.purge();
        return "Unable to upload due to " + e.getMessage();
    }
    Map<String, Processor> processors = _dlMgr.getProcessors();
    for (Processor processor : processors.values()) {
        FormatInfo info = null;
        try {
            info = processor.process(resourcePath, null, templateName);
        } catch (InternalErrorException e) {
            s_logger.error("Template process exception ", e);
            return e.toString();
        }
        if (info != null) {
            loc.addFormat(info);
            uploadEntity.setVirtualSize(info.virtualSize);
            uploadEntity.setPhysicalSize(info.size);
            break;
        }
    }
    if (!loc.save()) {
        s_logger.warn("Cleaning up because we're unable to save the formats");
        loc.purge();
    }
    uploadEntity.setStatus(UploadEntity.Status.COMPLETED);
    uploadEntityStateMap.put(uploadEntity.getUuid(), uploadEntity);
    return null;
}