Examples with TemplateDownloader com.cloud.storage.template.TemplateDownloader used on opensource projects

Example 1 with TemplateDownloader

use of com.cloud.storage.template.TemplateDownloader in project cloudstack by apache.

the class DownloadManagerImpl method postLocalDownload.

/**
     * Post local download activity (install and cleanup). Executed in context of
     * downloader thread
     *
     * @throws IOException
     */
private String postLocalDownload(String jobId) {
    DownloadJob dnld = jobs.get(jobId);
    TemplateDownloader td = dnld.getTemplateDownloader();
    // path with mount
    String resourcePath = dnld.getInstallPathPrefix();
    // directory
    // template download
    String finalResourcePath = dnld.getTmpltPath();
    // path on secondary
    // storage
    ResourceType resourceType = dnld.getResourceType();
    File originalTemplate = new File(td.getDownloadLocalPath());
    String checkSum = computeCheckSum(originalTemplate);
    if (checkSum == null) {
        s_logger.warn("Something wrong happened when try to calculate the checksum of downloaded template!");
    }
    dnld.setCheckSum(checkSum);
    int imgSizeGigs = (int) Math.ceil(_storage.getSize(td.getDownloadLocalPath()) * 1.0d / (1024 * 1024 * 1024));
    // add one just in case
    imgSizeGigs++;
    long timeout = (long) imgSizeGigs * installTimeoutPerGig;
    Script scr = null;
    String script = resourceType == ResourceType.TEMPLATE ? createTmpltScr : createVolScr;
    scr = new Script(script, timeout, s_logger);
    scr.add("-s", Integer.toString(imgSizeGigs));
    scr.add("-S", Long.toString(td.getMaxTemplateSizeInBytes()));
    if (dnld.getDescription() != null && dnld.getDescription().length() > 1) {
        scr.add("-d", dnld.getDescription());
    }
    if (dnld.isHvm()) {
        scr.add("-h");
    }
    // add options common to ISO and template
    String extension = dnld.getFormat().getFileExtension();
    String templateName = "";
    if (extension.equals("iso")) {
        templateName = jobs.get(jobId).getTmpltName().trim().replace(" ", "_");
    } else {
        templateName = java.util.UUID.nameUUIDFromBytes((jobs.get(jobId).getTmpltName() + System.currentTimeMillis()).getBytes(StringUtils.getPreferredCharset())).toString();
    }
    // run script to mv the temporary template file to the final template
    // file
    String templateFilename = templateName + "." + extension;
    dnld.setTmpltPath(finalResourcePath + "/" + templateFilename);
    scr.add("-n", templateFilename);
    scr.add("-t", resourcePath);
    // this is the temporary
    scr.add("-f", td.getDownloadLocalPath());
    // template file downloaded
    if (dnld.getChecksum() != null && dnld.getChecksum().length() > 1) {
        scr.add("-c", dnld.getChecksum());
    }
    // cleanup
    scr.add("-u");
    String result;
    result = scr.execute();
    if (result != null) {
        return result;
    }
    // Set permissions for the downloaded template
    File downloadedTemplate = new File(resourcePath + "/" + templateFilename);
    _storage.setWorldReadableAndWriteable(downloadedTemplate);
    // Set permissions for template/volume.properties
    String propertiesFile = resourcePath;
    if (resourceType == ResourceType.TEMPLATE) {
        propertiesFile += "/template.properties";
    } else {
        propertiesFile += "/volume.properties";
    }
    File templateProperties = new File(propertiesFile);
    _storage.setWorldReadableAndWriteable(templateProperties);
    TemplateLocation loc = new TemplateLocation(_storage, resourcePath);
    try {
        loc.create(dnld.getId(), true, dnld.getTmpltName());
    } catch (IOException e) {
        s_logger.warn("Something is wrong with template location " + resourcePath, e);
        loc.purge();
        return "Unable to download due to " + e.getMessage();
    }
    Iterator<Processor> en = _processors.values().iterator();
    while (en.hasNext()) {
        Processor processor = en.next();
        FormatInfo info = null;
        try {
            info = processor.process(resourcePath, null, templateName);
        } catch (InternalErrorException e) {
            s_logger.error("Template process exception ", e);
            return e.toString();
        }
        if (info != null) {
            if (!loc.addFormat(info)) {
                loc.purge();
                return "Unable to install due to invalid file format";
            }
            dnld.setTemplatesize(info.virtualSize);
            dnld.setTemplatePhysicalSize(info.size);
            break;
        }
    }
    if (!loc.save()) {
        s_logger.warn("Cleaning up because we're unable to save the formats");
        loc.purge();
    }
    return null;
}

Example 2 with TemplateDownloader

use of com.cloud.storage.template.TemplateDownloader in project cloudstack by apache.

the class DownloadManagerImpl method setDownloadStatus.

/**
     * Get notified of change of job status. Executed in context of downloader
     * thread
     *
     * @param jobId
     *            the id of the job
     * @param status
     *            the status of the job
     */
public void setDownloadStatus(String jobId, Status status) {
    DownloadJob dj = jobs.get(jobId);
    if (dj == null) {
        s_logger.warn("setDownloadStatus for jobId: " + jobId + ", status=" + status + " no job found");
        return;
    }
    TemplateDownloader td = dj.getTemplateDownloader();
    s_logger.info("Download Completion for jobId: " + jobId + ", status=" + status);
    s_logger.info("local: " + td.getDownloadLocalPath() + ", bytes=" + td.getDownloadedBytes() + ", error=" + td.getDownloadError() + ", pct=" + td.getDownloadPercent());
    switch(status) {
        case ABORTED:
        case NOT_STARTED:
        case UNRECOVERABLE_ERROR:
            // TODO
            dj.cleanup();
            break;
        case UNKNOWN:
            return;
        case IN_PROGRESS:
            s_logger.info("Resuming jobId: " + jobId + ", status=" + status);
            td.setResume(true);
            threadPool.execute(td);
            break;
        case RECOVERABLE_ERROR:
            threadPool.execute(td);
            break;
        case DOWNLOAD_FINISHED:
            if (td instanceof S3TemplateDownloader) {
                // For S3 and Swift, which are considered "remote",
                // as in the file cannot be accessed locally,
                // we run the postRemoteDownload() method.
                td.setDownloadError("Download success, starting install ");
                String result = postRemoteDownload(jobId);
                if (result != null) {
                    s_logger.error("Failed post download install: " + result);
                    td.setStatus(Status.UNRECOVERABLE_ERROR);
                    td.setDownloadError("Failed post download install: " + result);
                    ((S3TemplateDownloader) td).cleanupAfterError();
                } else {
                    td.setStatus(Status.POST_DOWNLOAD_FINISHED);
                    td.setDownloadError("Install completed successfully at " + new SimpleDateFormat().format(new Date()));
                }
            } else {
                // For other TemplateDownloaders where files are locally available,
                // we run the postLocalDownload() method.
                td.setDownloadError("Download success, starting install ");
                String result = postLocalDownload(jobId);
                if (result != null) {
                    s_logger.error("Failed post download script: " + result);
                    td.setStatus(Status.UNRECOVERABLE_ERROR);
                    td.setDownloadError("Failed post download script: " + result);
                } else {
                    td.setStatus(Status.POST_DOWNLOAD_FINISHED);
                    td.setDownloadError("Install completed successfully at " + new SimpleDateFormat().format(new Date()));
                }
            }
            dj.cleanup();
            break;
        default:
            break;
    }
}

Example 3 with TemplateDownloader

use of com.cloud.storage.template.TemplateDownloader in project cloudstack by apache.

the class DownloadManagerImpl method downloadS3Template.

@Override
public String downloadS3Template(S3TO s3, long id, String url, String name, ImageFormat format, boolean hvm, Long accountId, String descr, String cksum, String installPathPrefix, String user, String password, long maxTemplateSizeInBytes, Proxy proxy, ResourceType resourceType) {
    UUID uuid = UUID.randomUUID();
    String jobId = uuid.toString();
    URI uri;
    try {
        uri = new URI(url);
    } catch (URISyntaxException e) {
        throw new CloudRuntimeException("URI is incorrect: " + url);
    }
    TemplateDownloader td;
    if ((uri != null) && (uri.getScheme() != null)) {
        if (uri.getScheme().equalsIgnoreCase("http") || uri.getScheme().equalsIgnoreCase("https")) {
            td = new S3TemplateDownloader(s3, url, installPathPrefix, new Completion(jobId), maxTemplateSizeInBytes, user, password, proxy, resourceType);
        } else {
            throw new CloudRuntimeException("Scheme is not supported " + url);
        }
    } else {
        throw new CloudRuntimeException("Unable to download from URL: " + url);
    }
    DownloadJob dj = new DownloadJob(td, jobId, id, name, format, hvm, accountId, descr, cksum, installPathPrefix, resourceType);
    dj.setTmpltPath(installPathPrefix);
    jobs.put(jobId, dj);
    threadPool.execute(td);
    return jobId;
}

Example 4 with TemplateDownloader

use of com.cloud.storage.template.TemplateDownloader in project cloudstack by apache.

the class DownloadManagerImpl method downloadPublicTemplate.

@Override
public String downloadPublicTemplate(long id, String url, String name, ImageFormat format, boolean hvm, Long accountId, String descr, String cksum, String installPathPrefix, String templatePath, String user, String password, long maxTemplateSizeInBytes, Proxy proxy, ResourceType resourceType) {
    UUID uuid = UUID.randomUUID();
    String jobId = uuid.toString();
    String tmpDir = installPathPrefix;
    try {
        if (!_storage.mkdirs(tmpDir)) {
            s_logger.warn("Unable to create " + tmpDir);
            return "Unable to create " + tmpDir;
        }
        // TO DO - define constant for volume properties.
        File file = ResourceType.TEMPLATE == resourceType ? _storage.getFile(tmpDir + File.separator + TemplateLocation.Filename) : _storage.getFile(tmpDir + File.separator + "volume.properties");
        if (file.exists()) {
            if (!file.delete()) {
                s_logger.warn("Deletion of file '" + file.getAbsolutePath() + "' failed.");
            }
        }
        if (!file.createNewFile()) {
            s_logger.warn("Unable to create new file: " + file.getAbsolutePath());
            return "Unable to create new file: " + file.getAbsolutePath();
        }
        URI uri;
        try {
            uri = new URI(url);
        } catch (URISyntaxException e) {
            throw new CloudRuntimeException("URI is incorrect: " + url);
        }
        TemplateDownloader td;
        if ((uri != null) && (uri.getScheme() != null)) {
            if (uri.getScheme().equalsIgnoreCase("http") || uri.getScheme().equalsIgnoreCase("https")) {
                td = new HttpTemplateDownloader(_storage, url, tmpDir, new Completion(jobId), maxTemplateSizeInBytes, user, password, proxy, resourceType);
            } else if (uri.getScheme().equalsIgnoreCase("file")) {
                td = new LocalTemplateDownloader(_storage, url, tmpDir, maxTemplateSizeInBytes, new Completion(jobId));
            } else if (uri.getScheme().equalsIgnoreCase("scp")) {
                td = new ScpTemplateDownloader(_storage, url, tmpDir, maxTemplateSizeInBytes, new Completion(jobId));
            } else if (uri.getScheme().equalsIgnoreCase("nfs") || uri.getScheme().equalsIgnoreCase("cifs")) {
                td = null;
                // TODO: implement this.
                throw new CloudRuntimeException("Scheme is not supported " + url);
            } else {
                throw new CloudRuntimeException("Scheme is not supported " + url);
            }
        } else {
            throw new CloudRuntimeException("Unable to download from URL: " + url);
        }
        // NOTE the difference between installPathPrefix and templatePath
        // here. instalPathPrefix is the absolute path for template
        // including mount directory
        // on ssvm, while templatePath is the final relative path on
        // secondary storage.
        DownloadJob dj = new DownloadJob(td, jobId, id, name, format, hvm, accountId, descr, cksum, installPathPrefix, resourceType);
        dj.setTmpltPath(templatePath);
        jobs.put(jobId, dj);
        threadPool.execute(td);
        return jobId;
    } catch (IOException e) {
        s_logger.warn("Unable to download to " + tmpDir, e);
        return null;
    }
}

Example 5 with TemplateDownloader

use of com.cloud.storage.template.TemplateDownloader in project cloudstack by apache.

the class DownloadManagerImpl method handleDownloadProgressCmd.

private DownloadAnswer handleDownloadProgressCmd(SecondaryStorageResource resource, DownloadProgressCommand cmd) {
    String jobId = cmd.getJobId();
    DownloadAnswer answer;
    DownloadJob dj = null;
    if (jobId != null) {
        dj = jobs.get(jobId);
    }
    if (dj == null) {
        if (cmd.getRequest() == RequestType.GET_OR_RESTART) {
            DownloadCommand dcmd = new DownloadCommand(cmd);
            return handleDownloadCommand(resource, dcmd);
        } else {
            return new DownloadAnswer("Cannot find job", com.cloud.storage.VMTemplateStorageResourceAssoc.Status.UNKNOWN);
        }
    }
    TemplateDownloader td = dj.getTemplateDownloader();
    switch(cmd.getRequest()) {
        case GET_STATUS:
            break;
        case ABORT:
            td.stopDownload();
            sleep();
            break;
        case RESTART:
            td.stopDownload();
            sleep();
            threadPool.execute(td);
            break;
        case PURGE:
            td.stopDownload();
            answer = new DownloadAnswer(jobId, getDownloadPct(jobId), getDownloadError(jobId), getDownloadStatus2(jobId), getDownloadLocalPath(jobId), getInstallPath(jobId), getDownloadTemplateSize(jobId), getDownloadTemplatePhysicalSize(jobId), getDownloadCheckSum(jobId));
            jobs.remove(jobId);
            return answer;
        default:
            // TODO
            break;
    }
    return new DownloadAnswer(jobId, getDownloadPct(jobId), getDownloadError(jobId), getDownloadStatus2(jobId), getDownloadLocalPath(jobId), getInstallPath(jobId), getDownloadTemplateSize(jobId), getDownloadTemplatePhysicalSize(jobId), getDownloadCheckSum(jobId));
}